Emoji loading fixes (#37300)

app/javascript/mastodon/features/emoji/database.ts

@@ -5,11 +5,7 @@ import { openDB } from 'idb';
 
 import { EMOJI_DB_SHORTCODE_TEST } from './constants';
 import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
-import type {
+import type { CustomEmojiData, UnicodeEmojiData, EtagTypes } from './types';
-  CustomEmojiData,
-  UnicodeEmojiData,
-  LocaleOrCustom,
-} from './types';
 import { emojiLogger } from './utils';
 
 interface EmojiDB extends LocaleTables, DBSchema {
@@ -32,7 +28,7 @@ interface EmojiDB extends LocaleTables, DBSchema {
     };
   };
   etags: {
-    key: LocaleOrCustom;
+    key: EtagTypes;
     value: string;
   };
 }
@@ -197,10 +193,9 @@ export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) {
   await trx.done;
 }
 
-export async function putLatestEtag(etag: string, localeString: string) {
+export async function putLatestEtag(etag: string, name: EtagTypes) {
-  const locale = toSupportedLocaleOrCustom(localeString);
   const db = await loadDB();
-  await db.put('etags', etag, locale);
+  await db.put('etags', etag, name);
 }
 
 export async function clearEtag(localeString: string) {

app/javascript/mastodon/features/emoji/index.ts

@@ -1,13 +1,9 @@
-import type { Locale } from 'emojibase';
-
 import { initialState } from '@/mastodon/initial_state';
 
-import type { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
+import type { EMOJI_DB_NAME_SHORTCODES } from './constants';
 import { toSupportedLocale } from './locale';
 import type { LocaleOrCustom } from './types';
 import { emojiLogger } from './utils';
-// eslint-disable-next-line import/default -- Importing via worker loader.
-import EmojiWorker from './worker?worker&inline';
 
 const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
 
@@ -18,13 +14,14 @@ const log = emojiLogger('index');
 // This is too short, but better to fallback quickly than wait.
 const WORKER_TIMEOUT = 1_000;
 
-export function initializeEmoji() {
+export async function initializeEmoji() {
   log('initializing emojis');
 
   // Create a temp worker, and assign it to the module-level worker once we know it's ready.
   let tempWorker: Worker | null = null;
   if (!worker && 'Worker' in window) {
     try {
+      const { default: EmojiWorker } = await import('./worker?worker&inline');
       tempWorker = new EmojiWorker();
     } catch (err) {
       console.warn('Error creating web worker:', err);
@@ -64,7 +61,7 @@ async function fallbackLoad() {
   await loadCustomEmoji();
   const { importLegacyShortcodes } = await import('./loader');
   const shortcodes = await importLegacyShortcodes();
-  if (shortcodes.length) {
+  if (shortcodes?.length) {
     log('loaded %d legacy shortcodes', shortcodes.length);
   }
   await loadEmojiLocale(userLocale);
@@ -72,14 +69,11 @@ async function fallbackLoad() {
 
 async function loadEmojiLocale(localeString: string) {
   const locale = toSupportedLocale(localeString);
-  const { importEmojiData, localeToEmojiPath, localeToShortcodesPath } =
+  const { importEmojiData } = await import('./loader');
-    await import('./loader');
 
   if (worker) {
-    const path = await localeToEmojiPath(locale);
+    log('asking worker to load locale %s', locale);
-    const shortcodesPath = await localeToShortcodesPath(locale);
+    messageWorker(locale);
-    log('asking worker to load locale %s from %s', locale, path);
-    messageWorker(locale, path, shortcodesPath);
   } else {
     const emojis = await importEmojiData(locale);
     if (emojis) {
@@ -100,17 +94,11 @@ export async function loadCustomEmoji() {
   }
 }
 
-function messageWorker(
-  locale: typeof EMOJI_TYPE_CUSTOM | typeof EMOJI_DB_NAME_SHORTCODES,
-): void;
-function messageWorker(locale: Locale, path: string, shortcodes?: string): void;
 function messageWorker(
   locale: LocaleOrCustom | typeof EMOJI_DB_NAME_SHORTCODES,
-  path?: string,
-  shortcodes?: string,
 ) {
   if (!worker) {
     return;
   }
-  worker.postMessage({ locale, path, shortcodes });
+  worker.postMessage({ locale });
 }

app/javascript/mastodon/features/emoji/loader.ts

@@ -13,46 +13,35 @@ import {
   putLatestEtag,
   putLegacyShortcodes,
 } from './database';
-import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
+import { toSupportedLocale, toValidEtagName } from './locale';
 import type { CustomEmojiData } from './types';
+import { emojiLogger } from './utils';
 
-export async function importEmojiData(
+const log = emojiLogger('loader');
-  localeString: string,
+
-  path?: string,
+export async function importEmojiData(localeString: string, shortcodes = true) {
-  shortcodes: boolean | string = true,
-) {
   const locale = toSupportedLocale(localeString);
 
-  // Validate the provided path.
+  log(
-  if (path && !/^[/a-z]*\/packs\/assets\/compact-\w+\.json$/.test(path)) {
+    'importing emoji data for locale %s%s',
-    throw new Error('Invalid path for emoji data');
+    locale,
-  } else {
+    shortcodes ? ' and shortcodes' : '',
-    // Otherwise get the path if not provided.
+  );
-    path ??= await localeToEmojiPath(locale);
-  }
 
-  const emojis = await fetchAndCheckEtag<CompactEmoji[]>(locale, path);
+  const emojis = await fetchAndCheckEtag<CompactEmoji[]>({
+    etagString: locale,
+    path: localeToEmojiPath(locale),
+  });
   if (!emojis) {
     return;
   }
 
   const shortcodesData: ShortcodesDataset[] = [];
   if (shortcodes) {
-    if (
+    const shortcodesResponse = await fetchAndCheckEtag<ShortcodesDataset>({
-      typeof shortcodes === 'string' &&
+      etagString: `${locale}-shortcodes`,
-      !/^[/a-z]*\/packs\/assets\/shortcodes\/cldr\.json$/.test(shortcodes)
+      path: localeToShortcodesPath(locale),
-    ) {
+    });
-      throw new Error('Invalid path for shortcodes data');
-    }
-    const shortcodesPath =
-      typeof shortcodes === 'string'
-        ? shortcodes
-        : await localeToShortcodesPath(locale);
-    const shortcodesResponse = await fetchAndCheckEtag<ShortcodesDataset>(
-      locale,
-      shortcodesPath,
-      false,
-    );
     if (shortcodesResponse) {
       shortcodesData.push(shortcodesResponse);
     } else {
@@ -69,10 +58,10 @@ export async function importEmojiData(
 }
 
 export async function importCustomEmojiData() {
-  const emojis = await fetchAndCheckEtag<CustomEmojiData[]>(
+  const emojis = await fetchAndCheckEtag<CustomEmojiData[]>({
-    'custom',
+    etagString: 'custom',
-    '/api/v1/custom_emojis',
+    path: '/api/v1/custom_emojis',
-  );
+  });
   if (!emojis) {
     return;
   }
@@ -81,76 +70,76 @@ export async function importCustomEmojiData() {
 }
 
 export async function importLegacyShortcodes() {
-  const { default: shortcodesPath } =
+  const globPaths = import.meta.glob<string>(
-    await import('emojibase-data/en/shortcodes/iamcal.json?url');
+    // We use import.meta.glob to eagerly load the URL, as the regular import() doesn't work inside the Web Worker.
-  const response = await fetch(shortcodesPath);
+    '../../../../../node_modules/emojibase-data/en/shortcodes/iamcal.json',
-  if (!response.ok) {
+    { eager: true, import: 'default', query: '?url' },
-    throw new Error(
-      `Failed to fetch legacy shortcodes data: ${response.statusText}`,
   );
+  const path = Object.values(globPaths)[0];
+  if (!path) {
+    throw new Error('IAMCAL shortcodes path not found');
+  }
+  const shortcodesData = await fetchAndCheckEtag<ShortcodesDataset>({
+    checkEtag: true,
+    etagString: 'shortcodes',
+    path,
+  });
+  if (!shortcodesData) {
+    return;
   }
-  const shortcodesData = (await response.json()) as ShortcodesDataset;
   await putLegacyShortcodes(shortcodesData);
   return Object.keys(shortcodesData);
 }
 
-const emojiModules = new Map(
+function localeToEmojiPath(locale: Locale) {
-  Object.entries(
+  const key = `../../../../../node_modules/emojibase-data/${locale}/compact.json`;
-    import.meta.glob<string>(
+  const emojiModules = import.meta.glob<string>(
     '../../../../../node_modules/emojibase-data/**/compact.json',
     {
       query: '?url',
       import: 'default',
+      eager: true,
     },
-    ),
-  ).map(([key, loader]) => {
-    const match = /emojibase-data\/([^/]+)\/compact\.json$/.exec(key);
-    return [match?.at(1) ?? key, loader];
-  }),
   );
-
+  const path = emojiModules[key];
-export function localeToEmojiPath(locale: Locale) {
-  const path = emojiModules.get(locale);
   if (!path) {
     throw new Error(`Unsupported locale: ${locale}`);
   }
-  return path();
+  return path;
 }
 
-const shortcodesModules = new Map(
+function localeToShortcodesPath(locale: Locale) {
-  Object.entries(
+  const key = `../../../../../node_modules/emojibase-data/${locale}/shortcodes/cldr.json`;
-    import.meta.glob<string>(
+  const shortcodesModules = import.meta.glob<string>(
     '../../../../../node_modules/emojibase-data/**/shortcodes/cldr.json',
     {
       query: '?url',
       import: 'default',
+      eager: true,
     },
-    ),
-  ).map(([key, loader]) => {
-    const match = /emojibase-data\/([^/]+)\/shortcodes\/cldr\.json$/.exec(key);
-    return [match?.at(1) ?? key, loader];
-  }),
   );
-
+  const path = shortcodesModules[key];
-export function localeToShortcodesPath(locale: Locale) {
-  const path = shortcodesModules.get(locale);
   if (!path) {
     throw new Error(`Unsupported locale for shortcodes: ${locale}`);
   }
-  return path();
+  return path;
 }
 
-export async function fetchAndCheckEtag<ResultType extends object[] | object>(
+async function fetchAndCheckEtag<ResultType extends object[] | object>({
-  localeString: string,
+  etagString,
-  path: string,
+  path,
-  checkEtag = true,
+  checkEtag = false,
-): Promise<ResultType | null> {
+}: {
-  const locale = toSupportedLocaleOrCustom(localeString);
+  etagString: string;
+  path: string;
+  checkEtag?: boolean;
+}): Promise<ResultType | null> {
+  const etagName = toValidEtagName(etagString);
 
   // Use location.origin as this script may be loaded from a CDN domain.
   const url = new URL(path, location.origin);
 
-  const oldEtag = checkEtag ? await loadLatestEtag(locale) : null;
+  const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
   const response = await fetch(url, {
     headers: {
       'Content-Type': 'application/json',
@@ -163,7 +152,7 @@ export async function fetchAndCheckEtag<ResultType extends object[] | object>(
   }
   if (!response.ok) {
     throw new Error(
-      `Failed to fetch emoji data for ${locale}: ${response.statusText}`,
+      `Failed to fetch emoji data for ${etagName}: ${response.statusText}`,
     );
   }
 
@@ -172,7 +161,8 @@ export async function fetchAndCheckEtag<ResultType extends object[] | object>(
   // Store the ETag for future requests
   const etag = response.headers.get('ETag');
   if (etag && checkEtag) {
-    await putLatestEtag(etag, localeString);
+    log(`storing new etag for ${etagName}: ${etag}`);
+    await putLatestEtag(etag, etagName);
   }
 
   return data;

app/javascript/mastodon/features/emoji/locale.ts

@@ -1,7 +1,8 @@
 import type { Locale } from 'emojibase';
 import { SUPPORTED_LOCALES } from 'emojibase';
 
-import type { LocaleOrCustom } from './types';
+import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
+import type { EtagTypes, LocaleOrCustom, LocaleWithShortcodes } from './types';
 
 export function toSupportedLocale(localeBase: string): Locale {
   const locale = localeBase.toLowerCase();
@@ -12,12 +13,35 @@ export function toSupportedLocale(localeBase: string): Locale {
 }
 
 export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom {
-  if (locale.toLowerCase() === 'custom') {
+  if (locale.toLowerCase() === EMOJI_TYPE_CUSTOM) {
-    return 'custom';
+    return EMOJI_TYPE_CUSTOM;
   }
   return toSupportedLocale(locale);
 }
 
-function isSupportedLocale(locale: string): locale is Locale {
+export function toValidEtagName(input: string): EtagTypes {
-  return SUPPORTED_LOCALES.includes(locale.toLowerCase() as Locale);
+  const lower = input.toLowerCase();
+  if (lower === EMOJI_TYPE_CUSTOM || lower === EMOJI_DB_NAME_SHORTCODES) {
+    return lower;
+  }
+
+  if (isLocaleWithShortcodes(lower)) {
+    return lower;
+  }
+
+  return toSupportedLocale(lower);
+}
+
+function isSupportedLocale(locale: string): locale is Locale {
+  return SUPPORTED_LOCALES.includes(locale as Locale);
+}
+
+function isLocaleWithShortcodes(input: string): input is LocaleWithShortcodes {
+  const [baseLocale, shortcodes] = input.split('-');
+  return (
+    !!baseLocale &&
+    !!shortcodes &&
+    isSupportedLocale(baseLocale) &&
+    shortcodes === EMOJI_DB_NAME_SHORTCODES
+  );
 }

app/javascript/mastodon/features/emoji/render.ts

@@ -4,12 +4,6 @@ import {
   EMOJI_TYPE_UNICODE,
   EMOJI_TYPE_CUSTOM,
 } from './constants';
-import {
-  loadEmojiByHexcode,
-  loadLegacyShortcodesByShortcode,
-  LocaleNotLoadedError,
-} from './database';
-import { importEmojiData } from './loader';
 import { emojiToUnicodeHex } from './normalize';
 import type {
   EmojiLoadedState,
@@ -121,6 +115,12 @@ export async function loadEmojiDataToState(
     return null;
   }
 
+  const {
+    loadLegacyShortcodesByShortcode,
+    loadEmojiByHexcode,
+    LocaleNotLoadedError,
+  } = await import('./database');
+
   // First, try to load the data from IndexedDB.
   try {
     const legacyCode = await loadLegacyShortcodesByShortcode(state.code);
@@ -155,6 +155,7 @@ export async function loadEmojiDataToState(
         state.code,
         locale,
       );
+      const { importEmojiData } = await import('./loader');
       await importEmojiData(locale); // Use this from the loader file as it can be awaited.
       return loadEmojiDataToState(state, locale, true);
     }

app/javascript/mastodon/features/emoji/types.ts

@@ -7,6 +7,7 @@ import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
 import type { RequiredExcept } from '@/mastodon/utils/types';
 
 import type {
+  EMOJI_DB_NAME_SHORTCODES,
   EMOJI_MODE_NATIVE,
   EMOJI_MODE_NATIVE_WITH_FLAGS,
   EMOJI_MODE_TWEMOJI,
@@ -20,6 +21,11 @@ export type EmojiMode =
   | typeof EMOJI_MODE_TWEMOJI;
 
 export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM;
+export type LocaleWithShortcodes = `${Locale}-shortcodes`;
+export type EtagTypes =
+  | LocaleOrCustom
+  | typeof EMOJI_DB_NAME_SHORTCODES
+  | LocaleWithShortcodes;
 
 export interface EmojiAppState {
   locales: Locale[];

app/javascript/mastodon/features/emoji/worker.ts

@@ -8,24 +8,23 @@ import {
 addEventListener('message', handleMessage);
 self.postMessage('ready'); // After the worker is ready, notify the main thread
 
-function handleMessage(event: MessageEvent<{ locale: string; path?: string }>) {
+function handleMessage(event: MessageEvent<{ locale: string }>) {
   const {
-    data: { locale, path },
+    data: { locale },
   } = event;
-  void loadData(locale, path);
+  void loadData(locale);
 }
 
-async function loadData(locale: string, path?: string) {
+async function loadData(locale: string) {
   let importCount: number | undefined;
   if (locale === EMOJI_TYPE_CUSTOM) {
     importCount = (await importCustomEmojiData())?.length;
   } else if (locale === EMOJI_DB_NAME_SHORTCODES) {
-    importCount = (await importLegacyShortcodes()).length;
+    importCount = (await importLegacyShortcodes())?.length;
-  } else if (path) {
-    importCount = (await importEmojiData(locale, path))?.length;
   } else {
-    throw new Error('Path is required for loading locale emoji data');
+    importCount = (await importEmojiData(locale))?.length;
   }
+
   if (importCount) {
     self.postMessage(`loaded ${importCount} emojis into ${locale}`);
   }

app/javascript/mastodon/main.tsx

@@ -30,7 +30,7 @@ function main() {
     }
 
     const { initializeEmoji } = await import('./features/emoji/index');
-    initializeEmoji();
+    await initializeEmoji();
 
     const root = createRoot(mountNode);
     root.render(<Mastodon {...props} />);