Emoji: Cache data by path instead of just Etag (#37858)

app/javascript/mastodon/actions/importer/emoji.ts

@@ -7,7 +7,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
   }
 
   // First, check if we already have them all.
-  const { searchCustomEmojisByShortcodes, clearEtag } =
+  const { searchCustomEmojisByShortcodes, clearCache } =
     await import('@/mastodon/features/emoji/database');
 
   const existingEmojis = await searchCustomEmojisByShortcodes(
@@ -16,7 +16,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
 
   // If there's a mismatch, re-import all custom emojis.
   if (existingEmojis.length < emojis.length) {
-    await clearEtag('custom');
+    await clearCache('custom');
     await loadCustomEmoji();
   }
 }

app/javascript/mastodon/features/emoji/database.test.ts

@@ -3,7 +3,6 @@ import { IDBFactory } from 'fake-indexeddb';
 
 import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
 
-import { EMOJI_DB_SHORTCODE_TEST } from './constants';
 import {
   putEmojiData,
   loadEmojiByHexcode,
@@ -12,8 +11,6 @@ import {
   putCustomEmojiData,
   putLegacyShortcodes,
   loadLegacyShortcodesByShortcode,
-  loadLatestEtag,
-  putLatestEtag,
 } from './database';
 
 function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
@@ -120,36 +117,4 @@ describe('emoji database', () => {
       ).resolves.toEqual(data);
     });
   });
-
-  describe('loadLatestEtag', () => {
-    beforeEach(async () => {
-      await putLatestEtag('etag', 'en');
-      await putEmojiData([unicodeEmojiFactory()], 'en');
-      await putLatestEtag('fr-etag', 'fr');
-    });
-
-    test('retrieves the etag for loaded locale', async () => {
-      await putEmojiData(
-        [unicodeEmojiFactory({ hexcode: EMOJI_DB_SHORTCODE_TEST })],
-        'en',
-      );
-      const etag = await loadLatestEtag('en');
-      expect(etag).toBe('etag');
-    });
-
-    test('returns null if locale has no shortcodes', async () => {
-      const etag = await loadLatestEtag('en');
-      expect(etag).toBeNull();
-    });
-
-    test('returns null if locale not loaded', async () => {
-      const etag = await loadLatestEtag('de');
-      expect(etag).toBeNull();
-    });
-
-    test('returns null if locale has no data', async () => {
-      const etag = await loadLatestEtag('fr');
-      expect(etag).toBeNull();
-    });
-  });
 });

app/javascript/mastodon/features/emoji/database.ts

@@ -3,21 +3,16 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
 
-import { EMOJI_DB_SHORTCODE_TEST } from './constants';
 import { openEmojiDB } from './db-schema';
 import type { Database } from './db-schema';
-import {
+import { localeToSegmenter, toSupportedLocale } from './locale';
-  localeToSegmenter,
-  toSupportedLocale,
-  toSupportedLocaleOrCustom,
-} from './locale';
 import {
   extractTokens,
   skinHexcodeToEmoji,
   transformCustomEmojiData,
   transformEmojiData,
 } from './normalize';
-import type { AnyEmojiData, EtagTypes } from './types';
+import type { AnyEmojiData, CacheKey } from './types';
 import { emojiLogger } from './utils';
 
 const loadedLocales = new Set<Locale>();
@@ -214,16 +209,21 @@ export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) {
   await trx.done;
 }
 
-export async function putLatestEtag(etag: string, name: EtagTypes) {
+export async function loadCacheValue(key: CacheKey) {
   const db = await loadDB();
-  await db.put('etags', etag, name);
+  const value = await db.get('etags', key);
+  return value;
 }
 
-export async function clearEtag(localeString: string) {
+export async function putCacheValue(key: CacheKey, value: string) {
-  const locale = toSupportedLocaleOrCustom(localeString);
   const db = await loadDB();
-  await db.delete('etags', locale);
+  await db.put('etags', value, key);
-  log('Cleared etag for %s', locale);
+}
+
+export async function clearCache(key: CacheKey) {
+  const db = await loadDB();
+  await db.delete('etags', key);
+  log('Cleared cache for %s', key);
 }
 
 export async function loadEmojiByHexcode(
@@ -276,26 +276,6 @@ export async function loadLegacyShortcodesByShortcode(shortcode: string) {
   );
 }
 
-export async function loadLatestEtag(localeString: string) {
-  const locale = toSupportedLocaleOrCustom(localeString);
-  const db = await loadDB();
-  const rowCount = await db.count(locale);
-  if (!rowCount) {
-    return null; // No data for this locale, return null even if there is an etag.
-  }
-
-  // Check if shortcodes exist for the given Unicode locale.
-  if (locale !== 'custom') {
-    const result = await db.get(locale, EMOJI_DB_SHORTCODE_TEST);
-    if (!result?.shortcodes) {
-      return null;
-    }
-  }
-
-  const etag = await db.get('etags', locale);
-  return etag ?? null;
-}
-
 // Private functions
 
 async function syncLocales(db: Database) {

app/javascript/mastodon/features/emoji/db-schema.ts

@@ -10,7 +10,7 @@ import type {
   StoreNames,
 } from 'idb';
 
-import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types';
+import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types';
 import { emojiLogger } from './utils';
 
 const log = emojiLogger('database');
@@ -35,7 +35,7 @@ interface EmojiDB extends LocaleTables, DBSchema {
     };
   };
   etags: {
-    key: EtagTypes;
+    key: CacheKey;
     value: string;
   };
 }

app/javascript/mastodon/features/emoji/loader.ts

@@ -4,11 +4,11 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
 import {
   putEmojiData,
   putCustomEmojiData,
-  loadLatestEtag,
+  putCacheValue,
-  putLatestEtag,
   putLegacyShortcodes,
+  loadCacheValue,
 } from './database';
-import { toSupportedLocale, toValidEtagName } from './locale';
+import { toSupportedLocale, toValidCacheKey } from './locale';
 import type { CustomEmojiData } from './types';
 import { emojiLogger } from './utils';
 
@@ -23,8 +23,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
     shortcodes ? ' and shortcodes' : '',
   );
 
-  let emojis = await fetchAndCheckEtag<CompactEmoji[]>({
+  let emojis = await fetchIfNotLoaded<CompactEmoji[]>({
-    etagString: locale,
+    key: locale,
     path: localeToEmojiPath(locale),
   });
   if (!emojis) {
@@ -33,8 +33,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
 
   const shortcodesData: ShortcodesDataset[] = [];
   if (shortcodes) {
-    const shortcodesResponse = await fetchAndCheckEtag<ShortcodesDataset>({
+    const shortcodesResponse = await fetchIfNotLoaded<ShortcodesDataset>({
-      etagString: `${locale}-shortcodes`,
+      key: `${locale}-shortcodes`,
       path: localeToShortcodesPath(locale),
     });
     if (shortcodesResponse) {
@@ -51,13 +51,24 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
 }
 
 export async function importCustomEmojiData() {
-  const emojis = await fetchAndCheckEtag<CustomEmojiData[]>({
+  const response = await fetchAndCheckEtag({
-    etagString: 'custom',
+    oldEtag: await loadCacheValue('custom'),
     path: '/api/v1/custom_emojis',
   });
-  if (!emojis) {
+
+  if (!response) {
     return;
   }
+
+  const etag = response.headers.get('ETag');
+  if (etag) {
+    log('Custom emoji data fetched successfully, storing etag %s', etag);
+    await putCacheValue('custom', etag);
+  } else {
+    log('No etag found in response for custom emoji data');
+  }
+
+  const emojis = (await response.json()) as CustomEmojiData[];
   await putCustomEmojiData({ emojis, clear: true });
   return emojis;
 }
@@ -72,9 +83,8 @@ export async function importLegacyShortcodes() {
   if (!path) {
     throw new Error('IAMCAL shortcodes path not found');
   }
-  const shortcodesData = await fetchAndCheckEtag<ShortcodesDataset>({
+  const shortcodesData = await fetchIfNotLoaded<ShortcodesDataset>({
-    checkEtag: true,
+    key: 'shortcodes',
-    etagString: 'shortcodes',
     path,
   });
   if (!shortcodesData) {
@@ -118,48 +128,64 @@ function localeToShortcodesPath(locale: Locale) {
   return path;
 }
 
-async function fetchAndCheckEtag<ResultType extends object[] | object>({
+async function fetchIfNotLoaded<ResultType extends object[] | object>({
-  etagString,
+  key: rawKey,
   path,
-  checkEtag = false,
 }: {
-  etagString: string;
+  key: string;
   path: string;
-  checkEtag?: boolean;
 }): Promise<ResultType | null> {
-  const etagName = toValidEtagName(etagString);
+  const key = toValidCacheKey(rawKey);
-  const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
+
+  const value = await loadCacheValue(key);
+
+  if (value === path) {
+    log('data for %s already loaded, skipping fetch', key);
+    return null;
+  }
+
+  const response = await fetchAndCheckEtag({ path });
+  if (!response) {
+    return null;
+  }
+
+  log('data for %s fetched successfully, storing etag', key);
+  await putCacheValue(key, path);
+
+  return (await response.json()) as ResultType;
+}
+
+async function fetchAndCheckEtag({
+  oldEtag,
+  path,
+}: {
+  oldEtag?: string;
+  path: string;
+}) {
+  const headers = new Headers({
+    'Content-Type': 'application/json',
+  });
+  if (oldEtag) {
+    headers.set('If-None-Match', oldEtag);
+  }
 
   // Use location.origin as this script may be loaded from a CDN domain.
   const url = new URL(path, location.origin);
-
   const response = await fetch(url, {
-    headers: {
+    headers,
-      'Content-Type': 'application/json',
-      'If-None-Match': oldEtag ?? '', // Send the old ETag to check for modifications
-    },
   });
+
   // If not modified, return null
   if (response.status === 304) {
-    log('etag not modified for %s', etagName);
+    log('etag not modified for %s', path);
     return null;
   }
+
   if (!response.ok) {
     throw new Error(
-      `Failed to fetch emoji data for ${etagName}: ${response.statusText}`,
+      `Failed to fetch emoji data for ${path}: ${response.statusText}`,
     );
   }
 
-  const data = (await response.json()) as ResultType;
+  return response;
-
-  // Store the ETag for future requests
-  const etag = response.headers.get('ETag');
-  if (etag && checkEtag) {
-    log(`storing new etag for ${etagName}: ${etag}`);
-    await putLatestEtag(etag, etagName);
-  } else if (!etag) {
-    log(`no etag found in response for ${etagName}`);
-  }
-
-  return data;
 }

app/javascript/mastodon/features/emoji/locale.ts

@@ -2,7 +2,7 @@ import type { Locale } from 'emojibase';
 import { SUPPORTED_LOCALES } from 'emojibase';
 
 import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
-import type { EtagTypes, LocaleOrCustom, LocaleWithShortcodes } from './types';
+import type { CacheKey, LocaleOrCustom, LocaleWithShortcodes } from './types';
 
 export function toSupportedLocale(localeBase: string): Locale {
   const locale = localeBase.toLowerCase();
@@ -19,7 +19,7 @@ export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom {
   return toSupportedLocale(locale);
 }
 
-export function toValidEtagName(input: string): EtagTypes {
+export function toValidCacheKey(input: string): CacheKey {
   const lower = input.toLowerCase();
   if (lower === EMOJI_TYPE_CUSTOM || lower === EMOJI_DB_NAME_SHORTCODES) {
     return lower;

app/javascript/mastodon/features/emoji/types.ts

@@ -22,7 +22,7 @@ export type EmojiMode =
 
 export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM;
 export type LocaleWithShortcodes = `${Locale}-shortcodes`;
-export type EtagTypes =
+export type CacheKey =
   | LocaleOrCustom
   | typeof EMOJI_DB_NAME_SHORTCODES
   | LocaleWithShortcodes;