[Glitch] Add more resilience to Emoji IDB upgrades

Port 1dc2abb9f5b430f80e771f214343983919cd543d to glitch-soc

Signed-off-by: Claire <claire.github-309c@sitedethib.com>

app/javascript/flavours/glitch/actions/app.ts

@@ -10,3 +10,5 @@ interface ChangeLayoutPayload {
 }
 export const changeLayout =
   createAction<ChangeLayoutPayload>('APP_LAYOUT_CHANGE');
+
+export const needsReload = createAction('APP_NEED_RELOAD');

app/javascript/flavours/glitch/components/alerts_controller.tsx

@@ -1,6 +1,6 @@
 import { useState, useEffect } from 'react';
 
-import { useIntl } from 'react-intl';
+import { defineMessage, useIntl } from 'react-intl';
 import type { IntlShape } from 'react-intl';
 
 import { dismissAlert } from 'flavours/glitch/actions/alerts';
@@ -75,6 +75,9 @@ const TimedAlert: React.FC<{
 
 export const AlertsController: React.FC = () => {
   const alerts = useAppSelector((state) => state.alerts);
+  const needsReload = useAppSelector(
+    (state) => !!state.meta.get('needsReload'),
+  );
 
   return (
     <A11yLiveRegion className='notification-list'>
@@ -85,6 +88,18 @@ export const AlertsController: React.FC = () => {
           dismissAfter={5000 + idx * 1000}
         />
       ))}
+      {needsReload && <ReloadAlert />}
     </A11yLiveRegion>
   );
 };
+
+const reloadMessage = defineMessage({
+  id: 'alert.need_reload.message',
+  defaultMessage:
+    'Mastodon has been updated. Some things may not work correctly until you reload the page.',
+});
+
+const ReloadAlert: React.FC = () => {
+  const intl = useIntl();
+  return <Alert isActive message={intl.formatMessage(reloadMessage)} />;
+};

app/javascript/flavours/glitch/features/emoji/constants.ts

@@ -26,8 +26,8 @@ export const EMOJI_TYPE_UNICODE = 'unicode';
 export const EMOJI_TYPE_CUSTOM = 'custom';
 
 export const EMOJI_DB_NAME_SHORTCODES = 'shortcodes';
-
 export const EMOJI_DB_SHORTCODE_TEST = '2122'; // 2122 is the trademark sign, which we know has shortcodes in all datasets.
+export const EMOJI_DB_RELOAD_EVENT = 'emojiDbReload';
 
 export const EMOJIS_WITH_DARK_BORDER = [
   '🎱', // 1F3B1

app/javascript/flavours/glitch/features/emoji/database.ts

@@ -2,6 +2,7 @@ import { SUPPORTED_LOCALES } from 'emojibase';
 import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
+import { onceAsync } from '@/flavours/glitch/utils/promises';
 
 import { openEmojiDB } from './db-schema';
 import type { Database } from './db-schema';
@@ -22,8 +23,6 @@ const log = emojiLogger('database');
 
 // Loads the database in a way that ensures it's only loaded once.
 const loadDB = (() => {
-  let dbPromise: Promise<Database> | null = null;
-
   // Actually load the DB.
   async function initDB() {
     const db = await openEmojiDB();
@@ -32,17 +31,14 @@ const loadDB = (() => {
     return db;
   }
 
+  let dbPromise = onceAsync(initDB);
+
   // Loads the database, or returns the existing promise if it hasn't resolved yet.
-  const loadPromise = async (): Promise<Database> => {
-    if (dbPromise) {
-      return dbPromise;
-    }
-    dbPromise = initDB();
-    return dbPromise;
-  };
+  const loadPromise = () => dbPromise();
+
   // Special way to reset the database, used for unit testing.
   loadPromise.reset = () => {
-    dbPromise = null;
+    dbPromise = onceAsync(initDB);
   };
   return loadPromise;
 })();
@@ -335,13 +331,6 @@ export async function clearCache(key: CacheKey) {
   log('Cleared cache for %s', key);
 }
 
-export async function resetDatabase() {
-  const db = await loadDB();
-  const storeNames = [...db.objectStoreNames];
-  await Promise.all(storeNames.map((storeName) => db.clear(storeName)));
-  log(storeNames, 'Reset emoji database stores:');
-}
-
 export async function loadEmojiByHexcode(
   hexcode: string,
   localeString: string,

app/javascript/flavours/glitch/features/emoji/db-schema.test.ts

@@ -0,0 +1,67 @@
+import { IDBFactory } from 'fake-indexeddb';
+
+import { EMOJI_DB_RELOAD_EVENT } from './constants';
+import { openEmojiDB } from './db-schema';
+
+describe('openEmojiDB', () => {
+  afterEach(() => {
+    indexedDB = new IDBFactory();
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  describe('versionchange handler', () => {
+    test('dispatches reload event in main thread context', async () => {
+      const dispatchSpy = vi.fn();
+      window.addEventListener(EMOJI_DB_RELOAD_EVENT, dispatchSpy);
+
+      await openEmojiDB();
+
+      // Opening a higher version on the same IDBFactory triggers versionchange
+      // on all existing connections to that database.
+      indexedDB.open('mastodon-emoji', 9999);
+
+      await vi.waitFor(() => {
+        expect(dispatchSpy).toHaveBeenCalledOnce();
+      });
+
+      window.removeEventListener(EMOJI_DB_RELOAD_EVENT, dispatchSpy);
+    });
+
+    test('posts db-blocked message in worker context', async () => {
+      const postMessageSpy = vi.fn();
+      vi.stubGlobal('window', undefined);
+      vi.stubGlobal('self', { postMessage: postMessageSpy });
+
+      await openEmojiDB();
+
+      indexedDB.open('mastodon-emoji', 9999);
+
+      await vi.waitFor(() => {
+        expect(postMessageSpy).toHaveBeenCalledWith({ type: 'db-blocked' });
+      });
+    });
+
+    test('closes all open connections when versionchange fires', async () => {
+      const dispatchSpy = vi.fn();
+      window.addEventListener(EMOJI_DB_RELOAD_EVENT, dispatchSpy);
+
+      const db1 = await openEmojiDB();
+      const db2 = await openEmojiDB();
+
+      // Opening a higher version on the same IDBFactory triggers versionchange
+      // on all existing connections to that database.
+      indexedDB.open('mastodon-emoji', 9999);
+
+      await vi.waitFor(() => {
+        expect(dispatchSpy).toHaveBeenCalledTimes(2);
+      });
+
+      window.removeEventListener(EMOJI_DB_RELOAD_EVENT, dispatchSpy);
+
+      // Both connections should now be closed.
+      expect(() => db1.transaction('etags', 'readonly')).toThrow();
+      expect(() => db2.transaction('etags', 'readonly')).toThrow();
+    });
+  });
+});

app/javascript/flavours/glitch/features/emoji/db-schema.ts

@@ -1,6 +1,6 @@
 import { SUPPORTED_LOCALES } from 'emojibase';
 import type { Locale } from 'emojibase';
-import { openDB } from 'idb';
+import { deleteDB, openDB } from 'idb';
 import type {
   DBSchema,
   IDBPDatabase,
@@ -10,8 +10,13 @@ import type {
   StoreNames,
 } from 'idb';
 
-import { resetDatabase } from './database';
-import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types';
+import { EMOJI_DB_RELOAD_EVENT } from './constants';
+import type {
+  CustomEmojiData,
+  CacheKey,
+  UnicodeEmojiData,
+  EmojiWorkerMessage,
+} from './types';
 import { emojiLogger } from './utils';
 
 const log = emojiLogger('database');
@@ -58,72 +63,95 @@ type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
 
 export type Database = IDBPDatabase<EmojiDB>;
 
+const DATABASE_NAME = 'mastodon-emoji';
 const SCHEMA_VERSION = 4;
 
 export async function openEmojiDB() {
-  const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
-    upgrade(database, oldVersion, newVersion, trx) {
-      if (!database.objectStoreNames.contains('custom')) {
-        database.createObjectStore('custom', {
-          keyPath: 'shortcode',
-          autoIncrement: false,
-        });
-      }
-      maybeAddIndex({ trx, storeName: 'custom', indexName: 'category' });
-      maybeAddIndex({
-        trx,
-        storeName: 'custom',
-        indexName: 'tokens',
-        options: { multiEntry: true },
-      });
-
-      if (!database.objectStoreNames.contains('etags')) {
-        database.createObjectStore('etags');
-      }
-
-      SUPPORTED_LOCALES.forEach((locale) => {
-        createLocaleTable(locale, database, trx);
-      });
-
-      const shortcodeTable = database.objectStoreNames.contains('shortcodes')
-        ? trx.objectStore('shortcodes')
-        : database.createObjectStore('shortcodes', {
-            keyPath: 'hexcode',
+  try {
+    const db = await openDB<EmojiDB>(DATABASE_NAME, SCHEMA_VERSION, {
+      upgrade(database, oldVersion, newVersion, trx) {
+        if (!database.objectStoreNames.contains('custom')) {
+          database.createObjectStore('custom', {
+            keyPath: 'shortcode',
             autoIncrement: false,
           });
-      maybeAddIndex({
-        trx,
-        storeName: 'shortcodes',
-        indexName: 'shortcodes',
-        options: { multiEntry: true },
-      });
-      deleteOldIndexes(shortcodeTable, ['hexcode']);
+        }
+        maybeAddIndex({ trx, storeName: 'custom', indexName: 'category' });
+        maybeAddIndex({
+          trx,
+          storeName: 'custom',
+          indexName: 'tokens',
+          options: { multiEntry: true },
+        });
 
-      void resetDatabase();
+        if (!database.objectStoreNames.contains('etags')) {
+          database.createObjectStore('etags');
+        }
 
-      log(
-        'Upgraded emoji database from version %d to %d',
-        oldVersion,
-        newVersion,
-      );
-    },
-    blocked(currentVersion, blockedVersion) {
-      log(
-        'Emoji database upgrade from version %d to %d is blocked',
-        currentVersion,
-        blockedVersion,
-      );
-    },
-    blocking(currentVersion, blockedVersion) {
-      log(
-        'Emoji database upgrade from version %d is blocking upgrade to %d',
-        currentVersion,
-        blockedVersion,
-      );
-    },
-  });
+        SUPPORTED_LOCALES.forEach((locale) => {
+          createLocaleTable(locale, database, trx);
+        });
 
-  return db;
+        const shortcodeTable = database.objectStoreNames.contains('shortcodes')
+          ? trx.objectStore('shortcodes')
+          : database.createObjectStore('shortcodes', {
+              keyPath: 'hexcode',
+              autoIncrement: false,
+            });
+        maybeAddIndex({
+          trx,
+          storeName: 'shortcodes',
+          indexName: 'shortcodes',
+          options: { multiEntry: true },
+        });
+        deleteOldIndexes(shortcodeTable, ['hexcode']);
+
+        // Reset all database stores.
+        for (const storeName of database.objectStoreNames) {
+          void trx.objectStore(storeName).clear();
+        }
+
+        log(
+          'Upgraded emoji database from version %d to %d',
+          oldVersion,
+          newVersion,
+        );
+      },
+      blocked(currentVersion, blockedVersion) {
+        log(
+          'Emoji database upgrade from version %d to %d is blocked',
+          currentVersion,
+          blockedVersion,
+        );
+      },
+      blocking(currentVersion, blockedVersion) {
+        log(
+          'Emoji database upgrade from version %d is blocking upgrade to %d, closing connection',
+          currentVersion,
+          blockedVersion,
+        );
+        // Close the database connection immediately.
+        db.close();
+
+        if (typeof window !== 'undefined') {
+          window.dispatchEvent(new Event(EMOJI_DB_RELOAD_EVENT));
+        } else {
+          self.postMessage({ type: 'db-blocked' } satisfies EmojiWorkerMessage);
+        }
+      },
+    });
+
+    return db;
+  } catch (error) {
+    if (error instanceof DOMException && error.name === 'VersionError') {
+      log(
+        'Emoji database version is newer than expected, deleting and recreating',
+      );
+      await deleteDB(DATABASE_NAME);
+      return openEmojiDB();
+    }
+    throw error;
+  }
 }
 
 function maybeAddIndex<StoreName extends StoreNames<EmojiDB>>({

app/javascript/flavours/glitch/features/emoji/index.ts

@@ -1,5 +1,6 @@
 import { initialState } from '@/flavours/glitch/initial_state';
 
+import { EMOJI_DB_RELOAD_EVENT } from './constants';
 import { toSupportedLocale } from './locale';
 import type { EmojiWorkerMessage } from './types';
 import { emojiLogger } from './utils';
@@ -14,6 +15,12 @@ const workerLog = emojiLogger('worker');
 // This is too short, but better to fallback quickly than wait.
 const WORKER_TIMEOUT = 2_000;
 
+// Handle reload events
+window.addEventListener(
+  EMOJI_DB_RELOAD_EVENT,
+  () => void handleEmojiDbReload(),
+);
+
 export async function initializeEmoji() {
   log('initializing emojis');
 
@@ -51,6 +58,8 @@ export async function initializeEmoji() {
         workerLog(message.message);
       } else if (type === 'done' && message.storeName === 'custom') {
         void loadEmojisToStore();
+      } else if (type === 'db-blocked') {
+        window.dispatchEvent(new Event(EMOJI_DB_RELOAD_EVENT));
       }
 
       if (type !== 'ready') {
@@ -131,3 +140,10 @@ async function loadEmojisToStore() {
 
   log('loaded emoji data into store');
 }
+
+async function handleEmojiDbReload() {
+  log('Emoji database reload needed, triggering warning');
+  const { store } = await import('@/flavours/glitch/store');
+  const { needsReload } = await import('@/flavours/glitch/actions/app');
+  store.dispatch(needsReload());
+}

app/javascript/flavours/glitch/features/emoji/loader.ts

@@ -1,6 +1,8 @@
 import { joinShortcodes } from 'emojibase';
 import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
 
+import { onceAsyncByArgs } from '@/flavours/glitch/utils/promises';
+
 import {
   putEmojiData,
   putCustomEmojiData,
@@ -17,6 +19,12 @@ const log = emojiLogger('loader');
 export async function importEmojiData(localeString: string, shortcodes = true) {
   const locale = toSupportedLocale(localeString);
 
+  return importEmojiDataOnce(locale, shortcodes);
+}
+
+const importEmojiDataOnce = onceAsyncByArgs(importEmojiDataImpl);
+
+async function importEmojiDataImpl(locale: Locale, shortcodes: boolean) {
   log(
     'importing emoji data for locale %s%s',
     locale,
@@ -171,9 +179,7 @@ async function fetchAndCheckEtag({
 
   // Use location.origin as this script may be loaded from a CDN domain.
   const url = new URL(path, location.origin);
-  const response = await fetch(url, {
-    headers,
-  });
+  const response = await fetch(url, { headers });
 
   // If not modified, return null
   if (response.status === 304) {

app/javascript/flavours/glitch/features/emoji/types.ts

@@ -82,6 +82,7 @@ export type ExtraCustomEmojiMap = Record<
 
 export type EmojiWorkerMessage =
   | { type: 'ready' }
+  | { type: 'db-blocked' }
   | {
       type: 'load';
       storeName: string;

app/javascript/flavours/glitch/reducers/meta.js

@@ -1,6 +1,6 @@
 import { Map as ImmutableMap } from 'immutable';
 
-import { changeLayout } from 'flavours/glitch/actions/app';
+import { changeLayout, needsReload } from 'flavours/glitch/actions/app';
 import { STORE_HYDRATE } from 'flavours/glitch/actions/store';
 import { layoutFromWindow } from 'flavours/glitch/is_mobile';
 
@@ -8,6 +8,7 @@ const initialState = ImmutableMap({
   streaming_api_base_url: null,
   layout: layoutFromWindow(),
   permissions: '0',
+  needsReload: false,
 });
 
 export default function meta(state = initialState, action) {
@@ -17,6 +18,8 @@ export default function meta(state = initialState, action) {
     return state.merge(action.state.get('meta')).delete('access_token').set('permissions', action.state.getIn(['role', 'permissions']));
   case changeLayout.type:
     return state.set('layout', action.payload.layout);
+  case needsReload.type:
+    return state.set('needsReload', true);
   default:
     return state;
   }

app/javascript/flavours/glitch/utils/promises.test.ts

@@ -0,0 +1,203 @@
+import { onceAsync, onceAsyncByArgs } from './promises';
+
+describe('onceAsync', () => {
+  test('calls async callback once for concurrent calls', async () => {
+    let callCount = 0;
+    let resolveValue!: (value: string) => void;
+
+    const wrapped = onceAsync(async () => {
+      callCount += 1;
+      return new Promise<string>((resolve) => {
+        resolveValue = resolve;
+      });
+    });
+
+    const first = wrapped();
+    const second = wrapped();
+
+    expect(callCount).toBe(1);
+    expect(first).toBe(second);
+
+    resolveValue('ok');
+
+    await expect(first).resolves.toBe('ok');
+    await expect(second).resolves.toBe('ok');
+  });
+
+  test('returns cached response for later calls without re-running callback', async () => {
+    let callCount = 0;
+    const wrapped = onceAsync(() => {
+      callCount += 1;
+      return Promise.resolve('cached');
+    });
+
+    await expect(wrapped()).resolves.toBe('cached');
+    await expect(wrapped()).resolves.toBe('cached');
+
+    expect(callCount).toBe(1);
+  });
+
+  test('keeps the same rejection promise and does not re-run callback', async () => {
+    let callCount = 0;
+    const error = new Error('boom');
+
+    const wrapped = onceAsync(() => {
+      callCount += 1;
+      return Promise.reject(error);
+    });
+
+    await expect(wrapped()).rejects.toThrow('boom');
+    await expect(wrapped()).rejects.toThrow('boom');
+
+    expect(callCount).toBe(1);
+  });
+});
+
+describe('onceAsyncByArgs', () => {
+  test('shares one in-flight callback for the same argument key', async () => {
+    let callCount = 0;
+    let resolveValue!: (value: string) => void;
+
+    const wrapped = onceAsyncByArgs((arg: string, bool: boolean) => {
+      callCount += 1;
+      return new Promise<string>((resolve) => {
+        resolveValue = (value: string) => {
+          resolve(`${arg}:${bool}:${value}`);
+        };
+      });
+    });
+
+    const first = wrapped('test', true);
+    const second = wrapped('test', true);
+
+    expect(callCount).toBe(1);
+    expect(first).toBe(second);
+
+    resolveValue('done');
+
+    await expect(first).resolves.toBe('test:true:done');
+    await expect(second).resolves.toBe('test:true:done');
+  });
+
+  test('runs separate callbacks for different argument keys', async () => {
+    let callCount = 0;
+
+    const wrapped = onceAsyncByArgs((arg: string, bool: boolean) => {
+      callCount += 1;
+      return Promise.resolve(`${arg}:${bool}`);
+    });
+
+    await expect(wrapped('test', false)).resolves.toBe('test:false');
+    await expect(wrapped('test', true)).resolves.toBe('test:true');
+
+    expect(callCount).toBe(2);
+  });
+
+  test('allows the same key to run again after previous promise settles', async () => {
+    let callCount = 0;
+
+    const wrapped = onceAsyncByArgs((arg: string) => {
+      callCount += 1;
+      return Promise.resolve(arg);
+    });
+
+    await expect(wrapped('test')).resolves.toBe('test');
+    await expect(wrapped('test')).resolves.toBe('test');
+
+    expect(callCount).toBe(2);
+  });
+
+  test('returns resolved promise again if deleteOnComplete is false', async () => {
+    let callCount = 0;
+
+    const wrapped = onceAsyncByArgs(
+      (arg: string) => {
+        callCount += 1;
+        return Promise.resolve(arg);
+      },
+      { deleteOnComplete: false },
+    );
+
+    const first = wrapped('test');
+    await expect(first).resolves.toBe('test');
+
+    const second = wrapped('test');
+    await expect(second).resolves.toBe('test');
+
+    expect(first).toBe(second);
+    expect(callCount).toBe(1);
+  });
+
+  test('calls log function when provided', async () => {
+    let resolveValue!: (value: string) => void;
+    const log = vi.fn<(format: string, ...args: unknown[]) => void>();
+
+    const wrapped = onceAsyncByArgs(
+      (arg: string) =>
+        new Promise<string>((resolve) => {
+          resolveValue = (value: string) => {
+            resolve(`${arg}:${value}`);
+          };
+        }),
+      { log },
+    );
+
+    const first = wrapped('test');
+    const second = wrapped('test');
+    expect(first).toBe(second);
+
+    resolveValue('done');
+    await expect(first).resolves.toBe('test:done');
+
+    expect(log).toHaveBeenCalledWith(
+      expect.stringContaining('added promise with key'),
+    );
+    expect(log).toHaveBeenCalledWith(
+      expect.stringContaining('already have in-flight promise with key'),
+    );
+    expect(log).toHaveBeenCalledWith(
+      expect.stringContaining('deleted completed promise with key'),
+    );
+  });
+
+  test('re-runs callback for same key after rejection with default deleteOnComplete', async () => {
+    let callCount = 0;
+    const wrapped = onceAsyncByArgs((arg: string) => {
+      callCount += 1;
+      return Promise.reject(new Error(`boom:${arg}`));
+    });
+
+    await expect(wrapped('test')).rejects.toThrow('boom');
+    await expect(wrapped('test')).rejects.toThrow('boom');
+
+    expect(callCount).toBe(2);
+  });
+
+  test('uses keyFromArgs to dedupe by custom key', async () => {
+    let callCount = 0;
+    let resolveValue!: (value: string) => void;
+
+    const wrapped = onceAsyncByArgs(
+      ({ id }: { id: string; payload: string }) => {
+        callCount += 1;
+        return new Promise<string>((resolve) => {
+          resolveValue = (value: string) => {
+            resolve(`${id}:${value}`);
+          };
+        });
+      },
+      {
+        keyFromArgs: ({ id }) => id,
+      },
+    );
+
+    const first = wrapped({ id: '1', payload: 'a' });
+    const second = wrapped({ id: '1', payload: 'b' });
+
+    expect(first).toBe(second);
+    expect(callCount).toBe(1);
+
+    resolveValue('ok');
+    await expect(first).resolves.toBe('1:ok');
+  });
+});

app/javascript/flavours/glitch/utils/promises.ts

@@ -0,0 +1,57 @@
+/**
+ * Wraps a function that returns a promise so that it only executes once, and subsequent calls return the same promise.
+ */
+export function onceAsync<TArgs extends unknown[], TResult>(
+  callback: (...args: TArgs) => Promise<TResult>,
+): (...args: TArgs) => Promise<TResult> {
+  let promise: Promise<TResult> | null = null;
+
+  return (...args: TArgs) => {
+    promise ??= callback(...args);
+    return promise;
+  };
+}
+
+interface OnceAsyncByArgs<TArgs extends unknown[]> {
+  /** Callback to derive a string key from arguments. Defaults to JSON.stringify. */
+  keyFromArgs?: (...args: TArgs) => string;
+  /** Whether to remove a promise from the cache once it resolves. Defaults to true. */
+  deleteOnComplete?: boolean;
+  /** Optional callback to log promise changes. */
+  log?: (format: string, ...args: unknown[]) => void;
+}
+
+/**
+ * Wraps an async function so that only one in-flight call exists per argument key.
+ * Concurrent calls with the same key share one promise.
+ */
+export function onceAsyncByArgs<TArgs extends unknown[], TResult>(
+  callback: (...args: TArgs) => Promise<TResult>,
+  {
+    keyFromArgs = (...args) => JSON.stringify(args),
+    deleteOnComplete = true,
+    log,
+  }: OnceAsyncByArgs<TArgs> = {},
+): (...args: TArgs) => Promise<TResult> {
+  const inFlight = new Map<string, Promise<TResult>>();
+
+  return (...args: TArgs) => {
+    const key = keyFromArgs(...args);
+    const existingPromise = inFlight.get(key);
+
+    if (existingPromise) {
+      log?.(`already have in-flight promise with key ${key}`);
+      return existingPromise;
+    }
+
+    const promise = callback(...args).finally(() => {
+      if (deleteOnComplete) {
+        log?.(`deleted completed promise with key ${key}`);
+        inFlight.delete(key);
+      }
+    });
+    log?.(`added promise with key ${key}`);
+    inFlight.set(key, promise);
+    return promise;
+  };
+}