diff --git a/app/javascript/mastodon/components/emoji/index.tsx b/app/javascript/mastodon/components/emoji/index.tsx index 0b1ba7fef3..9b917df6ea 100644 --- a/app/javascript/mastodon/components/emoji/index.tsx +++ b/app/javascript/mastodon/components/emoji/index.tsx @@ -88,7 +88,10 @@ export const Emoji: FC = ({ ); } - const src = unicodeHexToUrl(state.code, appState.darkTheme); + const src = unicodeHexToUrl({ + unicodeHex: state.code, + ...appState, + }); return ( = {}): CompactEmoji { + return { + ...unicodeEmojiFactory(), + tags: ['test', 'emoji'], + ...data, + }; +} + describe('emoji database', () => { afterEach(() => { testClear(); @@ -32,7 +39,7 @@ describe('emoji database', () => { }); test('loads emoji into indexedDB', async () => { - await putEmojiData([unicodeEmojiFactory()], 'en'); + await putEmojiData([rawEmojiFactory()], 'en'); const { db } = await testGet(); await expect(db.get('en', 'test')).resolves.toEqual( unicodeEmojiFactory(), @@ -60,7 +67,7 @@ describe('emoji database', () => { }); await expect(db.get('custom', 'emoji1')).resolves.toBeUndefined(); await expect(db.get('custom', 'emoji2')).resolves.toEqual( - customEmojiFactory({ shortcode: 'emoji2' }), + customEmojiFactory({ shortcode: 'emoji2', tokens: ['emoji2'] }), ); }); }); @@ -79,12 +86,6 @@ describe('emoji database', () => { }); describe('loadEmojiByHexcode', () => { - test('throws if the locale is not loaded', async () => { - await expect(loadEmojiByHexcode('en', 'test')).rejects.toThrowError( - 'Locale en', - ); - }); - test('retrieves the emoji', async () => { await putEmojiData([unicodeEmojiFactory()], 'en'); await expect(loadEmojiByHexcode('test', 'en')).resolves.toEqual( @@ -98,90 +99,6 @@ describe('emoji database', () => { }); }); - describe('searchEmojisByHexcodes', () => { - const data = [ - unicodeEmojiFactory({ hexcode: 'not a number' }), - unicodeEmojiFactory({ hexcode: '1' }), - unicodeEmojiFactory({ hexcode: '2' }), - unicodeEmojiFactory({ hexcode: '3' }), - unicodeEmojiFactory({ hexcode: 'another not a number' }), - ]; - beforeEach(async () => { - await putEmojiData(data, 'en'); - }); - test('finds emoji in consecutive range', async () => { - const actual = await searchEmojisByHexcodes(['1', '2', '3'], 'en'); - expect(actual).toHaveLength(3); - }); - - test('finds emoji in split range', async () => { - const actual = await searchEmojisByHexcodes(['1', '3'], 'en'); - expect(actual).toHaveLength(2); - expect(actual).toContainEqual(data.at(1)); - expect(actual).toContainEqual(data.at(3)); - }); - - test('finds emoji with non-numeric range', async () => { - const actual = await searchEmojisByHexcodes( - ['3', 'not a number', '1'], - 'en', - ); - expect(actual).toHaveLength(3); - expect(actual).toContainEqual(data.at(0)); - expect(actual).toContainEqual(data.at(1)); - expect(actual).toContainEqual(data.at(3)); - }); - - test('not found emoji are not returned', async () => { - const actual = await searchEmojisByHexcodes(['not found'], 'en'); - expect(actual).toHaveLength(0); - }); - - test('only found emojis are returned', async () => { - const actual = await searchEmojisByHexcodes( - ['another not a number', 'not found'], - 'en', - ); - expect(actual).toHaveLength(1); - expect(actual).toContainEqual(data.at(4)); - }); - }); - - describe('searchEmojisByTag', () => { - const data = [ - unicodeEmojiFactory({ hexcode: 'test1', tags: ['test 1'] }), - unicodeEmojiFactory({ - hexcode: 'test2', - tags: ['test 2', 'something else'], - }), - unicodeEmojiFactory({ hexcode: 'test3', tags: ['completely different'] }), - ]; - beforeEach(async () => { - await putEmojiData(data, 'en'); - }); - test('finds emojis with tag', async () => { - const actual = await searchEmojisByTag('test 1', 'en'); - expect(actual).toHaveLength(1); - expect(actual).toContainEqual(data.at(0)); - }); - - test('finds emojis starting with tag', async () => { - const actual = await searchEmojisByTag('test', 'en'); - expect(actual).toHaveLength(2); - expect(actual).not.toContainEqual(data.at(2)); - }); - - test('does not find emojis ending with tag', async () => { - const actual = await searchEmojisByTag('else', 'en'); - expect(actual).toHaveLength(0); - }); - - test('finds nothing with invalid tag', async () => { - const actual = await searchEmojisByTag('not found', 'en'); - expect(actual).toHaveLength(0); - }); - }); - describe('loadLegacyShortcodesByShortcode', () => { const data = { hexcode: 'test_hexcode', diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index fe4010a861..f64f3fb80d 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -1,55 +1,25 @@ import { SUPPORTED_LOCALES } from 'emojibase'; -import type { Locale, ShortcodesDataset } from 'emojibase'; -import type { DBSchema, IDBPDatabase } from 'idb'; -import { openDB } from 'idb'; +import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; import { EMOJI_DB_SHORTCODE_TEST } from './constants'; -import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; -import type { CustomEmojiData, UnicodeEmojiData, EtagTypes } from './types'; +import { openEmojiDB } from './db-schema'; +import type { Database } from './db-schema'; +import { + localeToSegmenter, + toSupportedLocale, + toSupportedLocaleOrCustom, +} from './locale'; +import { + extractTokens, + skinHexcodeToEmoji, + transformCustomEmojiData, + transformEmojiData, +} from './normalize'; +import type { AnyEmojiData, EtagTypes } from './types'; import { emojiLogger } from './utils'; -interface EmojiDB extends LocaleTables, DBSchema { - custom: { - key: string; - value: CustomEmojiData; - indexes: { - category: string; - }; - }; - shortcodes: { - key: string; - value: { - hexcode: string; - shortcodes: string[]; - }; - indexes: { - hexcode: string; - shortcodes: string[]; - }; - }; - etags: { - key: EtagTypes; - value: string; - }; -} - -interface LocaleTable { - key: string; - value: UnicodeEmojiData; - indexes: { - group: number; - label: string; - order: number; - tags: string[]; - shortcodes: string[]; - }; -} -type LocaleTables = Record; - -type Database = IDBPDatabase; - -const SCHEMA_VERSION = 2; - const loadedLocales = new Set(); const log = emojiLogger('database'); @@ -60,75 +30,7 @@ const loadDB = (() => { // Actually load the DB. async function initDB() { - const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { - upgrade(database, oldVersion, newVersion, trx) { - if (!database.objectStoreNames.contains('custom')) { - const customTable = database.createObjectStore('custom', { - keyPath: 'shortcode', - autoIncrement: false, - }); - customTable.createIndex('category', 'category'); - } - - if (!database.objectStoreNames.contains('etags')) { - database.createObjectStore('etags'); - } - - for (const locale of SUPPORTED_LOCALES) { - if (!database.objectStoreNames.contains(locale)) { - const localeTable = database.createObjectStore(locale, { - keyPath: 'hexcode', - autoIncrement: false, - }); - localeTable.createIndex('group', 'group'); - localeTable.createIndex('label', 'label'); - localeTable.createIndex('order', 'order'); - localeTable.createIndex('tags', 'tags', { multiEntry: true }); - localeTable.createIndex('shortcodes', 'shortcodes', { - multiEntry: true, - }); - } - // Added in version 2. - const localeTable = trx.objectStore(locale); - if (!localeTable.indexNames.contains('shortcodes')) { - localeTable.createIndex('shortcodes', 'shortcodes', { - multiEntry: true, - }); - } - } - - if (!database.objectStoreNames.contains('shortcodes')) { - const shortcodeTable = database.createObjectStore('shortcodes', { - keyPath: 'hexcode', - autoIncrement: false, - }); - shortcodeTable.createIndex('hexcode', 'hexcode'); - shortcodeTable.createIndex('shortcodes', 'shortcodes', { - multiEntry: true, - }); - } - - log( - 'Upgraded emoji database from version %d to %d', - oldVersion, - newVersion, - ); - }, - blocked(currentVersion, blockedVersion) { - log( - 'Emoji database upgrade from version %d to %d is blocked', - currentVersion, - blockedVersion, - ); - }, - blocking(currentVersion, blockedVersion) { - log( - 'Emoji database upgrade from version %d is blocking upgrade to %d', - currentVersion, - blockedVersion, - ); - }, - }); + const db = await openEmojiDB(); await syncLocales(db); log('Loaded database version %d', db.version); return db; @@ -149,11 +51,128 @@ const loadDB = (() => { return loadPromise; })(); -export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) { +export async function search({ + query, + locale: localeString, + limit = 0, +}: { + query: string; + locale: string; + limit?: number; +}) { + performance.mark('emoji-search-start'); + + // Get the locale, and extract tokens from the query. + const locale = await toLoadedLocale(localeString); + const segmenter = localeToSegmenter(locale); + const queryTokens = extractTokens(query, segmenter); + + if (queryTokens.length === 0) { + log('no tokens extracted from query "%s"', query); + return []; + } + const lastToken = queryTokens.at(-1); + if (!lastToken) { + throw new Error('Missing tokens from query'); + } + + log('searching for tokens %o in locale %s', queryTokens, locale); + + // Create an array of emoji results + const db = await loadDB(); + const resultArrays: Map[] = []; + for (let i = 0; i < queryTokens.length; i++) { + const token = queryTokens[i]; + if (!token) continue; + + // Only query the range for the last token to allow partial matches. + const range = + i === queryTokens.length - 1 + ? IDBKeyRange.bound(token, token + '\uffff') + : IDBKeyRange.only(token); + + const [unicodeResults, customResults] = await Promise.all([ + db.getAllFromIndex(locale, 'tokens', range), + db.getAllFromIndex('custom', 'tokens', range), + ]); + const resultMap = new Map([ + ...unicodeResults.map( + (emoji) => [emoji.hexcode, emoji] as [string, AnyEmojiData], + ), + ...customResults.map( + (emoji) => [emoji.shortcode, emoji] as [string, AnyEmojiData], + ), + ]); + log('found %d results for token "%s"', resultMap.size, token); + resultArrays.push(resultMap); + } + + // Utilize maps to find the intersection of all result sets. + const results = Array.from( + resultArrays + .reduce((prev, curr) => { + const intersection = new Map(); + for (const [code, emoji] of prev) { + if (curr.has(code)) { + intersection.set(code, emoji); + } + } + return intersection; + }) + .values(), + ); + + results.sort((a, b) => { + // Checks if a or b has the last token exactly, or only a prefix. + const aHasToken = a.tokens.includes(lastToken); + const bHasToken = b.tokens.includes(lastToken); + if (aHasToken && !bHasToken) { + return -1; + } else if (!aHasToken && bHasToken) { + return 1; + } + + // If one is a custom emoji, prioritize it over Unicode emojis. + if ('category' in a) { + return -1; + } else if ('category' in b) { + return 1; + } + + // If both are Unicode emojis, prioritize by order. + if ('order' in a && 'order' in b) { + return (a.order ?? 0) - (b.order ?? 0); // If these are both Unicode emojis, sort by order. + } + + // ยฏ\_(ใƒ„)_/ยฏ + return 0; + }); + + const time = performance.measure('emoji-search-end', 'emoji-search-start'); + log( + 'search for "%s" in locale %s returned %d results and took %dms', + query, + locale, + results.length, + time.duration, + ); + if (limit > 0) { + return results.slice(0, limit); + } + return results; +} + +export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) { loadedLocales.add(locale); const db = await loadDB(); const trx = db.transaction(locale, 'readwrite'); - await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); + await trx.store.clear(); + const segmenter = localeToSegmenter(locale); + await Promise.all( + emojis + .sort((a, b) => (a.order ?? 0) - (b.order ?? 0)) + .map((emoji) => trx.store.put(transformEmojiData(emoji, segmenter))), + ); await trx.done; } @@ -161,7 +180,7 @@ export async function putCustomEmojiData({ emojis, clear = false, }: { - emojis: CustomEmojiData[]; + emojis: ApiCustomEmojiJSON[]; clear?: boolean; }) { const db = await loadDB(); @@ -173,7 +192,9 @@ export async function putCustomEmojiData({ log('Cleared existing custom emojis in database'); } - await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); + await Promise.all( + emojis.map((emoji) => trx.store.put(transformCustomEmojiData(emoji))), + ); await trx.done; log('Imported %d custom emojis into database', emojis.length); @@ -210,32 +231,25 @@ export async function loadEmojiByHexcode( localeString: string, ) { const db = await loadDB(); - const locale = toLoadedLocale(localeString); - return db.get(locale, hexcode); -} + const locale = await toLoadedLocale(localeString); + const result = await db.get(locale, hexcode); + if (result) { + return result; + } -export async function searchEmojisByHexcodes( - hexcodes: string[], - localeString: string, -) { - const db = await loadDB(); - const locale = toLoadedLocale(localeString); - const sortedCodes = hexcodes.toSorted(); - const results = await db.getAll( + // If the emoji wasn't found, check if it's a skin tone variant. + const skinResult = await db.getFromIndex( locale, - IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)), + 'skinHexcodes', + IDBKeyRange.only(hexcode), ); - return results.filter((emoji) => hexcodes.includes(emoji.hexcode)); -} -export async function searchEmojisByTag(tag: string, localeString: string) { - const db = await loadDB(); - const locale = toLoadedLocale(localeString); - const range = IDBKeyRange.bound( - tag.toLowerCase(), - `${tag.toLowerCase()}\uffff`, - ); - return db.getAllFromIndex(locale, 'tags', range); + if (!skinResult) { + return skinResult; + } + + // Reconstruct the full unicode string from the skin tone hexcode. + return skinHexcodeToEmoji(hexcode, skinResult); } export async function loadCustomEmojiByShortcode(shortcode: string) { @@ -301,13 +315,16 @@ async function syncLocales(db: Database) { log('Loaded %d locales: %o', loadedLocales.size, loadedLocales); } -function toLoadedLocale(localeString: string) { +async function toLoadedLocale(localeString: string) { const locale = toSupportedLocale(localeString); if (localeString !== locale) { log(`Locale ${locale} is different from provided ${localeString}`); } if (!loadedLocales.has(locale)) { - throw new LocaleNotLoadedError(locale); + log('Locale %s not loaded, importing...', locale); + const { importEmojiData } = await import('./loader'); + await importEmojiData(locale); + return locale; } return locale; } diff --git a/app/javascript/mastodon/features/emoji/db-schema.ts b/app/javascript/mastodon/features/emoji/db-schema.ts new file mode 100644 index 0000000000..f5582cf0c3 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/db-schema.ts @@ -0,0 +1,198 @@ +import { SUPPORTED_LOCALES } from 'emojibase'; +import type { Locale } from 'emojibase'; +import { openDB } from 'idb'; +import type { + DBSchema, + IDBPDatabase, + IDBPObjectStore, + IDBPTransaction, + IndexNames, + StoreNames, +} from 'idb'; + +import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types'; +import { emojiLogger } from './utils'; + +const log = emojiLogger('database'); + +interface EmojiDB extends LocaleTables, DBSchema { + custom: { + key: string; + value: CustomEmojiData; + indexes: { + tokens: string[]; + category: string; + }; + }; + shortcodes: { + key: string; + value: { + hexcode: string; + shortcodes: string[]; + }; + indexes: { + shortcodes: string[]; + }; + }; + etags: { + key: EtagTypes; + value: string; + }; +} + +interface LocaleTable { + key: string; + value: UnicodeEmojiData; + indexes: { + shortcodes: string[]; + groupOrder: [number, number]; + tokens: string[]; + skinHexcodes: string[]; + }; +} +type LocaleTables = Record; + +type Transaction = + IDBPTransaction[], Mode>; + +export type Database = IDBPDatabase; + +const SCHEMA_VERSION = 3; + +export async function openEmojiDB() { + const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { + upgrade(database, oldVersion, newVersion, trx) { + if (!database.objectStoreNames.contains('custom')) { + database.createObjectStore('custom', { + keyPath: 'shortcode', + autoIncrement: false, + }); + } + maybeAddIndex({ trx, storeName: 'custom', indexName: 'category' }); + maybeAddIndex({ + trx, + storeName: 'custom', + indexName: 'tokens', + options: { multiEntry: true }, + }); + + if (!database.objectStoreNames.contains('etags')) { + database.createObjectStore('etags'); + } + + SUPPORTED_LOCALES.forEach((locale) => { + createLocaleTable(locale, database, trx); + }); + + const shortcodeTable = database.objectStoreNames.contains('shortcodes') + ? trx.objectStore('shortcodes') + : database.createObjectStore('shortcodes', { + keyPath: 'hexcode', + autoIncrement: false, + }); + maybeAddIndex({ + trx, + storeName: 'shortcodes', + indexName: 'shortcodes', + options: { multiEntry: true }, + }); + deleteOldIndexes(shortcodeTable, ['hexcode']); + + log( + 'Upgraded emoji database from version %d to %d', + oldVersion, + newVersion, + ); + }, + blocked(currentVersion, blockedVersion) { + log( + 'Emoji database upgrade from version %d to %d is blocked', + currentVersion, + blockedVersion, + ); + }, + blocking(currentVersion, blockedVersion) { + log( + 'Emoji database upgrade from version %d is blocking upgrade to %d', + currentVersion, + blockedVersion, + ); + }, + }); + + return db; +} + +function maybeAddIndex>({ + trx, + storeName, + indexName, + keys, + options, +}: { + trx: Transaction; + storeName: StoreName; + indexName: IndexNames; + keys?: string | string[]; + options?: IDBIndexParameters; +}) { + const store = trx.objectStore(storeName); + if (!store.indexNames.contains(indexName)) { + store.createIndex(indexName, keys ?? indexName, options); + } +} + +function createLocaleTable( + locale: Locale, + database: Database, + trx: Transaction, +) { + if (!database.objectStoreNames.contains(locale)) { + database.createObjectStore(locale, { + keyPath: 'hexcode', + autoIncrement: false, + }); + } + + maybeAddIndex({ + trx, + storeName: locale, + indexName: 'shortcodes', + options: { multiEntry: true }, + }); + maybeAddIndex({ + trx, + storeName: locale, + indexName: 'groupOrder', + keys: ['group', 'order'], + }); + maybeAddIndex({ + trx, + storeName: locale, + indexName: 'tokens', + keys: 'tokens', + options: { multiEntry: true }, + }); + maybeAddIndex({ + trx, + storeName: locale, + indexName: 'skinHexcodes', + keys: 'skinHexcodes', + options: { multiEntry: true }, + }); + + const oldIndexes = ['group', 'order', 'tag', 'label'] as const; + deleteOldIndexes(trx.objectStore(locale), oldIndexes); +} + +function deleteOldIndexes( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Type is too complex, so only any works here. + table: IDBPObjectStore, + indexes: readonly string[], +) { + for (const index of indexes) { + if (table.indexNames.contains(index)) { + table.deleteIndex(index); + } + } +} diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts index c6b64fe29c..7a94d604a9 100644 --- a/app/javascript/mastodon/features/emoji/loader.ts +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -1,10 +1,5 @@ -import { flattenEmojiData } from 'emojibase'; -import type { - CompactEmoji, - FlatCompactEmoji, - Locale, - ShortcodesDataset, -} from 'emojibase'; +import { joinShortcodes } from 'emojibase'; +import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase'; import { putEmojiData, @@ -28,7 +23,7 @@ export async function importEmojiData(localeString: string, shortcodes = true) { shortcodes ? ' and shortcodes' : '', ); - const emojis = await fetchAndCheckEtag({ + let emojis = await fetchAndCheckEtag({ etagString: locale, path: localeToEmojiPath(locale), }); @@ -49,12 +44,10 @@ export async function importEmojiData(localeString: string, shortcodes = true) { } } - const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData( - emojis, - shortcodesData, - ); - await putEmojiData(flattenedEmojis, locale); - return flattenedEmojis; + emojis = joinShortcodes(emojis, shortcodesData); + + await putEmojiData(emojis, locale); + return emojis; } export async function importCustomEmojiData() { @@ -135,11 +128,11 @@ async function fetchAndCheckEtag({ checkEtag?: boolean; }): Promise { const etagName = toValidEtagName(etagString); + const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null; // Use location.origin as this script may be loaded from a CDN domain. const url = new URL(path, location.origin); - const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null; const response = await fetch(url, { headers: { 'Content-Type': 'application/json', @@ -148,6 +141,7 @@ async function fetchAndCheckEtag({ }); // If not modified, return null if (response.status === 304) { + log('etag not modified for %s', etagName); return null; } if (!response.ok) { @@ -163,6 +157,8 @@ async function fetchAndCheckEtag({ if (etag && checkEtag) { log(`storing new etag for ${etagName}: ${etag}`); await putLatestEtag(etag, etagName); + } else if (!etag) { + log(`no etag found in response for ${etagName}`); } return data; diff --git a/app/javascript/mastodon/features/emoji/locale.ts b/app/javascript/mastodon/features/emoji/locale.ts index f39b56d47c..e8f2df340f 100644 --- a/app/javascript/mastodon/features/emoji/locale.ts +++ b/app/javascript/mastodon/features/emoji/locale.ts @@ -32,6 +32,13 @@ export function toValidEtagName(input: string): EtagTypes { return toSupportedLocale(lower); } +export function localeToSegmenter(locale: Locale): Intl.Segmenter | null { + if (typeof Intl.Segmenter === 'function') { + return new Intl.Segmenter(locale, { granularity: 'word' }); + } + return null; +} + function isSupportedLocale(locale: string): locale is Locale { return SUPPORTED_LOCALES.includes(locale as Locale); } diff --git a/app/javascript/mastodon/features/emoji/mode.ts b/app/javascript/mastodon/features/emoji/mode.ts index 6950626375..6763fc4b50 100644 --- a/app/javascript/mastodon/features/emoji/mode.ts +++ b/app/javascript/mastodon/features/emoji/mode.ts @@ -2,6 +2,7 @@ // See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js import { createAppSelector, useAppSelector } from '@/mastodon/store'; +import { assetHost } from '@/mastodon/utils/config'; import { isDevelopment } from '@/mastodon/utils/environment'; import { isDarkMode } from '@/mastodon/utils/theme'; @@ -29,6 +30,7 @@ export function useEmojiAppState(): EmojiAppState { locales: [locale], mode, darkTheme: isDarkMode(), + assetHost, }; } diff --git a/app/javascript/mastodon/features/emoji/normalize.test.ts b/app/javascript/mastodon/features/emoji/normalize.test.ts index 8222ab81e5..8c6346709b 100644 --- a/app/javascript/mastodon/features/emoji/normalize.test.ts +++ b/app/javascript/mastodon/features/emoji/normalize.test.ts @@ -4,11 +4,7 @@ import { basename, resolve } from 'path'; import { flattenEmojiData } from 'emojibase'; import unicodeRawEmojis from 'emojibase-data/en/data.json'; -import { - twemojiToUnicodeInfo, - unicodeToTwemojiHex, - emojiToUnicodeHex, -} from './normalize'; +import { unicodeToTwemojiHex } from './normalize'; const emojiSVGFiles = await readdir( // This assumes tests are run from project root @@ -26,23 +22,6 @@ const svgFileNamesWithoutBorder = svgFileNames.filter( const unicodeEmojis = flattenEmojiData(unicodeRawEmojis); -describe('emojiToUnicodeHex', () => { - test.concurrent.for([ - ['๐ŸŽฑ', '1F3B1'], - ['๐Ÿœ', '1F41C'], - ['โšซ', '26AB'], - ['๐Ÿ–ค', '1F5A4'], - ['๐Ÿ’€', '1F480'], - ['โค๏ธ', '2764'], // Checks for trailing variation selector removal. - ['๐Ÿ’‚โ€โ™‚๏ธ', '1F482-200D-2642-FE0F'], - ] as const)( - 'emojiToUnicodeHex converts %s to %s', - ([emoji, hexcode], { expect }) => { - expect(emojiToUnicodeHex(emoji)).toBe(hexcode); - }, - ); -}); - describe('unicodeToTwemojiHex', () => { test.concurrent.for( unicodeEmojis @@ -54,26 +33,3 @@ describe('unicodeToTwemojiHex', () => { expect(svgFileNamesWithoutBorder).toContain(result); }); }); - -describe('twemojiToUnicodeInfo', () => { - const unicodeCodeSet = new Set(unicodeEmojis.map((emoji) => emoji.hexcode)); - - test.concurrent.for(svgFileNamesWithoutBorder)( - 'verifying SVG file %s maps to Unicode emoji', - (svgFileName, { expect }) => { - assert(!!svgFileName); - const result = twemojiToUnicodeInfo(svgFileName); - const hexcode = typeof result === 'string' ? result : result.unqualified; - if (!hexcode) { - // No hexcode means this is a special case like the Shibuya 109 emoji - expect(result).toHaveProperty('label'); - return; - } - assert(!!hexcode); - expect( - unicodeCodeSet.has(hexcode), - `${hexcode} (${svgFileName}) not found`, - ).toBeTruthy(); - }, - ); -}); diff --git a/app/javascript/mastodon/features/emoji/normalize.ts b/app/javascript/mastodon/features/emoji/normalize.ts index 24df808ae1..bf06e058ef 100644 --- a/app/javascript/mastodon/features/emoji/normalize.ts +++ b/app/javascript/mastodon/features/emoji/normalize.ts @@ -1,48 +1,124 @@ import { isList } from 'immutable'; -import { assetHost } from '@/mastodon/utils/config'; +import type { CompactEmoji, SkinTone } from 'emojibase'; +import { fromHexcodeToCodepoint } from 'emojibase'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; import { VARIATION_SELECTOR_CODE, KEYCAP_CODE, - GENDER_FEMALE_CODE, - GENDER_MALE_CODE, - SKIN_TONE_CODES, EMOJIS_WITH_DARK_BORDER, EMOJIS_WITH_LIGHT_BORDER, EMOJIS_REQUIRING_INVERSION_IN_LIGHT_MODE, EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE, + EMOJI_MIN_TOKEN_LENGTH, } from './constants'; -import type { CustomEmojiMapArg, ExtraCustomEmojiMap } from './types'; +import type { + CustomEmojiData, + CustomEmojiMapArg, + ExtraCustomEmojiMap, + UnicodeEmojiData, +} from './types'; +import { emojiToUnicodeHex } from './utils'; -// Misc codes that have special handling -const SKIER_CODE = 0x26f7; -const CHRISTMAS_TREE_CODE = 0x1f384; -const MR_CLAUS_CODE = 0x1f385; -const EYE_CODE = 0x1f441; -const LEVITATING_PERSON_CODE = 0x1f574; -const SPEECH_BUBBLE_CODE = 0x1f5e8; -const MS_CLAUS_CODE = 0x1f936; +const SKIN_TONE_MAP: Record = { + 0x1f3fb: 1, // Light skin tone + 0x1f3fc: 2, // Medium-light skin tone + 0x1f3fd: 3, // Medium skin tone + 0x1f3fe: 4, // Medium-dark skin tone + 0x1f3ff: 5, // Dark skin tone +}; -export function emojiToUnicodeHex(emoji: string): string { - const codes: number[] = []; - for (const char of emoji) { - const code = char.codePointAt(0); - if (code !== undefined) { - codes.push(code); +export function transformEmojiData( + emoji: CompactEmoji, + segmenter: Intl.Segmenter | null, +): UnicodeEmojiData { + const { + shortcodes = [], + tags = [], + label, + emoticon, + hexcode, + unicode, + group, + order, + skins = [], + } = emoji; + const extract = (str: string) => extractTokens(str, segmenter); + + let normalizedEmoticons: string[] | undefined = undefined; + if (emoticon) { + normalizedEmoticons = Array.isArray(emoticon) ? emoticon : [emoticon]; + } + + const tokens = [ + ...new Set([ + ...shortcodes.map(extract).flat(), + ...tags.map(extract).flat(), + ...extract(label), + ...(normalizedEmoticons ?? []), + ]), + ].sort((a, b) => a.localeCompare(b)); + + const res: UnicodeEmojiData = { + tokens, + shortcodes, + label, + emoticons: normalizedEmoticons, + hexcode, + unicode, + group, + order, + }; + + for (const skin of skins) { + res.skinHexcodes ??= []; + res.skinHexcodes.push(skin.hexcode); + + res.skinTones ??= []; + for (const codePoint of skin.unicode) { + const tone = SKIN_TONE_MAP[codePoint.codePointAt(0) ?? 0]; + if (tone) { + res.skinTones.push(tone); + break; + } } } - // Handles how Emojibase removes the variation selector for single code emojis. - // See: https://emojibase.dev/docs/spec/#merged-variation-selectors - if (codes.at(1) === VARIATION_SELECTOR_CODE && codes.length === 2) { - codes.pop(); - } - return hexNumbersToString(codes); + return res; } +export function transformCustomEmojiData( + emoji: ApiCustomEmojiJSON, +): CustomEmojiData { + const tokens = emoji.shortcode + .split('_') + .filter((word) => word.length >= EMOJI_MIN_TOKEN_LENGTH) + .map((word) => word.toLowerCase()); + return { + ...emoji, + tokens, + }; +} + +export function skinHexcodeToEmoji( + skinHexcode: string, + emoji: UnicodeEmojiData, +): UnicodeEmojiData { + return { + ...emoji, + unicode: String.fromCodePoint(...fromHexcodeToCodepoint(skinHexcode)), + hexcode: skinHexcode, + }; +} + +// Misc codes that have special handling +const EYE_CODE = 0x1f441; +const SPEECH_BUBBLE_CODE = 0x1f5e8; + export function unicodeToTwemojiHex(unicodeHex: string): string { - const codes = hexStringToNumbers(unicodeHex); + const codes = fromHexcodeToCodepoint(unicodeHex); const normalizedCodes: number[] = []; for (let i = 0; i < codes.length; i++) { const code = codes[i]; @@ -64,19 +140,28 @@ export function unicodeToTwemojiHex(unicodeHex: string): string { normalizedCodes.push(code); } - return hexNumbersToString(normalizedCodes, 0).toLowerCase(); + return normalizedCodes + .map((code) => code.toString(16)) + .join('-') + .toLowerCase(); } -export const CODES_WITH_DARK_BORDER = - EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex); +const CODES_WITH_DARK_BORDER = EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex); -export const CODES_WITH_LIGHT_BORDER = - EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex); +const CODES_WITH_LIGHT_BORDER = EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex); -export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string { +export function unicodeHexToUrl({ + unicodeHex, + darkTheme, + assetHost, +}: { + unicodeHex: string; + darkTheme: boolean; + assetHost: string; +}): string { const normalizedHex = unicodeToTwemojiHex(unicodeHex); let url = `${assetHost}/emoji/${normalizedHex}`; - if (darkMode && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) { + if (darkTheme && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) { url += '_border'; } if (CODES_WITH_DARK_BORDER.includes(normalizedHex)) { @@ -86,78 +171,6 @@ export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string { return url; } -interface TwemojiSpecificEmoji { - unqualified?: string; - gender?: number; - skin?: number; - label?: string; -} - -// Normalize man/woman to male/female -const GENDER_CODES_MAP: Record = { - [GENDER_FEMALE_CODE]: GENDER_FEMALE_CODE, - [GENDER_MALE_CODE]: GENDER_MALE_CODE, - // These are man/woman markers, but are used for gender sometimes. - [0x1f468]: GENDER_MALE_CODE, - [0x1f469]: GENDER_FEMALE_CODE, -}; - -const TWEMOJI_SPECIAL_CASES: Record = { - '1F441-200D-1F5E8': '1F441-FE0F-200D-1F5E8-FE0F', // Eye in speech bubble - // An emoji that was never ported to the Unicode standard. - // See: https://emojipedia.org/shibuya - E50A: { label: 'Shibuya 109' }, -}; - -export function twemojiToUnicodeInfo( - twemojiHex: string, -): TwemojiSpecificEmoji | string { - const specialCase = TWEMOJI_SPECIAL_CASES[twemojiHex.toUpperCase()]; - if (specialCase) { - return specialCase; - } - const codes = hexStringToNumbers(twemojiHex); - let gender: undefined | number; - let skin: undefined | number; - for (const code of codes) { - if (!gender && code in GENDER_CODES_MAP) { - gender = GENDER_CODES_MAP[code]; - } else if (!skin && code in SKIN_TONE_CODES) { - skin = code; - } - - // Exit if we have both skin and gender - if (skin && gender) { - break; - } - } - - let mappedCodes: unknown[] = codes; - - if (codes.at(-1) === CHRISTMAS_TREE_CODE && codes.length >= 3 && gender) { - // Twemoji uses the christmas tree with a ZWJ for Mr. and Mrs. Claus, - // but in Unicode that only works for Mx. Claus. - const START_CODE = - gender === GENDER_FEMALE_CODE ? MS_CLAUS_CODE : MR_CLAUS_CODE; - mappedCodes = [START_CODE, skin]; - } else if (codes.at(-1) === KEYCAP_CODE && codes.length === 2) { - // For key emoji, insert the variation selector - mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE]; - } else if ( - (codes.at(0) === SKIER_CODE || codes.at(0) === LEVITATING_PERSON_CODE) && - codes.length > 1 - ) { - // Twemoji offers more gender and skin options for the skier and levitating person emoji. - return { - unqualified: hexNumbersToString([codes.at(0)]), - skin, - gender, - }; - } - - return hexNumbersToString(mappedCodes); -} - export function emojiToInversionClassName(emoji: string): string | null { if (EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE.includes(emoji)) { return 'invert-on-dark'; @@ -189,19 +202,37 @@ export function cleanExtraEmojis(extraEmojis?: CustomEmojiMapArg) { return extraEmojis; } -function hexStringToNumbers(hexString: string): number[] { - return hexString - .split('-') - .map((code) => Number.parseInt(code, 16)) - .filter((code) => !Number.isNaN(code)); -} +/** + * Tokenizes an input string into words, using Intl.Segmenter if available. + * @param input Any input string. + * @param segmenter Segmenter, if available. + * @returns Array of tokens in lowercase. + */ +export function extractTokens( + input: string, + segmenter: Intl.Segmenter | null, +): string[] { + if (!input.trim()) { + return []; + } + const tokens: string[] = []; -function hexNumbersToString(codes: unknown[], padding = 4): string { - return codes - .filter( - (code): code is number => - typeof code === 'number' && code > 0 && !Number.isNaN(code), - ) - .map((code) => code.toString(16).padStart(padding, '0').toUpperCase()) - .join('-'); + // Prefer to use Intl.Segmenter if available for better locale support. + if (segmenter) { + for (const { isWordLike, segment } of segmenter.segment( + input.replaceAll('_', ' '), // Handle underscores from shortcodes. + )) { + if (isWordLike && segment.length >= EMOJI_MIN_TOKEN_LENGTH) { + tokens.push(segment.toLowerCase()); + } + } + } else { + // Fallback to simple splitting. + input.split(/[\s_-]+/).forEach((word) => { + if (/\w/.test(word) && word.length >= EMOJI_MIN_TOKEN_LENGTH) { + tokens.push(word.toLowerCase()); + } + }); + } + return tokens; } diff --git a/app/javascript/mastodon/features/emoji/render.ts b/app/javascript/mastodon/features/emoji/render.ts index 8fe311014a..4b65f3abde 100644 --- a/app/javascript/mastodon/features/emoji/render.ts +++ b/app/javascript/mastodon/features/emoji/render.ts @@ -4,7 +4,6 @@ import { EMOJI_TYPE_UNICODE, EMOJI_TYPE_CUSTOM, } from './constants'; -import { emojiToUnicodeHex } from './normalize'; import type { EmojiLoadedState, EmojiMode, @@ -16,6 +15,7 @@ import type { import { anyEmojiRegex, emojiLogger, + emojiToUnicodeHex, isCustomEmoji, isUnicodeEmoji, stringHasUnicodeFlags, @@ -140,12 +140,7 @@ export async function loadEmojiDataToState( } // If not found, assume it's not an emoji and return null. - log( - 'Could not find emoji %s of type %s for locale %s', - state.code, - state.type, - locale, - ); + log('Could not find emoji %s for locale %s', state.code, locale); return null; } catch (err: unknown) { // If the locale is not loaded, load it and retry once. diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index 03002dda64..8ab756972a 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -1,6 +1,6 @@ import type { List as ImmutableList } from 'immutable'; -import type { FlatCompactEmoji, Locale } from 'emojibase'; +import type { CompactEmoji, Locale, SkinTone } from 'emojibase'; import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; @@ -32,10 +32,20 @@ export interface EmojiAppState { currentLocale: Locale; mode: EmojiMode; darkTheme: boolean; + assetHost: string; } -export type CustomEmojiData = ApiCustomEmojiJSON; -export type UnicodeEmojiData = FlatCompactEmoji; +export type CustomEmojiData = ApiCustomEmojiJSON & { tokens: string[] }; +export interface UnicodeEmojiData extends Omit< + CompactEmoji, + 'emoticon' | 'skins' | 'tags' +> { + shortcodes: string[]; + tokens: string[]; + emoticons?: string[]; + skinHexcodes?: string[]; + skinTones?: (SkinTone | SkinTone[])[]; +} export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData; type CustomEmojiRenderFields = Pick< diff --git a/app/javascript/mastodon/features/emoji/utils.ts b/app/javascript/mastodon/features/emoji/utils.ts index c567afc2cc..cce8151fb3 100644 --- a/app/javascript/mastodon/features/emoji/utils.ts +++ b/app/javascript/mastodon/features/emoji/utils.ts @@ -1,4 +1,5 @@ import debug from 'debug'; +import { fromUnicodeToHexcode } from 'emojibase'; import { emojiRegexPolyfill } from '@/mastodon/polyfills'; @@ -44,6 +45,10 @@ export function anyEmojiRegex() { ); } +export function emojiToUnicodeHex(emoji: string): string { + return fromUnicodeToHexcode(emoji, false); +} + function supportsRegExpSets() { return 'unicodeSets' in RegExp.prototype; } diff --git a/app/javascript/testing/factories.ts b/app/javascript/testing/factories.ts index 26b020d8c2..6f2a45e58f 100644 --- a/app/javascript/testing/factories.ts +++ b/app/javascript/testing/factories.ts @@ -119,6 +119,9 @@ export function unicodeEmojiFactory( label: 'Test', unicode: '๐Ÿงช', shortcodes: ['test_emoji'], + tokens: ['emoji', 'test'], + group: 1, + order: 1, ...data, }; } @@ -131,6 +134,7 @@ export function customEmojiFactory( static_url: '/custom-emoji/logo.svg', url: '/custom-emoji/logo.svg', visible_in_picker: true, + tokens: ['custom'], ...data, }; }