diff --git a/app/javascript/flavours/glitch/actions/compose.js b/app/javascript/flavours/glitch/actions/compose.js index 8f66e5c668..4973d228b0 100644 --- a/app/javascript/flavours/glitch/actions/compose.js +++ b/app/javascript/flavours/glitch/actions/compose.js @@ -603,7 +603,7 @@ const fetchComposeSuggestionsTags = throttle((dispatch, token) => { }, 200, { leading: true, trailing: true }); export function fetchComposeSuggestions(token) { - return (dispatch, getState) => { + return (dispatch) => { switch (token[0]) { case ':': void fetchComposeSuggestionsEmojis(dispatch, token); diff --git a/app/javascript/flavours/glitch/components/autosuggest/utils.test.ts b/app/javascript/flavours/glitch/components/autosuggest/utils.test.ts index 3e92e2b154..79b8b767b8 100644 --- a/app/javascript/flavours/glitch/components/autosuggest/utils.test.ts +++ b/app/javascript/flavours/glitch/components/autosuggest/utils.test.ts @@ -10,6 +10,14 @@ describe('textAtCursorMatchesToken', () => { ['#hash tag', 8, ['#']], [1, '#hash tag'], ], + [ + [':+1', 2, [':']], + [1, ':+1'], + ], + [ + [':-1', 2, [':']], + [1, ':-1'], + ], [ ['#ハッシュタグ', 6, ['#']], [1, '#ハッシュタグ'], diff --git a/app/javascript/flavours/glitch/components/autosuggest/utils.ts b/app/javascript/flavours/glitch/components/autosuggest/utils.ts index 52268bf3cd..3e02114a97 100644 --- a/app/javascript/flavours/glitch/components/autosuggest/utils.ts +++ b/app/javascript/flavours/glitch/components/autosuggest/utils.ts @@ -8,7 +8,7 @@ export const textAtCursorMatchesToken = ( let word: string; const regex = new RegExp( - `[${searchTokens.join('')}${WORD}]+(\\s[${WORD}]+)?$`, + `[${searchTokens.join('')}${WORD}+-]+(\\s[${WORD}]+)?$`, 'iu', ); const left = str.slice(0, caretPosition).search(regex); diff --git a/app/javascript/flavours/glitch/features/emoji/database.test.ts b/app/javascript/flavours/glitch/features/emoji/database.test.ts index f51d1af30c..27f958f0a2 100644 --- a/app/javascript/flavours/glitch/features/emoji/database.test.ts +++ b/app/javascript/flavours/glitch/features/emoji/database.test.ts @@ -5,7 +5,6 @@ import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories'; import { putEmojiData, - search, loadEmojiByHexcode, testClear, testGet, @@ -15,10 +14,11 @@ import { } from './database'; function rawEmojiFactory(data: Partial = {}): CompactEmoji { + const factory = unicodeEmojiFactory(); return { - ...unicodeEmojiFactory(), - tags: ['test', 'emoji'], + ...factory, ...data, + tags: data.tags ?? factory.tokens, }; } @@ -32,108 +32,6 @@ describe('emoji database', () => { indexedDB = new IDBFactory(); }); - describe('search', () => { - beforeEach(async () => { - await putEmojiData([], 'en'); - }); - - test('test no query tokens', async () => { - await putEmojiData([rawEmojiFactory()], 'en'); - await expect(search({ query: ' ', locale: 'en' })).resolves.toEqual([]); - }); - - test('unicode results', async () => { - await putEmojiData( - [ - rawEmojiFactory({ - hexcode: 'unicode_hex', - label: 'Party Popper', - shortcodes: ['party_popper'], - unicode: '🎉', - }), - ], - 'en', - ); - - await expect( - search({ query: 'party', locale: 'en' }), - ).resolves.toContainEqual( - expect.objectContaining({ - hexcode: 'unicode_hex', - }), - ); - }); - - test('custom results', async () => { - await putCustomEmojiData({ - emojis: [customEmojiFactory({ shortcode: 'party_custom' })], - }); - - await expect( - search({ query: 'party', locale: 'en' }), - ).resolves.toContainEqual( - expect.objectContaining({ - shortcode: 'party_custom', - }), - ); - }); - - test('shortcode results', async () => { - await putEmojiData([rawEmojiFactory()], 'en'); - await putLegacyShortcodes({ - test: ['legacy_smile'], - }); - - await expect( - search({ query: 'legacy', locale: 'en' }), - ).resolves.toContainEqual( - expect.objectContaining({ - hexcode: 'test', - }), - ); - }); - - test('full custom emoji search', async () => { - await putCustomEmojiData({ - emojis: [ - customEmojiFactory({ shortcode: 'arrow' }), - customEmojiFactory({ shortcode: 'party_parrot' }), - ], - }); - - const result = await search({ query: 'arro', locale: 'en' }); - expect(result).toContainEqual( - // Test for ordinary IDB search - expect.objectContaining({ - shortcode: 'arrow', - }), - ); - expect(result).toContainEqual( - // Test for manual iteration search - expect.objectContaining({ - shortcode: 'party_parrot', - }), - ); - }); - - test('limit test', async () => { - await putCustomEmojiData({ - emojis: [ - customEmojiFactory({ shortcode: 'limit' }), - customEmojiFactory({ shortcode: 'limit_extra' }), - ], - }); - - await expect( - search({ query: 'limit', locale: 'en', limit: 1 }), - ).resolves.toEqual([ - expect.objectContaining({ - shortcode: 'limit', - }), - ]); - }); - }); - describe('putEmojiData', () => { test('adds to loaded locales', async () => { const { loadedLocales } = await testGet(); diff --git a/app/javascript/flavours/glitch/features/emoji/database.ts b/app/javascript/flavours/glitch/features/emoji/database.ts index 36d009f5f4..2d24491d93 100644 --- a/app/javascript/flavours/glitch/features/emoji/database.ts +++ b/app/javascript/flavours/glitch/features/emoji/database.ts @@ -9,12 +9,11 @@ import type { Database } from './db-schema'; import { importEmojiData } from './loader'; import { localeToSegmenter, toSupportedLocale } from './locale'; import { - extractTokens, skinHexcodeToEmoji, transformCustomEmojiData, transformEmojiData, } from './normalize'; -import type { AnyEmojiData, CacheKey, CustomEmojiData } from './types'; +import type { CacheKey } from './types'; import { emojiLogger } from './utils'; const loadedLocales = new Set(); @@ -43,223 +42,22 @@ const loadDB = (() => { return loadPromise; })(); -type ScoreMap = Map; - -export async function search({ - query, - locale: localeString, - limit = 0, -}: { - query: string; - locale: string; - limit?: number; -}) { - performance.mark('emoji-search-start'); - - // Get the locale, and extract tokens from the query. - const locale = await toLoadedLocale(localeString); - const segmenter = localeToSegmenter(locale); - const queryTokens = extractTokens(query, segmenter); - - if (queryTokens.length === 0) { - log('no tokens extracted from query "%s"', query); - return []; - } - const lastToken = queryTokens.at(-1); - if (!lastToken) { - throw new Error('Missing tokens from query'); - } - - log('searching for tokens %o in locale %s', queryTokens, locale); - - // Create an array of emoji results +export async function rawSearch(query: string, locale: Locale, prefix = true) { + await toLoadedLocale(locale); const db = await loadDB(); - const resultArrays: ScoreMap[] = []; - const existingCustomShortcodes = new Set(); - - for (let i = 0; i < queryTokens.length; i++) { - const token = queryTokens[i]; - if (!token) continue; - - // Only query the range for the last token to allow partial matches. - const range = - i === queryTokens.length - 1 - ? IDBKeyRange.lowerBound(token) - : IDBKeyRange.only(token); - - const [unicodeResults, customResults, shortcodeResults] = await Promise.all( - [ - db.getAllFromIndex(locale, 'tokens', range), - db.getAllFromIndex('custom', 'tokens', range), - db.getAllFromIndex('shortcodes', 'shortcodes', range), - ], - ); - const resultMap: ScoreMap = new Map(); - - for (const emoji of unicodeResults) { - const score = getScoreForEmoji(emoji, token); - if (score === null) { - continue; - } - resultMap.set(emoji.hexcode, { ...emoji, score }); - } - - for (const emoji of customResults) { - const score = getScoreForEmoji(emoji, token); - if (score === null) { - continue; - } - existingCustomShortcodes.add(emoji.shortcode); - resultMap.set(emoji.shortcode, { ...emoji, score }); - } - - for (const shortcodeResult of shortcodeResults) { - if (resultMap.has(shortcodeResult.hexcode)) { - continue; - } - const emoji = await db.get(locale, shortcodeResult.hexcode); - if (!emoji) { - continue; - } - // Score the emoji with the legacy shortcode, even though it's not part of the emoji. - const score = getScoreForEmoji( - { - ...emoji, - shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes], - }, - token, - ); - if (score === null) { - continue; - } - resultMap.set(emoji.hexcode, { ...emoji, score }); - } - - log('found %d results for token "%s"', resultMap.size, token); - resultArrays.push(resultMap); - } - - // Utilize maps to find the intersection of all result sets. - const results = Array.from( - resultArrays - .reduce((prev, curr) => { - const intersection: ScoreMap = new Map(); - for (const [code, emoji] of prev) { - if (curr.has(code)) { - intersection.set(code, emoji); - } - } - return intersection; - }) - .values(), - ); - - // If there are no results, try a cursor-based custom emoji search instead. - if (results.length === 0 || results.length < limit) { - const customEmojisFound = await fullCustomSearch( - query, - existingCustomShortcodes, - ); - if (customEmojisFound.length > 0) { - log( - 'cursor search found %d results for "%s"', - customEmojisFound.length, - query, - ); - results.push(...customEmojisFound); - } - } - - // Sort by score, descending. - results.sort((a, b) => a.score - b.score); - - const time = performance.measure('emoji-search-end', 'emoji-search-start'); - log( - 'search for "%s" in locale %s returned %d results and took %dms', - query, - locale, - results.length, - time.duration, - ); - if (limit > 0) { - return results.slice(0, limit); - } - return results; -} - -function getScoreForEmoji( - emoji: AnyEmojiData, - query: string, - checkTokens = true, -) { - const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label; - if (id === query) { - return 0; - } - - let index = 1; - const searchTokens = [id]; - if (checkTokens) { - // Check shortcodes before tokens as they are more important. - if ('shortcodes' in emoji) { - searchTokens.push(...emoji.shortcodes); - } - searchTokens.push(...emoji.tokens); - } - for (const token of searchTokens) { - const tokenIndex = token.indexOf(query); - if (tokenIndex !== -1) { - return index + tokenIndex / token.length; - } - index++; - } - - return null; -} - -async function fullCustomSearch(query: string, existing = new Set()) { - const db = await loadDB(); - const trx = db.transaction('custom', 'readonly'); - const foundEmojis = new Set(); - - // First iterate over chunks of 1,000 custom emoji keys and find any matches. - const chunkSize = 1_000; - let lastKey: string | null = null; - let keys: string[] = []; - do { - const keyRange = lastKey ? IDBKeyRange.lowerBound(lastKey, true) : null; - keys = await trx.store.getAllKeys(keyRange, chunkSize); - - if (keys.length === 0) { - break; - } - log('cursor search got batch of %d emojis', keys.length); - lastKey = keys.at(-1) ?? null; - - for (const key of keys) { - if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) { - foundEmojis.add(key); - } - } - } while (keys.length === chunkSize); - - // Next get the full emojis for all matches. - const emojis = await Promise.all( - foundEmojis.keys().map((key) => trx.store.get(key)), - ); - const results: (CustomEmojiData & { score: number })[] = []; - for (const emoji of emojis) { - if (emoji) { - const score = getScoreForEmoji(emoji, query, false); - if (score && score > 0) { - results.push({ - score, - ...emoji, - }); - } - } - } - return results; + const range = prefix + ? IDBKeyRange.lowerBound(query) + : IDBKeyRange.only(query); + const [unicodeResults, customResults, shortcodeResults] = await Promise.all([ + db.getAllFromIndex(locale, 'tokens', range), + db.getAllFromIndex('custom', 'tokens', range), + db.getAllFromIndex('shortcodes', 'shortcodes', range), + ]); + return { + unicodeResults, + customResults, + shortcodeResults, + }; } export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) { @@ -369,6 +167,9 @@ export async function loadCustomEmojiByShortcode(shortcode: string) { } export async function searchCustomEmojisByShortcodes(shortcodes: string[]) { + if (shortcodes.length === 0) { + return []; + } const db = await loadDB(); const sortedCodes = shortcodes.toSorted(); const results = await db.getAll( @@ -378,6 +179,15 @@ export async function searchCustomEmojisByShortcodes(shortcodes: string[]) { return results.filter((emoji) => shortcodes.includes(emoji.shortcode)); } +export async function loadCustomEmojiKeys( + query?: string | null, + chunkSize = 1_000, +) { + const db = await loadDB(); + const keyRange = query ? IDBKeyRange.lowerBound(query, true) : null; + return db.getAllKeys('custom', keyRange, chunkSize); +} + export async function loadAllCustomEmoji() { const db = await loadDB(); const cacheValue = await db.get('etags', 'custom'); diff --git a/app/javascript/flavours/glitch/features/emoji/db-schema.ts b/app/javascript/flavours/glitch/features/emoji/db-schema.ts index b809ef63a8..661c658979 100644 --- a/app/javascript/flavours/glitch/features/emoji/db-schema.ts +++ b/app/javascript/flavours/glitch/features/emoji/db-schema.ts @@ -64,7 +64,7 @@ type Transaction = export type Database = IDBPDatabase; const DATABASE_NAME = 'mastodon-emoji'; -const SCHEMA_VERSION = 4; +const SCHEMA_VERSION = 5; export async function openEmojiDB() { try { diff --git a/app/javascript/flavours/glitch/features/emoji/normalize.ts b/app/javascript/flavours/glitch/features/emoji/normalize.ts index 2d77fc2944..7f2c6308b1 100644 --- a/app/javascript/flavours/glitch/features/emoji/normalize.ts +++ b/app/javascript/flavours/glitch/features/emoji/normalize.ts @@ -60,7 +60,7 @@ export function transformEmojiData( ...extract(label), ...(normalizedEmoticons ?? []), ]), - ].sort((a, b) => a.localeCompare(b)); + ]; const res: UnicodeEmojiData = { tokens, diff --git a/app/javascript/flavours/glitch/features/emoji/picker.ts b/app/javascript/flavours/glitch/features/emoji/picker.ts index c8abfd9f96..cd0dea99da 100644 --- a/app/javascript/flavours/glitch/features/emoji/picker.ts +++ b/app/javascript/flavours/glitch/features/emoji/picker.ts @@ -7,6 +7,7 @@ import { } from '@/flavours/glitch/store/typed_functions'; import { createLimitedCache } from '@/flavours/glitch/utils/cache'; +import { search } from './search'; import { emojiLogger } from './utils'; const log = emojiLogger('picker'); @@ -37,7 +38,6 @@ export async function emojiMartSearch( return cachedResult; } - const { search } = await import('./database'); const results = await search({ query, locale, limit }); const legacyResults = results.map((emoji) => 'shortcode' in emoji diff --git a/app/javascript/flavours/glitch/features/emoji/search.test.ts b/app/javascript/flavours/glitch/features/emoji/search.test.ts new file mode 100644 index 0000000000..c861fe4532 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/search.test.ts @@ -0,0 +1,188 @@ +import type { CompactEmoji } from 'emojibase'; + +import { unicodeEmojiFactory, customEmojiFactory } from '@/testing/factories'; + +import { + putEmojiData, + putCustomEmojiData, + putLegacyShortcodes, + testGet, + testClear, +} from './database'; +import { search } from './search'; + +function rawEmojiFactory(data: Partial = {}): CompactEmoji { + const factory = unicodeEmojiFactory(); + return { + ...factory, + ...data, + tags: data.tags ?? factory.tokens, + }; +} + +describe('search', () => { + beforeEach(async () => { + await testGet(); // Loads the database schema. + await putEmojiData([], 'en'); + }); + + afterEach(() => { + testClear(); + indexedDB = new IDBFactory(); + }); + + test('test no query tokens', async () => { + await putEmojiData([rawEmojiFactory()], 'en'); + await expect(search({ query: ' ', locale: 'en' })).resolves.toEqual([]); + }); + + test('unicode results', async () => { + await putEmojiData( + [ + rawEmojiFactory({ + hexcode: 'unicode_hex', + label: 'Party Popper', + shortcodes: ['party_popper'], + unicode: '🎉', + }), + ], + 'en', + ); + + await expect( + search({ query: 'party', locale: 'en' }), + ).resolves.toContainEqual( + expect.objectContaining({ + hexcode: 'unicode_hex', + }), + ); + }); + + test('custom results', async () => { + await putCustomEmojiData({ + emojis: [customEmojiFactory({ shortcode: 'party_custom' })], + }); + + await expect( + search({ query: 'party', locale: 'en' }), + ).resolves.toContainEqual( + expect.objectContaining({ + shortcode: 'party_custom', + }), + ); + }); + + test('shortcode results', async () => { + await putEmojiData([rawEmojiFactory()], 'en'); + await putLegacyShortcodes({ + test: ['legacy_smile'], + }); + + await expect( + search({ query: 'legacy', locale: 'en' }), + ).resolves.toContainEqual( + expect.objectContaining({ + hexcode: 'test', + }), + ); + }); + + test('full custom emoji search', async () => { + await putCustomEmojiData({ + emojis: [ + customEmojiFactory({ shortcode: 'arrow' }), + customEmojiFactory({ shortcode: 'party_parrot' }), + ], + }); + + const result = await search({ query: 'arro', locale: 'en' }); + expect(result).toContainEqual( + // Test for ordinary IDB search + expect.objectContaining({ + shortcode: 'arrow', + }), + ); + expect(result).toContainEqual( + // Test for manual iteration search + expect.objectContaining({ + shortcode: 'party_parrot', + }), + ); + }); + + test('limit test', async () => { + await putCustomEmojiData({ + emojis: [ + customEmojiFactory({ shortcode: 'limit' }), + customEmojiFactory({ shortcode: 'limit_extra' }), + ], + }); + + await expect( + search({ query: 'limit', locale: 'en', limit: 1 }), + ).resolves.toEqual([ + expect.objectContaining({ + shortcode: 'limit', + }), + ]); + }); + + test('prefix matches', async () => { + await putCustomEmojiData({ + emojis: [ + customEmojiFactory({ shortcode: 'sob_other' }), + customEmojiFactory({ shortcode: 'meow_sob' }), + ], + }); + await putEmojiData( + [ + rawEmojiFactory({ + label: 'loudly crying face', + hexcode: '1F62D', + shortcodes: ['loudly_crying_face'], + tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'], + emoticon: ":'o", + unicode: '😭', + }), + ], + 'en', + ); + + const results = await search({ query: 'sob', locale: 'en' }); + + expect(results).toHaveLength(3); + expect(results).toEqual([ + expect.objectContaining({ shortcode: 'sob_other' }), + expect.objectContaining({ shortcode: 'meow_sob' }), + expect.objectContaining({ hexcode: '1F62D' }), + ]); + }); + + test('shortcode matches', async () => { + await putLegacyShortcodes({ '1F62D': 'sob' }); + await putEmojiData( + [ + rawEmojiFactory({ + label: 'loudly crying face', + hexcode: '1F62D', + shortcodes: ['loudly_crying_face'], + tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'], + emoticon: ":'o", + unicode: '😭', + }), + ], + 'en', + ); + await putCustomEmojiData({ + emojis: [customEmojiFactory({ shortcode: 'sob_other' })], + }); + + const results = await search({ query: 'sob', locale: 'en' }); + + expect(results).toHaveLength(2); + expect(results).toEqual([ + expect.objectContaining({ hexcode: '1F62D' }), + expect.objectContaining({ shortcode: 'sob_other' }), + ]); + }); +}); diff --git a/app/javascript/flavours/glitch/features/emoji/search.ts b/app/javascript/flavours/glitch/features/emoji/search.ts new file mode 100644 index 0000000000..dcdd6be500 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/search.ts @@ -0,0 +1,407 @@ +import { log } from 'debug'; +import type { ArrayValues, KeysOfUnion } from 'type-fest'; + +import { + loadCustomEmojiKeys, + loadEmojiByHexcode, + rawSearch, + searchCustomEmojisByShortcodes, +} from './database'; +import { localeToSegmenter, toSupportedLocale } from './locale'; +import { extractTokens } from './normalize'; +import type { AnyEmojiData, CustomEmojiData } from './types'; + +/* +Emoji search logic: +1. When provided a string, extract all tokens and iterate over each. +2. For each token, do an IDB lookup to get any matches on emojis and shortcodes. +3. Score each emoji found (see below). +4. Create a final list of emojis that is the intersection of all token results, with the scores combined. +5. If not enough emoji are found, do a cursor search on custom emojis. +6. Sort and return the emoji by ranked score, stopping when the optional limit is reached. + +Scoring functions as follows: +- Go over every scoreRanking field in order and extract score data. +- Scores prefer exact match, then prefix match, and lastly substring match. +- When sorting, prefer identifier fields first, then score, then category, and lastly prioritize Unicode emojis. +*/ + +const scoreRanking = [ + 'label', + 'shortcode', + 'emoticons', + 'shortcodes', + 'tokens', +] as const satisfies KeysOfUnion[]; +type ScoreRankingKeys = ArrayValues; +type ScoreRanking = Record; + +const identifierFields = new Set([ + 'label', + 'shortcode', + 'emoticons', + 'shortcodes', +]); + +interface BestRank { + categoryWeight: number; + score: number; + fieldWeight: number; +} +type ScoredEmoji = AnyEmojiData & { scores: ScoreRanking }; +type RankedEmoji = AnyEmojiData & { rank: BestRank }; +type ScoreMap = Map; + +export async function search({ + query: rawQuery, + locale: localeString, + limit = 0, +}: { + query: string; + locale: string; + limit?: number; +}) { + performance.mark('emoji-search-start'); + + // Get the locale, and extract tokens from the query. + const locale = toSupportedLocale(localeString); + const segmenter = localeToSegmenter(locale); + const query = rawQuery.toLowerCase(); + const queryTokens = extractTokens(query, segmenter); + + if (queryTokens.length === 0) { + log('no tokens extracted from query "%s"', query); + return []; + } + const lastToken = queryTokens.at(-1); + if (!lastToken) { + throw new Error('Missing tokens from query'); + } + + log('searching for tokens %o in locale %s', queryTokens, locale); + + // Create an array of emoji results + const resultArrays: ScoreMap[] = []; + + for (let i = 0; i < queryTokens.length; i++) { + const token = queryTokens[i]; + if (!token) continue; + + // Only query the range for the last token to allow partial matches. + const { unicodeResults, customResults, shortcodeResults } = await rawSearch( + token, + locale, + i === queryTokens.length - 1, + ); + const resultMap: ScoreMap = new Map(); + const checkedSet = new Set(); + + // Score unicode results. + for (const emoji of unicodeResults) { + if (checkedSet.has(emoji.hexcode)) { + continue; + } + checkedSet.add(emoji.hexcode); + const scores = getScoreForEmoji(emoji, token); + if (scores) { + resultMap.set(emoji.hexcode, { ...emoji, scores }); + } + } + + // Score custom emojis. + for (const emoji of customResults) { + if (checkedSet.has(emoji.shortcode)) { + continue; + } + checkedSet.add(emoji.shortcode); + const scores = getScoreForEmoji(emoji, token); + if (scores) { + resultMap.set(emoji.shortcode, { ...emoji, scores }); + } + } + + // Score based on legacy shortcodes, using the higher score if there's a match. + for (const shortcodeResult of shortcodeResults) { + const emoji = + resultMap.get(shortcodeResult.hexcode) ?? + (await loadEmojiByHexcode(shortcodeResult.hexcode, locale)); + if (!emoji || !('hexcode' in emoji)) { + continue; + } + + const newScores = getScoreForEmoji( + { + ...emoji, + shortcodes: shortcodeResult.shortcodes, + }, + token, + ); + if (!newScores) { + continue; + } + const oldScores = resultMap.get(emoji.hexcode)?.scores; + const scores = oldScores + ? combineEmojiScores(oldScores, newScores) + : newScores; + resultMap.set(emoji.hexcode, { + ...emoji, + shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes], + scores, + }); + } + + log('found %d results for token "%s"', resultMap.size, token); + resultArrays.push(resultMap); + } + + // Iterate over all maps, getting a combined score for emojis that exist in all results. + const allEmojiIds = resultArrays.reduce((prev, map) => { + if (prev.size === 0) { + return new Set(map.keys()); + } + return new Set(map.keys()).intersection(prev); + }, new Set()); + + const finalMap: ScoreMap = new Map(); + for (const resultArray of resultArrays) { + for (const [id, emoji] of resultArray.entries()) { + if (!allEmojiIds.has(id)) { + continue; + } + const existingEmoji = finalMap.get(id); + if (!existingEmoji) { + finalMap.set(id, emoji); + } else { + finalMap.set(id, { + ...existingEmoji, + scores: combineEmojiScores(existingEmoji.scores, emoji.scores), + }); + } + } + } + const mixedResults = Array.from(finalMap.values()); + + // If there are no results, try a cursor-based custom emoji search instead. + if (mixedResults.length === 0 || mixedResults.length < limit) { + const customEmojisFound = await fullCustomSearch(query, allEmojiIds); + if (customEmojisFound.length > 0) { + log( + 'cursor search found %d results for "%s"', + customEmojisFound.length, + query, + ); + mixedResults.push(...customEmojisFound); + } + } + + const rankedEmojis = mixedResults.map(({ scores, ...emoji }) => ({ + ...emoji, + rank: getBestRank(scores), + })); + + const results: RankedEmoji[] = []; + const resultEmojis = new Set(); + for (const result of rankedEmojis.toSorted(compareRankedEmoji)) { + const id = getIdentifier(result); + if (resultEmojis.has(id)) { + continue; + } + results.push(result); + resultEmojis.add(id); + if (limit > 0 && results.length >= limit) { + break; + } + } + + const time = performance.measure('emoji-search-end', 'emoji-search-start'); + log( + 'search for "%s" in locale %s returned %d results and took %dms', + query, + locale, + mixedResults.length, + time.duration, + ); + return results; +} + +function hasField( + emoji: AnyEmojiData, + field: string, +): field is keyof typeof emoji { + return Object.hasOwn(emoji, field); +} + +function getIdentifier(emoji: AnyEmojiData) { + return 'shortcode' in emoji ? emoji.shortcode : emoji.hexcode; +} + +// Creates ranked scores for a given emoji. +function getScoreForEmoji(emoji: AnyEmojiData, query: string) { + const scores = Object.fromEntries( + scoreRanking.map((field) => [field, -1]), + ) as ScoreRanking; + let hasScore = false; + + for (const field of scoreRanking) { + if (hasField(emoji, field)) { + const value = emoji[field] as string | string[] | undefined; + if (value === undefined) { + continue; + } + const tokens = Array.isArray(value) ? value : [value]; + const score = getScoreForEmojiTokens( + tokens.map((token) => token.toLowerCase()), + query, + ); + + if (score >= 0) { + scores[field] = score; + hasScore = true; + } + } + } + + if (!hasScore) { + return null; + } + + return scores; +} + +function combineEmojiScores(a: ScoreRanking, b: ScoreRanking): ScoreRanking { + const scores = Object.fromEntries( + scoreRanking.map((field) => [field, -1]), + ) as ScoreRanking; + + for (const rank of scoreRanking) { + if (a[rank] === -1) { + scores[rank] = b[rank]; + } else if (b[rank] === -1) { + scores[rank] = a[rank]; + } else { + scores[rank] = Math.min(a[rank], b[rank]); + } + } + + return scores; +} + +// Compares two scored emojis by getting the best rank. +function compareRankedEmoji(a: RankedEmoji, b: RankedEmoji): number { + const rankA = a.rank; + const rankB = b.rank; + + // Identifier matches always outrank token matches, regardless of score. + if (rankA.categoryWeight !== rankB.categoryWeight) { + return rankA.categoryWeight - rankB.categoryWeight; + } + // Within the same category, compare the scores directly. + if (rankA.score !== rankB.score) { + return rankA.score - rankB.score; + } + // If equal, compare field weights. + if (rankA.fieldWeight !== rankB.fieldWeight) { + return rankA.fieldWeight - rankB.fieldWeight; + } + + // Lastly prioritize Unicode emojis. + const aIsCustom = hasField(a, 'shortcode'); + const bIsCustom = hasField(b, 'shortcode'); + if (aIsCustom !== bIsCustom) { + return aIsCustom ? -1 : 1; + } + + return 0; +} + +// Extracts the best rank for a score ranking. +function getBestRank(scores: ScoreRanking): BestRank { + let best: BestRank | null = null; + + // Use the index to determine field weight. + for (const [fieldWeight, field] of scoreRanking.entries()) { + const score = scores[field]; + if (score < 0) { + continue; + } + // Also weight identifier fields over other fields. + const categoryWeight = identifierFields.has(field) ? 0 : 1; + if ( + !best || + categoryWeight < best.categoryWeight || + (categoryWeight === best.categoryWeight && + (score < best.score || + (score === best.score && fieldWeight < best.fieldWeight))) + ) { + best = { categoryWeight, score, fieldWeight }; + } + } + + return ( + best ?? { + categoryWeight: 2, + score: Infinity, + fieldWeight: scoreRanking.length, + } + ); +} + +function getScoreForEmojiTokens(tokens: string[], query: string) { + let lowestScore = -1; + + for (const token of tokens) { + let score = -1; + // Priority: exact match, prefix match, substring, + if (token === query) { + score = 0; + } else if (token.startsWith(query)) { + score = 1 + query.length / token.length; + } else if (token.includes(query)) { + score = 2 + query.length / token.length; + } // TODO: Add fuzzy search if needed + + if (score >= 0 && (score < lowestScore || lowestScore < 0)) { + lowestScore = score; + } + } + + return lowestScore; +} + +async function fullCustomSearch(query: string, existing = new Set()) { + const foundEmojis = new Set(); + + // First iterate over chunks of 1,000 custom emoji keys and find any matches. + const chunkSize = 1_000; + let lastKey: string | null = null; + let keys: string[] = []; + do { + keys = await loadCustomEmojiKeys(lastKey, chunkSize); + + if (keys.length === 0) { + break; + } + log('cursor search got batch of %d emojis', keys.length); + lastKey = keys.at(-1) ?? null; + + for (const key of keys) { + if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) { + foundEmojis.add(key); + } + } + } while (keys.length === chunkSize); + + // Next get the full emojis for all matches. + const emojis = await searchCustomEmojisByShortcodes(Array.from(foundEmojis)); + const results: (CustomEmojiData & { scores: ScoreRanking })[] = []; + for (const emoji of emojis) { + const scores = getScoreForEmoji(emoji, query); + if (scores) { + results.push({ + scores, + ...emoji, + }); + } + } + return results; +}