tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve emoji search (#39815)

Browse Source
This commit is contained in:
Echo 2026-07-20 11:29:37 +02:00 committed by Tarrien
parent e0fd68866e
commit 11f28e1474
13 changed files with 652 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/javascript/mastodon/actions/compose.js
View File

@ -572,7 +572,7 @@ const fetchComposeSuggestionsTags = throttle((dispatch, token) => {
}, 200, { leading: true, trailing: true });
export function fetchComposeSuggestions(token) {
return (dispatch, getState) => {
return (dispatch) => {
switch (token[0]) {
case ':':
void fetchComposeSuggestionsEmojis(dispatch, token);

8
app/javascript/mastodon/components/autosuggest/utils.test.ts
View File

@ -10,6 +10,14 @@ describe('textAtCursorMatchesToken', () => {
['#hash tag', 8, ['#']],
[1, '#hash tag'],
],
[
[':+1', 2, [':']],
[1, ':+1'],
],
[
[':-1', 2, [':']],
[1, ':-1'],
],
[
['#ハッシュタグ', 6, ['#']],
[1, '#ハッシュタグ'],

2
app/javascript/mastodon/components/autosuggest/utils.ts
View File

@ -8,7 +8,7 @@ export const textAtCursorMatchesToken = (
let word: string;
const regex = new RegExp(
`[${searchTokens.join('')}${WORD}]+(\\s[${WORD}]+)?$`,
`[${searchTokens.join('')}${WORD}+-]+(\\s[${WORD}]+)?$`,
'iu',
);
const left = str.slice(0, caretPosition).search(regex);

108
app/javascript/mastodon/features/emoji/database.test.ts
View File

@ -5,7 +5,6 @@ import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
import {
putEmojiData,
search,
loadEmojiByHexcode,
testClear,
testGet,
@ -15,10 +14,11 @@ import {
} from './database';
function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
const factory = unicodeEmojiFactory();
return {
...unicodeEmojiFactory(),
tags: ['test', 'emoji'],
...factory,
...data,
tags: data.tags ?? factory.tokens,
};
}
@ -32,108 +32,6 @@ describe('emoji database', () => {
indexedDB = new IDBFactory();
});
describe('search', () => {
beforeEach(async () => {
await putEmojiData([], 'en');
});
test('test no query tokens', async () => {
await putEmojiData([rawEmojiFactory()], 'en');
await expect(search({ query: ' ', locale: 'en' })).resolves.toEqual([]);
});
test('unicode results', async () => {
await putEmojiData(
[
rawEmojiFactory({
hexcode: 'unicode_hex',
label: 'Party Popper',
shortcodes: ['party_popper'],
unicode: '🎉',
}),
],
'en',
);
await expect(
search({ query: 'party', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
hexcode: 'unicode_hex',
}),
);
});
test('custom results', async () => {
await putCustomEmojiData({
emojis: [customEmojiFactory({ shortcode: 'party_custom' })],
});
await expect(
search({ query: 'party', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
shortcode: 'party_custom',
}),
);
});
test('shortcode results', async () => {
await putEmojiData([rawEmojiFactory()], 'en');
await putLegacyShortcodes({
test: ['legacy_smile'],
});
await expect(
search({ query: 'legacy', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
hexcode: 'test',
}),
);
});
test('full custom emoji search', async () => {
await putCustomEmojiData({
emojis: [
customEmojiFactory({ shortcode: 'arrow' }),
customEmojiFactory({ shortcode: 'party_parrot' }),
],
});
const result = await search({ query: 'arro', locale: 'en' });
expect(result).toContainEqual(
// Test for ordinary IDB search
expect.objectContaining({
shortcode: 'arrow',
}),
);
expect(result).toContainEqual(
// Test for manual iteration search
expect.objectContaining({
shortcode: 'party_parrot',
}),
);
});
test('limit test', async () => {
await putCustomEmojiData({
emojis: [
customEmojiFactory({ shortcode: 'limit' }),
customEmojiFactory({ shortcode: 'limit_extra' }),
],
});
await expect(
search({ query: 'limit', locale: 'en', limit: 1 }),
).resolves.toEqual([
expect.objectContaining({
shortcode: 'limit',
}),
]);
});
});
describe('putEmojiData', () => {
test('adds to loaded locales', async () => {
const { loadedLocales } = await testGet();

240
app/javascript/mastodon/features/emoji/database.ts
View File

@ -9,12 +9,11 @@ import type { Database } from './db-schema';
import { importEmojiData } from './loader';
import { localeToSegmenter, toSupportedLocale } from './locale';
import {
extractTokens,
skinHexcodeToEmoji,
transformCustomEmojiData,
transformEmojiData,
} from './normalize';
import type { AnyEmojiData, CacheKey, CustomEmojiData } from './types';
import type { CacheKey } from './types';
import { emojiLogger } from './utils';
const loadedLocales = new Set<Locale>();
@ -43,223 +42,22 @@ const loadDB = (() => {
return loadPromise;
})();
type ScoreMap = Map<string, AnyEmojiData & { score: number }>;
export async function search({
query,
locale: localeString,
limit = 0,
}: {
query: string;
locale: string;
limit?: number;
}) {
performance.mark('emoji-search-start');
// Get the locale, and extract tokens from the query.
const locale = await toLoadedLocale(localeString);
const segmenter = localeToSegmenter(locale);
const queryTokens = extractTokens(query, segmenter);
if (queryTokens.length === 0) {
log('no tokens extracted from query "%s"', query);
return [];
}
const lastToken = queryTokens.at(-1);
if (!lastToken) {
throw new Error('Missing tokens from query');
}
log('searching for tokens %o in locale %s', queryTokens, locale);
// Create an array of emoji results
export async function rawSearch(query: string, locale: Locale, prefix = true) {
await toLoadedLocale(locale);
const db = await loadDB();
const resultArrays: ScoreMap[] = [];
const existingCustomShortcodes = new Set<string>();
for (let i = 0; i < queryTokens.length; i++) {
const token = queryTokens[i];
if (!token) continue;
// Only query the range for the last token to allow partial matches.
const range =
i === queryTokens.length - 1
? IDBKeyRange.lowerBound(token)
: IDBKeyRange.only(token);
const [unicodeResults, customResults, shortcodeResults] = await Promise.all(
[
const range = prefix
? IDBKeyRange.lowerBound(query)
: IDBKeyRange.only(query);
const [unicodeResults, customResults, shortcodeResults] = await Promise.all([
db.getAllFromIndex(locale, 'tokens', range),
db.getAllFromIndex('custom', 'tokens', range),
db.getAllFromIndex('shortcodes', 'shortcodes', range),
],
);
const resultMap: ScoreMap = new Map();
for (const emoji of unicodeResults) {
const score = getScoreForEmoji(emoji, token);
if (score === null) {
continue;
}
resultMap.set(emoji.hexcode, { ...emoji, score });
}
for (const emoji of customResults) {
const score = getScoreForEmoji(emoji, token);
if (score === null) {
continue;
}
existingCustomShortcodes.add(emoji.shortcode);
resultMap.set(emoji.shortcode, { ...emoji, score });
}
for (const shortcodeResult of shortcodeResults) {
if (resultMap.has(shortcodeResult.hexcode)) {
continue;
}
const emoji = await db.get(locale, shortcodeResult.hexcode);
if (!emoji) {
continue;
}
// Score the emoji with the legacy shortcode, even though it's not part of the emoji.
const score = getScoreForEmoji(
{
...emoji,
shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes],
},
token,
);
if (score === null) {
continue;
}
resultMap.set(emoji.hexcode, { ...emoji, score });
}
log('found %d results for token "%s"', resultMap.size, token);
resultArrays.push(resultMap);
}
// Utilize maps to find the intersection of all result sets.
const results = Array.from(
resultArrays
.reduce((prev, curr) => {
const intersection: ScoreMap = new Map();
for (const [code, emoji] of prev) {
if (curr.has(code)) {
intersection.set(code, emoji);
}
}
return intersection;
})
.values(),
);
// If there are no results, try a cursor-based custom emoji search instead.
if (results.length === 0 || results.length < limit) {
const customEmojisFound = await fullCustomSearch(
query,
existingCustomShortcodes,
);
if (customEmojisFound.length > 0) {
log(
'cursor search found %d results for "%s"',
customEmojisFound.length,
query,
);
results.push(...customEmojisFound);
}
}
// Sort by score, descending.
results.sort((a, b) => a.score - b.score);
const time = performance.measure('emoji-search-end', 'emoji-search-start');
log(
'search for "%s" in locale %s returned %d results and took %dms',
query,
locale,
results.length,
time.duration,
);
if (limit > 0) {
return results.slice(0, limit);
}
return results;
}
function getScoreForEmoji(
emoji: AnyEmojiData,
query: string,
checkTokens = true,
) {
const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label;
if (id === query) {
return 0;
}
let index = 1;
const searchTokens = [id];
if (checkTokens) {
// Check shortcodes before tokens as they are more important.
if ('shortcodes' in emoji) {
searchTokens.push(...emoji.shortcodes);
}
searchTokens.push(...emoji.tokens);
}
for (const token of searchTokens) {
const tokenIndex = token.indexOf(query);
if (tokenIndex !== -1) {
return index + tokenIndex / token.length;
}
index++;
}
return null;
}
async function fullCustomSearch(query: string, existing = new Set<string>()) {
const db = await loadDB();
const trx = db.transaction('custom', 'readonly');
const foundEmojis = new Set<string>();
// First iterate over chunks of 1,000 custom emoji keys and find any matches.
const chunkSize = 1_000;
let lastKey: string | null = null;
let keys: string[] = [];
do {
const keyRange = lastKey ? IDBKeyRange.lowerBound(lastKey, true) : null;
keys = await trx.store.getAllKeys(keyRange, chunkSize);
if (keys.length === 0) {
break;
}
log('cursor search got batch of %d emojis', keys.length);
lastKey = keys.at(-1) ?? null;
for (const key of keys) {
if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) {
foundEmojis.add(key);
}
}
} while (keys.length === chunkSize);
// Next get the full emojis for all matches.
const emojis = await Promise.all(
foundEmojis.keys().map((key) => trx.store.get(key)),
);
const results: (CustomEmojiData & { score: number })[] = [];
for (const emoji of emojis) {
if (emoji) {
const score = getScoreForEmoji(emoji, query, false);
if (score && score > 0) {
results.push({
score,
...emoji,
});
}
}
}
return results;
]);
return {
unicodeResults,
customResults,
shortcodeResults,
};
}
export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) {
@ -369,6 +167,9 @@ export async function loadCustomEmojiByShortcode(shortcode: string) {
}
export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
if (shortcodes.length === 0) {
return [];
}
const db = await loadDB();
const sortedCodes = shortcodes.toSorted();
const results = await db.getAll(
@ -378,6 +179,15 @@ export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
return results.filter((emoji) => shortcodes.includes(emoji.shortcode));
}
export async function loadCustomEmojiKeys(
query?: string | null,
chunkSize = 1_000,
) {
const db = await loadDB();
const keyRange = query ? IDBKeyRange.lowerBound(query, true) : null;
return db.getAllKeys('custom', keyRange, chunkSize);
}
export async function loadAllCustomEmoji() {
const db = await loadDB();
const cacheValue = await db.get('etags', 'custom');

2
app/javascript/mastodon/features/emoji/db-schema.ts
View File

@ -64,7 +64,7 @@ type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
export type Database = IDBPDatabase<EmojiDB>;
const DATABASE_NAME = 'mastodon-emoji';
const SCHEMA_VERSION = 4;
const SCHEMA_VERSION = 5;
export async function openEmojiDB() {
try {

2
app/javascript/mastodon/features/emoji/normalize.ts
View File

@ -60,7 +60,7 @@ export function transformEmojiData(
...extract(label),
...(normalizedEmoticons ?? []),
]),
].sort((a, b) => a.localeCompare(b));
];
const res: UnicodeEmojiData = {
tokens,

2
app/javascript/mastodon/features/emoji/picker.ts
View File

@ -7,6 +7,7 @@ import {
} from '@/mastodon/store/typed_functions';
import { createLimitedCache } from '@/mastodon/utils/cache';
import { search } from './search';
import { emojiLogger } from './utils';
const log = emojiLogger('picker');
@ -37,7 +38,6 @@ export async function emojiMartSearch(
return cachedResult;
}
const { search } = await import('./database');
const results = await search({ query, locale, limit });
const legacyResults = results.map((emoji) =>
'shortcode' in emoji

188
app/javascript/mastodon/features/emoji/search.test.ts Normal file
View File

@ -0,0 +1,188 @@
import type { CompactEmoji } from 'emojibase';
import { unicodeEmojiFactory, customEmojiFactory } from '@/testing/factories';
import {
putEmojiData,
putCustomEmojiData,
putLegacyShortcodes,
testGet,
testClear,
} from './database';
import { search } from './search';
function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
const factory = unicodeEmojiFactory();
return {
...factory,
...data,
tags: data.tags ?? factory.tokens,
};
}
describe('search', () => {
beforeEach(async () => {
await testGet(); // Loads the database schema.
await putEmojiData([], 'en');
});
afterEach(() => {
testClear();
indexedDB = new IDBFactory();
});
test('test no query tokens', async () => {
await putEmojiData([rawEmojiFactory()], 'en');
await expect(search({ query: ' ', locale: 'en' })).resolves.toEqual([]);
});
test('unicode results', async () => {
await putEmojiData(
[
rawEmojiFactory({
hexcode: 'unicode_hex',
label: 'Party Popper',
shortcodes: ['party_popper'],
unicode: '🎉',
}),
],
'en',
);
await expect(
search({ query: 'party', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
hexcode: 'unicode_hex',
}),
);
});
test('custom results', async () => {
await putCustomEmojiData({
emojis: [customEmojiFactory({ shortcode: 'party_custom' })],
});
await expect(
search({ query: 'party', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
shortcode: 'party_custom',
}),
);
});
test('shortcode results', async () => {
await putEmojiData([rawEmojiFactory()], 'en');
await putLegacyShortcodes({
test: ['legacy_smile'],
});
await expect(
search({ query: 'legacy', locale: 'en' }),
).resolves.toContainEqual(
expect.objectContaining({
hexcode: 'test',
}),
);
});
test('full custom emoji search', async () => {
await putCustomEmojiData({
emojis: [
customEmojiFactory({ shortcode: 'arrow' }),
customEmojiFactory({ shortcode: 'party_parrot' }),
],
});
const result = await search({ query: 'arro', locale: 'en' });
expect(result).toContainEqual(
// Test for ordinary IDB search
expect.objectContaining({
shortcode: 'arrow',
}),
);
expect(result).toContainEqual(
// Test for manual iteration search
expect.objectContaining({
shortcode: 'party_parrot',
}),
);
});
test('limit test', async () => {
await putCustomEmojiData({
emojis: [
customEmojiFactory({ shortcode: 'limit' }),
customEmojiFactory({ shortcode: 'limit_extra' }),
],
});
await expect(
search({ query: 'limit', locale: 'en', limit: 1 }),
).resolves.toEqual([
expect.objectContaining({
shortcode: 'limit',
}),
]);
});
test('prefix matches', async () => {
await putCustomEmojiData({
emojis: [
customEmojiFactory({ shortcode: 'sob_other' }),
customEmojiFactory({ shortcode: 'meow_sob' }),
],
});
await putEmojiData(
[
rawEmojiFactory({
label: 'loudly crying face',
hexcode: '1F62D',
shortcodes: ['loudly_crying_face'],
tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'],
emoticon: ":'o",
unicode: '😭',
}),
],
'en',
);
const results = await search({ query: 'sob', locale: 'en' });
expect(results).toHaveLength(3);
expect(results).toEqual([
expect.objectContaining({ shortcode: 'sob_other' }),
expect.objectContaining({ shortcode: 'meow_sob' }),
expect.objectContaining({ hexcode: '1F62D' }),
]);
});
test('shortcode matches', async () => {
await putLegacyShortcodes({ '1F62D': 'sob' });
await putEmojiData(
[
rawEmojiFactory({
label: 'loudly crying face',
hexcode: '1F62D',
shortcodes: ['loudly_crying_face'],
tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'],
emoticon: ":'o",
unicode: '😭',
}),
],
'en',
);
await putCustomEmojiData({
emojis: [customEmojiFactory({ shortcode: 'sob_other' })],
});
const results = await search({ query: 'sob', locale: 'en' });
expect(results).toHaveLength(2);
expect(results).toEqual([
expect.objectContaining({ hexcode: '1F62D' }),
expect.objectContaining({ shortcode: 'sob_other' }),
]);
});
});

407
app/javascript/mastodon/features/emoji/search.ts Normal file
View File

@ -0,0 +1,407 @@
import { log } from 'debug';
import type { ArrayValues, KeysOfUnion } from 'type-fest';
import {
loadCustomEmojiKeys,
loadEmojiByHexcode,
rawSearch,
searchCustomEmojisByShortcodes,
} from './database';
import { localeToSegmenter, toSupportedLocale } from './locale';
import { extractTokens } from './normalize';
import type { AnyEmojiData, CustomEmojiData } from './types';
/*
Emoji search logic:
1. When provided a string, extract all tokens and iterate over each.
2. For each token, do an IDB lookup to get any matches on emojis and shortcodes.
3. Score each emoji found (see below).
4. Create a final list of emojis that is the intersection of all token results, with the scores combined.
5. If not enough emoji are found, do a cursor search on custom emojis.
6. Sort and return the emoji by ranked score, stopping when the optional limit is reached.
Scoring functions as follows:
- Go over every scoreRanking field in order and extract score data.
- Scores prefer exact match, then prefix match, and lastly substring match.
- When sorting, prefer identifier fields first, then score, then category, and lastly prioritize Unicode emojis.
*/
const scoreRanking = [
'label',
'shortcode',
'emoticons',
'shortcodes',
'tokens',
] as const satisfies KeysOfUnion<AnyEmojiData>[];
type ScoreRankingKeys = ArrayValues<typeof scoreRanking>;
type ScoreRanking = Record<ScoreRankingKeys, number>;
const identifierFields = new Set<ScoreRankingKeys>([
'label',
'shortcode',
'emoticons',
'shortcodes',
]);
interface BestRank {
categoryWeight: number;
score: number;
fieldWeight: number;
}
type ScoredEmoji = AnyEmojiData & { scores: ScoreRanking };
type RankedEmoji = AnyEmojiData & { rank: BestRank };
type ScoreMap = Map<string, ScoredEmoji>;
export async function search({
query: rawQuery,
locale: localeString,
limit = 0,
}: {
query: string;
locale: string;
limit?: number;
}) {
performance.mark('emoji-search-start');
// Get the locale, and extract tokens from the query.
const locale = toSupportedLocale(localeString);
const segmenter = localeToSegmenter(locale);
const query = rawQuery.toLowerCase();
const queryTokens = extractTokens(query, segmenter);
if (queryTokens.length === 0) {
log('no tokens extracted from query "%s"', query);
return [];
}
const lastToken = queryTokens.at(-1);
if (!lastToken) {
throw new Error('Missing tokens from query');
}
log('searching for tokens %o in locale %s', queryTokens, locale);
// Create an array of emoji results
const resultArrays: ScoreMap[] = [];
for (let i = 0; i < queryTokens.length; i++) {
const token = queryTokens[i];
if (!token) continue;
// Only query the range for the last token to allow partial matches.
const { unicodeResults, customResults, shortcodeResults } = await rawSearch(
token,
locale,
i === queryTokens.length - 1,
);
const resultMap: ScoreMap = new Map();
const checkedSet = new Set<string>();
// Score unicode results.
for (const emoji of unicodeResults) {
if (checkedSet.has(emoji.hexcode)) {
continue;
}
checkedSet.add(emoji.hexcode);
const scores = getScoreForEmoji(emoji, token);
if (scores) {
resultMap.set(emoji.hexcode, { ...emoji, scores });
}
}
// Score custom emojis.
for (const emoji of customResults) {
if (checkedSet.has(emoji.shortcode)) {
continue;
}
checkedSet.add(emoji.shortcode);
const scores = getScoreForEmoji(emoji, token);
if (scores) {
resultMap.set(emoji.shortcode, { ...emoji, scores });
}
}
// Score based on legacy shortcodes, using the higher score if there's a match.
for (const shortcodeResult of shortcodeResults) {
const emoji =
resultMap.get(shortcodeResult.hexcode) ??
(await loadEmojiByHexcode(shortcodeResult.hexcode, locale));
if (!emoji || !('hexcode' in emoji)) {
continue;
}
const newScores = getScoreForEmoji(
{
...emoji,
shortcodes: shortcodeResult.shortcodes,
},
token,
);
if (!newScores) {
continue;
}
const oldScores = resultMap.get(emoji.hexcode)?.scores;
const scores = oldScores
? combineEmojiScores(oldScores, newScores)
: newScores;
resultMap.set(emoji.hexcode, {
...emoji,
shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes],
scores,
});
}
log('found %d results for token "%s"', resultMap.size, token);
resultArrays.push(resultMap);
}
// Iterate over all maps, getting a combined score for emojis that exist in all results.
const allEmojiIds = resultArrays.reduce((prev, map) => {
if (prev.size === 0) {
return new Set(map.keys());
}
return new Set(map.keys()).intersection(prev);
}, new Set<string>());
const finalMap: ScoreMap = new Map();
for (const resultArray of resultArrays) {
for (const [id, emoji] of resultArray.entries()) {
if (!allEmojiIds.has(id)) {
continue;
}
const existingEmoji = finalMap.get(id);
if (!existingEmoji) {
finalMap.set(id, emoji);
} else {
finalMap.set(id, {
...existingEmoji,
scores: combineEmojiScores(existingEmoji.scores, emoji.scores),
});
}
}
}
const mixedResults = Array.from(finalMap.values());
// If there are no results, try a cursor-based custom emoji search instead.
if (mixedResults.length === 0 || mixedResults.length < limit) {
const customEmojisFound = await fullCustomSearch(query, allEmojiIds);
if (customEmojisFound.length > 0) {
log(
'cursor search found %d results for "%s"',
customEmojisFound.length,
query,
);
mixedResults.push(...customEmojisFound);
}
}
const rankedEmojis = mixedResults.map(({ scores, ...emoji }) => ({
...emoji,
rank: getBestRank(scores),
}));
const results: RankedEmoji[] = [];
const resultEmojis = new Set<string>();
for (const result of rankedEmojis.toSorted(compareRankedEmoji)) {
const id = getIdentifier(result);
if (resultEmojis.has(id)) {
continue;
}
results.push(result);
resultEmojis.add(id);
if (limit > 0 && results.length >= limit) {
break;
}
}
const time = performance.measure('emoji-search-end', 'emoji-search-start');
log(
'search for "%s" in locale %s returned %d results and took %dms',
query,
locale,
mixedResults.length,
time.duration,
);
return results;
}
function hasField(
emoji: AnyEmojiData,
field: string,
): field is keyof typeof emoji {
return Object.hasOwn(emoji, field);
}
function getIdentifier(emoji: AnyEmojiData) {
return 'shortcode' in emoji ? emoji.shortcode : emoji.hexcode;
}
// Creates ranked scores for a given emoji.
function getScoreForEmoji(emoji: AnyEmojiData, query: string) {
const scores = Object.fromEntries(
scoreRanking.map((field) => [field, -1]),
) as ScoreRanking;
let hasScore = false;
for (const field of scoreRanking) {
if (hasField(emoji, field)) {
const value = emoji[field] as string | string[] | undefined;
if (value === undefined) {
continue;
}
const tokens = Array.isArray(value) ? value : [value];
const score = getScoreForEmojiTokens(
tokens.map((token) => token.toLowerCase()),
query,
);
if (score >= 0) {
scores[field] = score;
hasScore = true;
}
}
}
if (!hasScore) {
return null;
}
return scores;
}
function combineEmojiScores(a: ScoreRanking, b: ScoreRanking): ScoreRanking {
const scores = Object.fromEntries(
scoreRanking.map((field) => [field, -1]),
) as ScoreRanking;
for (const rank of scoreRanking) {
if (a[rank] === -1) {
scores[rank] = b[rank];
} else if (b[rank] === -1) {
scores[rank] = a[rank];
} else {
scores[rank] = Math.min(a[rank], b[rank]);
}
}
return scores;
}
// Compares two scored emojis by getting the best rank.
function compareRankedEmoji(a: RankedEmoji, b: RankedEmoji): number {
const rankA = a.rank;
const rankB = b.rank;
// Identifier matches always outrank token matches, regardless of score.
if (rankA.categoryWeight !== rankB.categoryWeight) {
return rankA.categoryWeight - rankB.categoryWeight;
}
// Within the same category, compare the scores directly.
if (rankA.score !== rankB.score) {
return rankA.score - rankB.score;
}
// If equal, compare field weights.
if (rankA.fieldWeight !== rankB.fieldWeight) {
return rankA.fieldWeight - rankB.fieldWeight;
}
// Lastly prioritize Unicode emojis.
const aIsCustom = hasField(a, 'shortcode');
const bIsCustom = hasField(b, 'shortcode');
if (aIsCustom !== bIsCustom) {
return aIsCustom ? -1 : 1;
}
return 0;
}
// Extracts the best rank for a score ranking.
function getBestRank(scores: ScoreRanking): BestRank {
let best: BestRank | null = null;
// Use the index to determine field weight.
for (const [fieldWeight, field] of scoreRanking.entries()) {
const score = scores[field];
if (score < 0) {
continue;
}
// Also weight identifier fields over other fields.
const categoryWeight = identifierFields.has(field) ? 0 : 1;
if (
!best ||
categoryWeight < best.categoryWeight ||
(categoryWeight === best.categoryWeight &&
(score < best.score ||
(score === best.score && fieldWeight < best.fieldWeight)))
) {
best = { categoryWeight, score, fieldWeight };
}
}
return (
best ?? {
categoryWeight: 2,
score: Infinity,
fieldWeight: scoreRanking.length,
}
);
}
function getScoreForEmojiTokens(tokens: string[], query: string) {
let lowestScore = -1;
for (const token of tokens) {
let score = -1;
// Priority: exact match, prefix match, substring,
if (token === query) {
score = 0;
} else if (token.startsWith(query)) {
score = 1 + query.length / token.length;
} else if (token.includes(query)) {
score = 2 + query.length / token.length;
} // TODO: Add fuzzy search if needed
if (score >= 0 && (score < lowestScore || lowestScore < 0)) {
lowestScore = score;
}
}
return lowestScore;
}
async function fullCustomSearch(query: string, existing = new Set<string>()) {
const foundEmojis = new Set<string>();
// First iterate over chunks of 1,000 custom emoji keys and find any matches.
const chunkSize = 1_000;
let lastKey: string | null = null;
let keys: string[] = [];
do {
keys = await loadCustomEmojiKeys(lastKey, chunkSize);
if (keys.length === 0) {
break;
}
log('cursor search got batch of %d emojis', keys.length);
lastKey = keys.at(-1) ?? null;
for (const key of keys) {
if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) {
foundEmojis.add(key);
}
}
} while (keys.length === chunkSize);
// Next get the full emojis for all matches.
const emojis = await searchCustomEmojisByShortcodes(Array.from(foundEmojis));
const results: (CustomEmojiData & { scores: ScoreRanking })[] = [];
for (const emoji of emojis) {
const scores = getScoreForEmoji(emoji, query);
if (scores) {
results.push({
scores,
...emoji,
});
}
}
return results;
}

3
app/javascript/testing/factories.ts
View File

@ -126,11 +126,12 @@ export function unicodeEmojiFactory(
data: Partial<UnicodeEmojiData> = {},
): UnicodeEmojiData {
return {
emoticons: undefined,
hexcode: 'test',
label: 'Test',
unicode: '🧪',
shortcodes: ['test_emoji'],
tokens: ['emoji', 'test'],
tokens: ['test', 'emoji'],
group: 1,
order: 1,
...data,

1
package.json
View File

@ -121,6 +121,7 @@
"tesseract.js": "^7.0.0",
"tiny-queue": "^0.2.1",
"twitter-text": "3.1.0",
"type-fest": "^5.8.0",
"use-debounce": "^10.0.0",
"vite": "^8.0.0",
"vite-plugin-manifest-sri": "^0.2.0",

10
yarn.lock
View File

@ -3014,6 +3014,7 @@ __metadata:
tesseract.js: "npm:^7.0.0"
tiny-queue: "npm:^0.2.1"
twitter-text: "npm:3.1.0"
type-fest: "npm:^5.8.0"
typescript: "npm:~6.0.0"
typescript-eslint: "npm:^8.55.0"
typescript-plugin-css-modules: "npm:^5.2.0"
@ -14009,6 +14010,15 @@ __metadata:
languageName: node
linkType: hard
"type-fest@npm:^5.8.0":
version: 5.8.0
resolution: "type-fest@npm:5.8.0"
dependencies:
tagged-tag: "npm:^1.0.0"
checksum: 10c0/c8aae118a763d550a9552a511dff6b71840a23dab4edf693cf1c4df22596942794e6f6723389bd9036a90182249d915158bacf0815a1ae87f05f901b1d5f574e
languageName: node
linkType: hard
"type-is@npm:^2.0.1":
version: 2.0.1
resolution: "type-is@npm:2.0.1"