[Glitch] Emoji: Adds search
Port 628fc9b95b064f7feba7f54f3c4146210828f18d to glitch-soc Co-authored-by: diondiondion <mail@diondiondion.com> Signed-off-by: Claire <claire.github-309c@sitedethib.com>
This commit is contained in:
Echo
Claire
parent
c5fd0d47da
commit
124d6e37eb
@ -88,7 +88,10 @@ export const Emoji: FC<EmojiProps> = ({
|
||||
);
|
||||
}
|
||||
|
||||
const src = unicodeHexToUrl(state.code, appState.darkTheme);
|
||||
const src = unicodeHexToUrl({
|
||||
unicodeHex: state.code,
|
||||
...appState,
|
||||
});
|
||||
|
||||
return (
|
||||
<img
|
||||
|
||||
@ -15,6 +15,8 @@ export const SKIN_TONE_CODES = [
|
||||
0x1f3ff, // Dark skin tone
|
||||
] as const;
|
||||
|
||||
export const EMOJI_MIN_TOKEN_LENGTH = 2;
|
||||
|
||||
// Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected.
|
||||
export const EMOJI_MODE_NATIVE = 'native';
|
||||
export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags';
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import type { CompactEmoji } from 'emojibase';
|
||||
import { IDBFactory } from 'fake-indexeddb';
|
||||
|
||||
import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
|
||||
@ -6,8 +7,6 @@ import { EMOJI_DB_SHORTCODE_TEST } from './constants';
|
||||
import {
|
||||
putEmojiData,
|
||||
loadEmojiByHexcode,
|
||||
searchEmojisByHexcodes,
|
||||
searchEmojisByTag,
|
||||
testClear,
|
||||
testGet,
|
||||
putCustomEmojiData,
|
||||
@ -17,6 +16,14 @@ import {
|
||||
putLatestEtag,
|
||||
} from './database';
|
||||
|
||||
function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
|
||||
return {
|
||||
...unicodeEmojiFactory(),
|
||||
tags: ['test', 'emoji'],
|
||||
...data,
|
||||
};
|
||||
}
|
||||
|
||||
describe('emoji database', () => {
|
||||
afterEach(() => {
|
||||
testClear();
|
||||
@ -32,7 +39,7 @@ describe('emoji database', () => {
|
||||
});
|
||||
|
||||
test('loads emoji into indexedDB', async () => {
|
||||
await putEmojiData([unicodeEmojiFactory()], 'en');
|
||||
await putEmojiData([rawEmojiFactory()], 'en');
|
||||
const { db } = await testGet();
|
||||
await expect(db.get('en', 'test')).resolves.toEqual(
|
||||
unicodeEmojiFactory(),
|
||||
@ -60,7 +67,7 @@ describe('emoji database', () => {
|
||||
});
|
||||
await expect(db.get('custom', 'emoji1')).resolves.toBeUndefined();
|
||||
await expect(db.get('custom', 'emoji2')).resolves.toEqual(
|
||||
customEmojiFactory({ shortcode: 'emoji2' }),
|
||||
customEmojiFactory({ shortcode: 'emoji2', tokens: ['emoji2'] }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -79,12 +86,6 @@ describe('emoji database', () => {
|
||||
});
|
||||
|
||||
describe('loadEmojiByHexcode', () => {
|
||||
test('throws if the locale is not loaded', async () => {
|
||||
await expect(loadEmojiByHexcode('en', 'test')).rejects.toThrowError(
|
||||
'Locale en',
|
||||
);
|
||||
});
|
||||
|
||||
test('retrieves the emoji', async () => {
|
||||
await putEmojiData([unicodeEmojiFactory()], 'en');
|
||||
await expect(loadEmojiByHexcode('test', 'en')).resolves.toEqual(
|
||||
@ -98,90 +99,6 @@ describe('emoji database', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchEmojisByHexcodes', () => {
|
||||
const data = [
|
||||
unicodeEmojiFactory({ hexcode: 'not a number' }),
|
||||
unicodeEmojiFactory({ hexcode: '1' }),
|
||||
unicodeEmojiFactory({ hexcode: '2' }),
|
||||
unicodeEmojiFactory({ hexcode: '3' }),
|
||||
unicodeEmojiFactory({ hexcode: 'another not a number' }),
|
||||
];
|
||||
beforeEach(async () => {
|
||||
await putEmojiData(data, 'en');
|
||||
});
|
||||
test('finds emoji in consecutive range', async () => {
|
||||
const actual = await searchEmojisByHexcodes(['1', '2', '3'], 'en');
|
||||
expect(actual).toHaveLength(3);
|
||||
});
|
||||
|
||||
test('finds emoji in split range', async () => {
|
||||
const actual = await searchEmojisByHexcodes(['1', '3'], 'en');
|
||||
expect(actual).toHaveLength(2);
|
||||
expect(actual).toContainEqual(data.at(1));
|
||||
expect(actual).toContainEqual(data.at(3));
|
||||
});
|
||||
|
||||
test('finds emoji with non-numeric range', async () => {
|
||||
const actual = await searchEmojisByHexcodes(
|
||||
['3', 'not a number', '1'],
|
||||
'en',
|
||||
);
|
||||
expect(actual).toHaveLength(3);
|
||||
expect(actual).toContainEqual(data.at(0));
|
||||
expect(actual).toContainEqual(data.at(1));
|
||||
expect(actual).toContainEqual(data.at(3));
|
||||
});
|
||||
|
||||
test('not found emoji are not returned', async () => {
|
||||
const actual = await searchEmojisByHexcodes(['not found'], 'en');
|
||||
expect(actual).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('only found emojis are returned', async () => {
|
||||
const actual = await searchEmojisByHexcodes(
|
||||
['another not a number', 'not found'],
|
||||
'en',
|
||||
);
|
||||
expect(actual).toHaveLength(1);
|
||||
expect(actual).toContainEqual(data.at(4));
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchEmojisByTag', () => {
|
||||
const data = [
|
||||
unicodeEmojiFactory({ hexcode: 'test1', tags: ['test 1'] }),
|
||||
unicodeEmojiFactory({
|
||||
hexcode: 'test2',
|
||||
tags: ['test 2', 'something else'],
|
||||
}),
|
||||
unicodeEmojiFactory({ hexcode: 'test3', tags: ['completely different'] }),
|
||||
];
|
||||
beforeEach(async () => {
|
||||
await putEmojiData(data, 'en');
|
||||
});
|
||||
test('finds emojis with tag', async () => {
|
||||
const actual = await searchEmojisByTag('test 1', 'en');
|
||||
expect(actual).toHaveLength(1);
|
||||
expect(actual).toContainEqual(data.at(0));
|
||||
});
|
||||
|
||||
test('finds emojis starting with tag', async () => {
|
||||
const actual = await searchEmojisByTag('test', 'en');
|
||||
expect(actual).toHaveLength(2);
|
||||
expect(actual).not.toContainEqual(data.at(2));
|
||||
});
|
||||
|
||||
test('does not find emojis ending with tag', async () => {
|
||||
const actual = await searchEmojisByTag('else', 'en');
|
||||
expect(actual).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('finds nothing with invalid tag', async () => {
|
||||
const actual = await searchEmojisByTag('not found', 'en');
|
||||
expect(actual).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadLegacyShortcodesByShortcode', () => {
|
||||
const data = {
|
||||
hexcode: 'test_hexcode',
|
||||
|
||||
@ -1,55 +1,25 @@
|
||||
import { SUPPORTED_LOCALES } from 'emojibase';
|
||||
import type { Locale, ShortcodesDataset } from 'emojibase';
|
||||
import type { DBSchema, IDBPDatabase } from 'idb';
|
||||
import { openDB } from 'idb';
|
||||
import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
|
||||
|
||||
import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
|
||||
|
||||
import { EMOJI_DB_SHORTCODE_TEST } from './constants';
|
||||
import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
|
||||
import type { CustomEmojiData, UnicodeEmojiData, EtagTypes } from './types';
|
||||
import { openEmojiDB } from './db-schema';
|
||||
import type { Database } from './db-schema';
|
||||
import {
|
||||
localeToSegmenter,
|
||||
toSupportedLocale,
|
||||
toSupportedLocaleOrCustom,
|
||||
} from './locale';
|
||||
import {
|
||||
extractTokens,
|
||||
skinHexcodeToEmoji,
|
||||
transformCustomEmojiData,
|
||||
transformEmojiData,
|
||||
} from './normalize';
|
||||
import type { AnyEmojiData, EtagTypes } from './types';
|
||||
import { emojiLogger } from './utils';
|
||||
|
||||
interface EmojiDB extends LocaleTables, DBSchema {
|
||||
custom: {
|
||||
key: string;
|
||||
value: CustomEmojiData;
|
||||
indexes: {
|
||||
category: string;
|
||||
};
|
||||
};
|
||||
shortcodes: {
|
||||
key: string;
|
||||
value: {
|
||||
hexcode: string;
|
||||
shortcodes: string[];
|
||||
};
|
||||
indexes: {
|
||||
hexcode: string;
|
||||
shortcodes: string[];
|
||||
};
|
||||
};
|
||||
etags: {
|
||||
key: EtagTypes;
|
||||
value: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface LocaleTable {
|
||||
key: string;
|
||||
value: UnicodeEmojiData;
|
||||
indexes: {
|
||||
group: number;
|
||||
label: string;
|
||||
order: number;
|
||||
tags: string[];
|
||||
shortcodes: string[];
|
||||
};
|
||||
}
|
||||
type LocaleTables = Record<Locale, LocaleTable>;
|
||||
|
||||
type Database = IDBPDatabase<EmojiDB>;
|
||||
|
||||
const SCHEMA_VERSION = 2;
|
||||
|
||||
const loadedLocales = new Set<Locale>();
|
||||
|
||||
const log = emojiLogger('database');
|
||||
@ -60,75 +30,7 @@ const loadDB = (() => {
|
||||
|
||||
// Actually load the DB.
|
||||
async function initDB() {
|
||||
const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
|
||||
upgrade(database, oldVersion, newVersion, trx) {
|
||||
if (!database.objectStoreNames.contains('custom')) {
|
||||
const customTable = database.createObjectStore('custom', {
|
||||
keyPath: 'shortcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
customTable.createIndex('category', 'category');
|
||||
}
|
||||
|
||||
if (!database.objectStoreNames.contains('etags')) {
|
||||
database.createObjectStore('etags');
|
||||
}
|
||||
|
||||
for (const locale of SUPPORTED_LOCALES) {
|
||||
if (!database.objectStoreNames.contains(locale)) {
|
||||
const localeTable = database.createObjectStore(locale, {
|
||||
keyPath: 'hexcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
localeTable.createIndex('group', 'group');
|
||||
localeTable.createIndex('label', 'label');
|
||||
localeTable.createIndex('order', 'order');
|
||||
localeTable.createIndex('tags', 'tags', { multiEntry: true });
|
||||
localeTable.createIndex('shortcodes', 'shortcodes', {
|
||||
multiEntry: true,
|
||||
});
|
||||
}
|
||||
// Added in version 2.
|
||||
const localeTable = trx.objectStore(locale);
|
||||
if (!localeTable.indexNames.contains('shortcodes')) {
|
||||
localeTable.createIndex('shortcodes', 'shortcodes', {
|
||||
multiEntry: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!database.objectStoreNames.contains('shortcodes')) {
|
||||
const shortcodeTable = database.createObjectStore('shortcodes', {
|
||||
keyPath: 'hexcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
shortcodeTable.createIndex('hexcode', 'hexcode');
|
||||
shortcodeTable.createIndex('shortcodes', 'shortcodes', {
|
||||
multiEntry: true,
|
||||
});
|
||||
}
|
||||
|
||||
log(
|
||||
'Upgraded emoji database from version %d to %d',
|
||||
oldVersion,
|
||||
newVersion,
|
||||
);
|
||||
},
|
||||
blocked(currentVersion, blockedVersion) {
|
||||
log(
|
||||
'Emoji database upgrade from version %d to %d is blocked',
|
||||
currentVersion,
|
||||
blockedVersion,
|
||||
);
|
||||
},
|
||||
blocking(currentVersion, blockedVersion) {
|
||||
log(
|
||||
'Emoji database upgrade from version %d is blocking upgrade to %d',
|
||||
currentVersion,
|
||||
blockedVersion,
|
||||
);
|
||||
},
|
||||
});
|
||||
const db = await openEmojiDB();
|
||||
await syncLocales(db);
|
||||
log('Loaded database version %d', db.version);
|
||||
return db;
|
||||
@ -149,11 +51,128 @@ const loadDB = (() => {
|
||||
return loadPromise;
|
||||
})();
|
||||
|
||||
export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) {
|
||||
export async function search({
|
||||
query,
|
||||
locale: localeString,
|
||||
limit = 0,
|
||||
}: {
|
||||
query: string;
|
||||
locale: string;
|
||||
limit?: number;
|
||||
}) {
|
||||
performance.mark('emoji-search-start');
|
||||
|
||||
// Get the locale, and extract tokens from the query.
|
||||
const locale = await toLoadedLocale(localeString);
|
||||
const segmenter = localeToSegmenter(locale);
|
||||
const queryTokens = extractTokens(query, segmenter);
|
||||
|
||||
if (queryTokens.length === 0) {
|
||||
log('no tokens extracted from query "%s"', query);
|
||||
return [];
|
||||
}
|
||||
const lastToken = queryTokens.at(-1);
|
||||
if (!lastToken) {
|
||||
throw new Error('Missing tokens from query');
|
||||
}
|
||||
|
||||
log('searching for tokens %o in locale %s', queryTokens, locale);
|
||||
|
||||
// Create an array of emoji results
|
||||
const db = await loadDB();
|
||||
const resultArrays: Map<string, AnyEmojiData>[] = [];
|
||||
for (let i = 0; i < queryTokens.length; i++) {
|
||||
const token = queryTokens[i];
|
||||
if (!token) continue;
|
||||
|
||||
// Only query the range for the last token to allow partial matches.
|
||||
const range =
|
||||
i === queryTokens.length - 1
|
||||
? IDBKeyRange.bound(token, token + '\uffff')
|
||||
: IDBKeyRange.only(token);
|
||||
|
||||
const [unicodeResults, customResults] = await Promise.all([
|
||||
db.getAllFromIndex(locale, 'tokens', range),
|
||||
db.getAllFromIndex('custom', 'tokens', range),
|
||||
]);
|
||||
const resultMap = new Map<string, AnyEmojiData>([
|
||||
...unicodeResults.map(
|
||||
(emoji) => [emoji.hexcode, emoji] as [string, AnyEmojiData],
|
||||
),
|
||||
...customResults.map(
|
||||
(emoji) => [emoji.shortcode, emoji] as [string, AnyEmojiData],
|
||||
),
|
||||
]);
|
||||
log('found %d results for token "%s"', resultMap.size, token);
|
||||
resultArrays.push(resultMap);
|
||||
}
|
||||
|
||||
// Utilize maps to find the intersection of all result sets.
|
||||
const results = Array.from(
|
||||
resultArrays
|
||||
.reduce((prev, curr) => {
|
||||
const intersection = new Map<string, AnyEmojiData>();
|
||||
for (const [code, emoji] of prev) {
|
||||
if (curr.has(code)) {
|
||||
intersection.set(code, emoji);
|
||||
}
|
||||
}
|
||||
return intersection;
|
||||
})
|
||||
.values(),
|
||||
);
|
||||
|
||||
results.sort((a, b) => {
|
||||
// Checks if a or b has the last token exactly, or only a prefix.
|
||||
const aHasToken = a.tokens.includes(lastToken);
|
||||
const bHasToken = b.tokens.includes(lastToken);
|
||||
if (aHasToken && !bHasToken) {
|
||||
return -1;
|
||||
} else if (!aHasToken && bHasToken) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// If one is a custom emoji, prioritize it over Unicode emojis.
|
||||
if ('category' in a) {
|
||||
return -1;
|
||||
} else if ('category' in b) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// If both are Unicode emojis, prioritize by order.
|
||||
if ('order' in a && 'order' in b) {
|
||||
return (a.order ?? 0) - (b.order ?? 0); // If these are both Unicode emojis, sort by order.
|
||||
}
|
||||
|
||||
// ¯\_(ツ)_/¯
|
||||
return 0;
|
||||
});
|
||||
|
||||
const time = performance.measure('emoji-search-end', 'emoji-search-start');
|
||||
log(
|
||||
'search for "%s" in locale %s returned %d results and took %dms',
|
||||
query,
|
||||
locale,
|
||||
results.length,
|
||||
time.duration,
|
||||
);
|
||||
if (limit > 0) {
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) {
|
||||
loadedLocales.add(locale);
|
||||
const db = await loadDB();
|
||||
const trx = db.transaction(locale, 'readwrite');
|
||||
await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
|
||||
await trx.store.clear();
|
||||
const segmenter = localeToSegmenter(locale);
|
||||
await Promise.all(
|
||||
emojis
|
||||
.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
|
||||
.map((emoji) => trx.store.put(transformEmojiData(emoji, segmenter))),
|
||||
);
|
||||
await trx.done;
|
||||
}
|
||||
|
||||
@ -161,7 +180,7 @@ export async function putCustomEmojiData({
|
||||
emojis,
|
||||
clear = false,
|
||||
}: {
|
||||
emojis: CustomEmojiData[];
|
||||
emojis: ApiCustomEmojiJSON[];
|
||||
clear?: boolean;
|
||||
}) {
|
||||
const db = await loadDB();
|
||||
@ -173,7 +192,9 @@ export async function putCustomEmojiData({
|
||||
log('Cleared existing custom emojis in database');
|
||||
}
|
||||
|
||||
await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
|
||||
await Promise.all(
|
||||
emojis.map((emoji) => trx.store.put(transformCustomEmojiData(emoji))),
|
||||
);
|
||||
await trx.done;
|
||||
|
||||
log('Imported %d custom emojis into database', emojis.length);
|
||||
@ -210,32 +231,25 @@ export async function loadEmojiByHexcode(
|
||||
localeString: string,
|
||||
) {
|
||||
const db = await loadDB();
|
||||
const locale = toLoadedLocale(localeString);
|
||||
return db.get(locale, hexcode);
|
||||
}
|
||||
const locale = await toLoadedLocale(localeString);
|
||||
const result = await db.get(locale, hexcode);
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function searchEmojisByHexcodes(
|
||||
hexcodes: string[],
|
||||
localeString: string,
|
||||
) {
|
||||
const db = await loadDB();
|
||||
const locale = toLoadedLocale(localeString);
|
||||
const sortedCodes = hexcodes.toSorted();
|
||||
const results = await db.getAll(
|
||||
// If the emoji wasn't found, check if it's a skin tone variant.
|
||||
const skinResult = await db.getFromIndex(
|
||||
locale,
|
||||
IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)),
|
||||
'skinHexcodes',
|
||||
IDBKeyRange.only(hexcode),
|
||||
);
|
||||
return results.filter((emoji) => hexcodes.includes(emoji.hexcode));
|
||||
}
|
||||
|
||||
export async function searchEmojisByTag(tag: string, localeString: string) {
|
||||
const db = await loadDB();
|
||||
const locale = toLoadedLocale(localeString);
|
||||
const range = IDBKeyRange.bound(
|
||||
tag.toLowerCase(),
|
||||
`${tag.toLowerCase()}\uffff`,
|
||||
);
|
||||
return db.getAllFromIndex(locale, 'tags', range);
|
||||
if (!skinResult) {
|
||||
return skinResult;
|
||||
}
|
||||
|
||||
// Reconstruct the full unicode string from the skin tone hexcode.
|
||||
return skinHexcodeToEmoji(hexcode, skinResult);
|
||||
}
|
||||
|
||||
export async function loadCustomEmojiByShortcode(shortcode: string) {
|
||||
@ -301,13 +315,16 @@ async function syncLocales(db: Database) {
|
||||
log('Loaded %d locales: %o', loadedLocales.size, loadedLocales);
|
||||
}
|
||||
|
||||
function toLoadedLocale(localeString: string) {
|
||||
async function toLoadedLocale(localeString: string) {
|
||||
const locale = toSupportedLocale(localeString);
|
||||
if (localeString !== locale) {
|
||||
log(`Locale ${locale} is different from provided ${localeString}`);
|
||||
}
|
||||
if (!loadedLocales.has(locale)) {
|
||||
throw new LocaleNotLoadedError(locale);
|
||||
log('Locale %s not loaded, importing...', locale);
|
||||
const { importEmojiData } = await import('./loader');
|
||||
await importEmojiData(locale);
|
||||
return locale;
|
||||
}
|
||||
return locale;
|
||||
}
|
||||
|
||||
198
app/javascript/flavours/glitch/features/emoji/db-schema.ts
Normal file
198
app/javascript/flavours/glitch/features/emoji/db-schema.ts
Normal file
@ -0,0 +1,198 @@
|
||||
import { SUPPORTED_LOCALES } from 'emojibase';
|
||||
import type { Locale } from 'emojibase';
|
||||
import { openDB } from 'idb';
|
||||
import type {
|
||||
DBSchema,
|
||||
IDBPDatabase,
|
||||
IDBPObjectStore,
|
||||
IDBPTransaction,
|
||||
IndexNames,
|
||||
StoreNames,
|
||||
} from 'idb';
|
||||
|
||||
import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types';
|
||||
import { emojiLogger } from './utils';
|
||||
|
||||
const log = emojiLogger('database');
|
||||
|
||||
interface EmojiDB extends LocaleTables, DBSchema {
|
||||
custom: {
|
||||
key: string;
|
||||
value: CustomEmojiData;
|
||||
indexes: {
|
||||
tokens: string[];
|
||||
category: string;
|
||||
};
|
||||
};
|
||||
shortcodes: {
|
||||
key: string;
|
||||
value: {
|
||||
hexcode: string;
|
||||
shortcodes: string[];
|
||||
};
|
||||
indexes: {
|
||||
shortcodes: string[];
|
||||
};
|
||||
};
|
||||
etags: {
|
||||
key: EtagTypes;
|
||||
value: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface LocaleTable {
|
||||
key: string;
|
||||
value: UnicodeEmojiData;
|
||||
indexes: {
|
||||
shortcodes: string[];
|
||||
groupOrder: [number, number];
|
||||
tokens: string[];
|
||||
skinHexcodes: string[];
|
||||
};
|
||||
}
|
||||
type LocaleTables = Record<Locale, LocaleTable>;
|
||||
|
||||
type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
|
||||
IDBPTransaction<EmojiDB, StoreNames<EmojiDB>[], Mode>;
|
||||
|
||||
export type Database = IDBPDatabase<EmojiDB>;
|
||||
|
||||
const SCHEMA_VERSION = 3;
|
||||
|
||||
export async function openEmojiDB() {
|
||||
const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
|
||||
upgrade(database, oldVersion, newVersion, trx) {
|
||||
if (!database.objectStoreNames.contains('custom')) {
|
||||
database.createObjectStore('custom', {
|
||||
keyPath: 'shortcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
}
|
||||
maybeAddIndex({ trx, storeName: 'custom', indexName: 'category' });
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: 'custom',
|
||||
indexName: 'tokens',
|
||||
options: { multiEntry: true },
|
||||
});
|
||||
|
||||
if (!database.objectStoreNames.contains('etags')) {
|
||||
database.createObjectStore('etags');
|
||||
}
|
||||
|
||||
SUPPORTED_LOCALES.forEach((locale) => {
|
||||
createLocaleTable(locale, database, trx);
|
||||
});
|
||||
|
||||
const shortcodeTable = database.objectStoreNames.contains('shortcodes')
|
||||
? trx.objectStore('shortcodes')
|
||||
: database.createObjectStore('shortcodes', {
|
||||
keyPath: 'hexcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: 'shortcodes',
|
||||
indexName: 'shortcodes',
|
||||
options: { multiEntry: true },
|
||||
});
|
||||
deleteOldIndexes(shortcodeTable, ['hexcode']);
|
||||
|
||||
log(
|
||||
'Upgraded emoji database from version %d to %d',
|
||||
oldVersion,
|
||||
newVersion,
|
||||
);
|
||||
},
|
||||
blocked(currentVersion, blockedVersion) {
|
||||
log(
|
||||
'Emoji database upgrade from version %d to %d is blocked',
|
||||
currentVersion,
|
||||
blockedVersion,
|
||||
);
|
||||
},
|
||||
blocking(currentVersion, blockedVersion) {
|
||||
log(
|
||||
'Emoji database upgrade from version %d is blocking upgrade to %d',
|
||||
currentVersion,
|
||||
blockedVersion,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
function maybeAddIndex<StoreName extends StoreNames<EmojiDB>>({
|
||||
trx,
|
||||
storeName,
|
||||
indexName,
|
||||
keys,
|
||||
options,
|
||||
}: {
|
||||
trx: Transaction;
|
||||
storeName: StoreName;
|
||||
indexName: IndexNames<EmojiDB, StoreName>;
|
||||
keys?: string | string[];
|
||||
options?: IDBIndexParameters;
|
||||
}) {
|
||||
const store = trx.objectStore(storeName);
|
||||
if (!store.indexNames.contains(indexName)) {
|
||||
store.createIndex(indexName, keys ?? indexName, options);
|
||||
}
|
||||
}
|
||||
|
||||
function createLocaleTable(
|
||||
locale: Locale,
|
||||
database: Database,
|
||||
trx: Transaction,
|
||||
) {
|
||||
if (!database.objectStoreNames.contains(locale)) {
|
||||
database.createObjectStore(locale, {
|
||||
keyPath: 'hexcode',
|
||||
autoIncrement: false,
|
||||
});
|
||||
}
|
||||
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: locale,
|
||||
indexName: 'shortcodes',
|
||||
options: { multiEntry: true },
|
||||
});
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: locale,
|
||||
indexName: 'groupOrder',
|
||||
keys: ['group', 'order'],
|
||||
});
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: locale,
|
||||
indexName: 'tokens',
|
||||
keys: 'tokens',
|
||||
options: { multiEntry: true },
|
||||
});
|
||||
maybeAddIndex({
|
||||
trx,
|
||||
storeName: locale,
|
||||
indexName: 'skinHexcodes',
|
||||
keys: 'skinHexcodes',
|
||||
options: { multiEntry: true },
|
||||
});
|
||||
|
||||
const oldIndexes = ['group', 'order', 'tag', 'label'] as const;
|
||||
deleteOldIndexes(trx.objectStore(locale), oldIndexes);
|
||||
}
|
||||
|
||||
function deleteOldIndexes(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Type is too complex, so only any works here.
|
||||
table: IDBPObjectStore<any, any, any, 'versionchange'>,
|
||||
indexes: readonly string[],
|
||||
) {
|
||||
for (const index of indexes) {
|
||||
if (table.indexNames.contains(index)) {
|
||||
table.deleteIndex(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,10 +1,5 @@
|
||||
import { flattenEmojiData } from 'emojibase';
|
||||
import type {
|
||||
CompactEmoji,
|
||||
FlatCompactEmoji,
|
||||
Locale,
|
||||
ShortcodesDataset,
|
||||
} from 'emojibase';
|
||||
import { joinShortcodes } from 'emojibase';
|
||||
import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
|
||||
|
||||
import {
|
||||
putEmojiData,
|
||||
@ -28,7 +23,7 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
|
||||
shortcodes ? ' and shortcodes' : '',
|
||||
);
|
||||
|
||||
const emojis = await fetchAndCheckEtag<CompactEmoji[]>({
|
||||
let emojis = await fetchAndCheckEtag<CompactEmoji[]>({
|
||||
etagString: locale,
|
||||
path: localeToEmojiPath(locale),
|
||||
});
|
||||
@ -49,12 +44,10 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
|
||||
}
|
||||
}
|
||||
|
||||
const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(
|
||||
emojis,
|
||||
shortcodesData,
|
||||
);
|
||||
await putEmojiData(flattenedEmojis, locale);
|
||||
return flattenedEmojis;
|
||||
emojis = joinShortcodes(emojis, shortcodesData);
|
||||
|
||||
await putEmojiData(emojis, locale);
|
||||
return emojis;
|
||||
}
|
||||
|
||||
export async function importCustomEmojiData() {
|
||||
@ -135,11 +128,11 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
|
||||
checkEtag?: boolean;
|
||||
}): Promise<ResultType | null> {
|
||||
const etagName = toValidEtagName(etagString);
|
||||
const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
|
||||
|
||||
// Use location.origin as this script may be loaded from a CDN domain.
|
||||
const url = new URL(path, location.origin);
|
||||
|
||||
const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
@ -148,6 +141,7 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
|
||||
});
|
||||
// If not modified, return null
|
||||
if (response.status === 304) {
|
||||
log('etag not modified for %s', etagName);
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
@ -163,6 +157,8 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
|
||||
if (etag && checkEtag) {
|
||||
log(`storing new etag for ${etagName}: ${etag}`);
|
||||
await putLatestEtag(etag, etagName);
|
||||
} else if (!etag) {
|
||||
log(`no etag found in response for ${etagName}`);
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
@ -32,6 +32,13 @@ export function toValidEtagName(input: string): EtagTypes {
|
||||
return toSupportedLocale(lower);
|
||||
}
|
||||
|
||||
export function localeToSegmenter(locale: Locale): Intl.Segmenter | null {
|
||||
if (typeof Intl.Segmenter === 'function') {
|
||||
return new Intl.Segmenter(locale, { granularity: 'word' });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isSupportedLocale(locale: string): locale is Locale {
|
||||
return SUPPORTED_LOCALES.includes(locale as Locale);
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js
|
||||
|
||||
import { createAppSelector, useAppSelector } from '@/flavours/glitch/store';
|
||||
import { assetHost } from '@/flavours/glitch/utils/config';
|
||||
import { isDevelopment } from '@/flavours/glitch/utils/environment';
|
||||
import { isDarkMode } from '@/flavours/glitch/utils/theme';
|
||||
|
||||
@ -29,6 +30,7 @@ export function useEmojiAppState(): EmojiAppState {
|
||||
locales: [locale],
|
||||
mode,
|
||||
darkTheme: isDarkMode(),
|
||||
assetHost,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -4,11 +4,7 @@ import { basename, resolve } from 'path';
|
||||
import { flattenEmojiData } from 'emojibase';
|
||||
import unicodeRawEmojis from 'emojibase-data/en/data.json';
|
||||
|
||||
import {
|
||||
twemojiToUnicodeInfo,
|
||||
unicodeToTwemojiHex,
|
||||
emojiToUnicodeHex,
|
||||
} from './normalize';
|
||||
import { unicodeToTwemojiHex } from './normalize';
|
||||
|
||||
const emojiSVGFiles = await readdir(
|
||||
// This assumes tests are run from project root
|
||||
@ -26,23 +22,6 @@ const svgFileNamesWithoutBorder = svgFileNames.filter(
|
||||
|
||||
const unicodeEmojis = flattenEmojiData(unicodeRawEmojis);
|
||||
|
||||
describe('emojiToUnicodeHex', () => {
|
||||
test.concurrent.for([
|
||||
['🎱', '1F3B1'],
|
||||
['🐜', '1F41C'],
|
||||
['⚫', '26AB'],
|
||||
['🖤', '1F5A4'],
|
||||
['💀', '1F480'],
|
||||
['❤️', '2764'], // Checks for trailing variation selector removal.
|
||||
['💂♂️', '1F482-200D-2642-FE0F'],
|
||||
] as const)(
|
||||
'emojiToUnicodeHex converts %s to %s',
|
||||
([emoji, hexcode], { expect }) => {
|
||||
expect(emojiToUnicodeHex(emoji)).toBe(hexcode);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('unicodeToTwemojiHex', () => {
|
||||
test.concurrent.for(
|
||||
unicodeEmojis
|
||||
@ -54,26 +33,3 @@ describe('unicodeToTwemojiHex', () => {
|
||||
expect(svgFileNamesWithoutBorder).toContain(result);
|
||||
});
|
||||
});
|
||||
|
||||
describe('twemojiToUnicodeInfo', () => {
|
||||
const unicodeCodeSet = new Set(unicodeEmojis.map((emoji) => emoji.hexcode));
|
||||
|
||||
test.concurrent.for(svgFileNamesWithoutBorder)(
|
||||
'verifying SVG file %s maps to Unicode emoji',
|
||||
(svgFileName, { expect }) => {
|
||||
assert(!!svgFileName);
|
||||
const result = twemojiToUnicodeInfo(svgFileName);
|
||||
const hexcode = typeof result === 'string' ? result : result.unqualified;
|
||||
if (!hexcode) {
|
||||
// No hexcode means this is a special case like the Shibuya 109 emoji
|
||||
expect(result).toHaveProperty('label');
|
||||
return;
|
||||
}
|
||||
assert(!!hexcode);
|
||||
expect(
|
||||
unicodeCodeSet.has(hexcode),
|
||||
`${hexcode} (${svgFileName}) not found`,
|
||||
).toBeTruthy();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@ -1,48 +1,124 @@
|
||||
import { isList } from 'immutable';
|
||||
|
||||
import { assetHost } from '@/flavours/glitch/utils/config';
|
||||
import type { CompactEmoji, SkinTone } from 'emojibase';
|
||||
import { fromHexcodeToCodepoint } from 'emojibase';
|
||||
|
||||
import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
|
||||
|
||||
import {
|
||||
VARIATION_SELECTOR_CODE,
|
||||
KEYCAP_CODE,
|
||||
GENDER_FEMALE_CODE,
|
||||
GENDER_MALE_CODE,
|
||||
SKIN_TONE_CODES,
|
||||
EMOJIS_WITH_DARK_BORDER,
|
||||
EMOJIS_WITH_LIGHT_BORDER,
|
||||
EMOJIS_REQUIRING_INVERSION_IN_LIGHT_MODE,
|
||||
EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE,
|
||||
EMOJI_MIN_TOKEN_LENGTH,
|
||||
} from './constants';
|
||||
import type { CustomEmojiMapArg, ExtraCustomEmojiMap } from './types';
|
||||
import type {
|
||||
CustomEmojiData,
|
||||
CustomEmojiMapArg,
|
||||
ExtraCustomEmojiMap,
|
||||
UnicodeEmojiData,
|
||||
} from './types';
|
||||
import { emojiToUnicodeHex } from './utils';
|
||||
|
||||
// Misc codes that have special handling
|
||||
const SKIER_CODE = 0x26f7;
|
||||
const CHRISTMAS_TREE_CODE = 0x1f384;
|
||||
const MR_CLAUS_CODE = 0x1f385;
|
||||
const EYE_CODE = 0x1f441;
|
||||
const LEVITATING_PERSON_CODE = 0x1f574;
|
||||
const SPEECH_BUBBLE_CODE = 0x1f5e8;
|
||||
const MS_CLAUS_CODE = 0x1f936;
|
||||
const SKIN_TONE_MAP: Record<number, SkinTone> = {
|
||||
0x1f3fb: 1, // Light skin tone
|
||||
0x1f3fc: 2, // Medium-light skin tone
|
||||
0x1f3fd: 3, // Medium skin tone
|
||||
0x1f3fe: 4, // Medium-dark skin tone
|
||||
0x1f3ff: 5, // Dark skin tone
|
||||
};
|
||||
|
||||
export function emojiToUnicodeHex(emoji: string): string {
|
||||
const codes: number[] = [];
|
||||
for (const char of emoji) {
|
||||
const code = char.codePointAt(0);
|
||||
if (code !== undefined) {
|
||||
codes.push(code);
|
||||
export function transformEmojiData(
|
||||
emoji: CompactEmoji,
|
||||
segmenter: Intl.Segmenter | null,
|
||||
): UnicodeEmojiData {
|
||||
const {
|
||||
shortcodes = [],
|
||||
tags = [],
|
||||
label,
|
||||
emoticon,
|
||||
hexcode,
|
||||
unicode,
|
||||
group,
|
||||
order,
|
||||
skins = [],
|
||||
} = emoji;
|
||||
const extract = (str: string) => extractTokens(str, segmenter);
|
||||
|
||||
let normalizedEmoticons: string[] | undefined = undefined;
|
||||
if (emoticon) {
|
||||
normalizedEmoticons = Array.isArray(emoticon) ? emoticon : [emoticon];
|
||||
}
|
||||
|
||||
const tokens = [
|
||||
...new Set([
|
||||
...shortcodes.map(extract).flat(),
|
||||
...tags.map(extract).flat(),
|
||||
...extract(label),
|
||||
...(normalizedEmoticons ?? []),
|
||||
]),
|
||||
].sort((a, b) => a.localeCompare(b));
|
||||
|
||||
const res: UnicodeEmojiData = {
|
||||
tokens,
|
||||
shortcodes,
|
||||
label,
|
||||
emoticons: normalizedEmoticons,
|
||||
hexcode,
|
||||
unicode,
|
||||
group,
|
||||
order,
|
||||
};
|
||||
|
||||
for (const skin of skins) {
|
||||
res.skinHexcodes ??= [];
|
||||
res.skinHexcodes.push(skin.hexcode);
|
||||
|
||||
res.skinTones ??= [];
|
||||
for (const codePoint of skin.unicode) {
|
||||
const tone = SKIN_TONE_MAP[codePoint.codePointAt(0) ?? 0];
|
||||
if (tone) {
|
||||
res.skinTones.push(tone);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handles how Emojibase removes the variation selector for single code emojis.
|
||||
// See: https://emojibase.dev/docs/spec/#merged-variation-selectors
|
||||
if (codes.at(1) === VARIATION_SELECTOR_CODE && codes.length === 2) {
|
||||
codes.pop();
|
||||
}
|
||||
return hexNumbersToString(codes);
|
||||
return res;
|
||||
}
|
||||
|
||||
export function transformCustomEmojiData(
|
||||
emoji: ApiCustomEmojiJSON,
|
||||
): CustomEmojiData {
|
||||
const tokens = emoji.shortcode
|
||||
.split('_')
|
||||
.filter((word) => word.length >= EMOJI_MIN_TOKEN_LENGTH)
|
||||
.map((word) => word.toLowerCase());
|
||||
return {
|
||||
...emoji,
|
||||
tokens,
|
||||
};
|
||||
}
|
||||
|
||||
export function skinHexcodeToEmoji(
|
||||
skinHexcode: string,
|
||||
emoji: UnicodeEmojiData,
|
||||
): UnicodeEmojiData {
|
||||
return {
|
||||
...emoji,
|
||||
unicode: String.fromCodePoint(...fromHexcodeToCodepoint(skinHexcode)),
|
||||
hexcode: skinHexcode,
|
||||
};
|
||||
}
|
||||
|
||||
// Misc codes that have special handling
|
||||
const EYE_CODE = 0x1f441;
|
||||
const SPEECH_BUBBLE_CODE = 0x1f5e8;
|
||||
|
||||
export function unicodeToTwemojiHex(unicodeHex: string): string {
|
||||
const codes = hexStringToNumbers(unicodeHex);
|
||||
const codes = fromHexcodeToCodepoint(unicodeHex);
|
||||
const normalizedCodes: number[] = [];
|
||||
for (let i = 0; i < codes.length; i++) {
|
||||
const code = codes[i];
|
||||
@ -64,19 +140,28 @@ export function unicodeToTwemojiHex(unicodeHex: string): string {
|
||||
normalizedCodes.push(code);
|
||||
}
|
||||
|
||||
return hexNumbersToString(normalizedCodes, 0).toLowerCase();
|
||||
return normalizedCodes
|
||||
.map((code) => code.toString(16))
|
||||
.join('-')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export const CODES_WITH_DARK_BORDER =
|
||||
EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex);
|
||||
const CODES_WITH_DARK_BORDER = EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex);
|
||||
|
||||
export const CODES_WITH_LIGHT_BORDER =
|
||||
EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex);
|
||||
const CODES_WITH_LIGHT_BORDER = EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex);
|
||||
|
||||
export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string {
|
||||
export function unicodeHexToUrl({
|
||||
unicodeHex,
|
||||
darkTheme,
|
||||
assetHost,
|
||||
}: {
|
||||
unicodeHex: string;
|
||||
darkTheme: boolean;
|
||||
assetHost: string;
|
||||
}): string {
|
||||
const normalizedHex = unicodeToTwemojiHex(unicodeHex);
|
||||
let url = `${assetHost}/emoji/${normalizedHex}`;
|
||||
if (darkMode && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) {
|
||||
if (darkTheme && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) {
|
||||
url += '_border';
|
||||
}
|
||||
if (CODES_WITH_DARK_BORDER.includes(normalizedHex)) {
|
||||
@ -86,78 +171,6 @@ export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string {
|
||||
return url;
|
||||
}
|
||||
|
||||
interface TwemojiSpecificEmoji {
|
||||
unqualified?: string;
|
||||
gender?: number;
|
||||
skin?: number;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
// Normalize man/woman to male/female
|
||||
const GENDER_CODES_MAP: Record<number, number> = {
|
||||
[GENDER_FEMALE_CODE]: GENDER_FEMALE_CODE,
|
||||
[GENDER_MALE_CODE]: GENDER_MALE_CODE,
|
||||
// These are man/woman markers, but are used for gender sometimes.
|
||||
[0x1f468]: GENDER_MALE_CODE,
|
||||
[0x1f469]: GENDER_FEMALE_CODE,
|
||||
};
|
||||
|
||||
const TWEMOJI_SPECIAL_CASES: Record<string, string | TwemojiSpecificEmoji> = {
|
||||
'1F441-200D-1F5E8': '1F441-FE0F-200D-1F5E8-FE0F', // Eye in speech bubble
|
||||
// An emoji that was never ported to the Unicode standard.
|
||||
// See: https://emojipedia.org/shibuya
|
||||
E50A: { label: 'Shibuya 109' },
|
||||
};
|
||||
|
||||
export function twemojiToUnicodeInfo(
|
||||
twemojiHex: string,
|
||||
): TwemojiSpecificEmoji | string {
|
||||
const specialCase = TWEMOJI_SPECIAL_CASES[twemojiHex.toUpperCase()];
|
||||
if (specialCase) {
|
||||
return specialCase;
|
||||
}
|
||||
const codes = hexStringToNumbers(twemojiHex);
|
||||
let gender: undefined | number;
|
||||
let skin: undefined | number;
|
||||
for (const code of codes) {
|
||||
if (!gender && code in GENDER_CODES_MAP) {
|
||||
gender = GENDER_CODES_MAP[code];
|
||||
} else if (!skin && code in SKIN_TONE_CODES) {
|
||||
skin = code;
|
||||
}
|
||||
|
||||
// Exit if we have both skin and gender
|
||||
if (skin && gender) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mappedCodes: unknown[] = codes;
|
||||
|
||||
if (codes.at(-1) === CHRISTMAS_TREE_CODE && codes.length >= 3 && gender) {
|
||||
// Twemoji uses the christmas tree with a ZWJ for Mr. and Mrs. Claus,
|
||||
// but in Unicode that only works for Mx. Claus.
|
||||
const START_CODE =
|
||||
gender === GENDER_FEMALE_CODE ? MS_CLAUS_CODE : MR_CLAUS_CODE;
|
||||
mappedCodes = [START_CODE, skin];
|
||||
} else if (codes.at(-1) === KEYCAP_CODE && codes.length === 2) {
|
||||
// For key emoji, insert the variation selector
|
||||
mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE];
|
||||
} else if (
|
||||
(codes.at(0) === SKIER_CODE || codes.at(0) === LEVITATING_PERSON_CODE) &&
|
||||
codes.length > 1
|
||||
) {
|
||||
// Twemoji offers more gender and skin options for the skier and levitating person emoji.
|
||||
return {
|
||||
unqualified: hexNumbersToString([codes.at(0)]),
|
||||
skin,
|
||||
gender,
|
||||
};
|
||||
}
|
||||
|
||||
return hexNumbersToString(mappedCodes);
|
||||
}
|
||||
|
||||
export function emojiToInversionClassName(emoji: string): string | null {
|
||||
if (EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE.includes(emoji)) {
|
||||
return 'invert-on-dark';
|
||||
@ -189,19 +202,37 @@ export function cleanExtraEmojis(extraEmojis?: CustomEmojiMapArg) {
|
||||
return extraEmojis;
|
||||
}
|
||||
|
||||
function hexStringToNumbers(hexString: string): number[] {
|
||||
return hexString
|
||||
.split('-')
|
||||
.map((code) => Number.parseInt(code, 16))
|
||||
.filter((code) => !Number.isNaN(code));
|
||||
}
|
||||
/**
|
||||
* Tokenizes an input string into words, using Intl.Segmenter if available.
|
||||
* @param input Any input string.
|
||||
* @param segmenter Segmenter, if available.
|
||||
* @returns Array of tokens in lowercase.
|
||||
*/
|
||||
export function extractTokens(
|
||||
input: string,
|
||||
segmenter: Intl.Segmenter | null,
|
||||
): string[] {
|
||||
if (!input.trim()) {
|
||||
return [];
|
||||
}
|
||||
const tokens: string[] = [];
|
||||
|
||||
function hexNumbersToString(codes: unknown[], padding = 4): string {
|
||||
return codes
|
||||
.filter(
|
||||
(code): code is number =>
|
||||
typeof code === 'number' && code > 0 && !Number.isNaN(code),
|
||||
)
|
||||
.map((code) => code.toString(16).padStart(padding, '0').toUpperCase())
|
||||
.join('-');
|
||||
// Prefer to use Intl.Segmenter if available for better locale support.
|
||||
if (segmenter) {
|
||||
for (const { isWordLike, segment } of segmenter.segment(
|
||||
input.replaceAll('_', ' '), // Handle underscores from shortcodes.
|
||||
)) {
|
||||
if (isWordLike && segment.length >= EMOJI_MIN_TOKEN_LENGTH) {
|
||||
tokens.push(segment.toLowerCase());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Fallback to simple splitting.
|
||||
input.split(/[\s_-]+/).forEach((word) => {
|
||||
if (/\w/.test(word) && word.length >= EMOJI_MIN_TOKEN_LENGTH) {
|
||||
tokens.push(word.toLowerCase());
|
||||
}
|
||||
});
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
@ -4,7 +4,6 @@ import {
|
||||
EMOJI_TYPE_UNICODE,
|
||||
EMOJI_TYPE_CUSTOM,
|
||||
} from './constants';
|
||||
import { emojiToUnicodeHex } from './normalize';
|
||||
import type {
|
||||
EmojiLoadedState,
|
||||
EmojiMode,
|
||||
@ -16,6 +15,7 @@ import type {
|
||||
import {
|
||||
anyEmojiRegex,
|
||||
emojiLogger,
|
||||
emojiToUnicodeHex,
|
||||
isCustomEmoji,
|
||||
isUnicodeEmoji,
|
||||
stringHasUnicodeFlags,
|
||||
@ -140,12 +140,7 @@ export async function loadEmojiDataToState(
|
||||
}
|
||||
|
||||
// If not found, assume it's not an emoji and return null.
|
||||
log(
|
||||
'Could not find emoji %s of type %s for locale %s',
|
||||
state.code,
|
||||
state.type,
|
||||
locale,
|
||||
);
|
||||
log('Could not find emoji %s for locale %s', state.code, locale);
|
||||
return null;
|
||||
} catch (err: unknown) {
|
||||
// If the locale is not loaded, load it and retry once.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import type { List as ImmutableList } from 'immutable';
|
||||
|
||||
import type { FlatCompactEmoji, Locale } from 'emojibase';
|
||||
import type { CompactEmoji, Locale, SkinTone } from 'emojibase';
|
||||
|
||||
import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
|
||||
import type { CustomEmoji } from '@/flavours/glitch/models/custom_emoji';
|
||||
@ -32,10 +32,20 @@ export interface EmojiAppState {
|
||||
currentLocale: Locale;
|
||||
mode: EmojiMode;
|
||||
darkTheme: boolean;
|
||||
assetHost: string;
|
||||
}
|
||||
|
||||
export type CustomEmojiData = ApiCustomEmojiJSON;
|
||||
export type UnicodeEmojiData = FlatCompactEmoji;
|
||||
export type CustomEmojiData = ApiCustomEmojiJSON & { tokens: string[] };
|
||||
export interface UnicodeEmojiData extends Omit<
|
||||
CompactEmoji,
|
||||
'emoticon' | 'skins' | 'tags'
|
||||
> {
|
||||
shortcodes: string[];
|
||||
tokens: string[];
|
||||
emoticons?: string[];
|
||||
skinHexcodes?: string[];
|
||||
skinTones?: (SkinTone | SkinTone[])[];
|
||||
}
|
||||
export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData;
|
||||
|
||||
type CustomEmojiRenderFields = Pick<
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import debug from 'debug';
|
||||
import { fromUnicodeToHexcode } from 'emojibase';
|
||||
|
||||
import { emojiRegexPolyfill } from '@/flavours/glitch/polyfills';
|
||||
|
||||
@ -44,6 +45,10 @@ export function anyEmojiRegex() {
|
||||
);
|
||||
}
|
||||
|
||||
export function emojiToUnicodeHex(emoji: string): string {
|
||||
return fromUnicodeToHexcode(emoji, false);
|
||||
}
|
||||
|
||||
function supportsRegExpSets() {
|
||||
return 'unicodeSets' in RegExp.prototype;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user