perf: ⚡ update emojis, cache in IndexedDb

Closes #9959
May fix #9724

package.json

@@ -6,7 +6,7 @@
 		"type": "git",
 		"url": "https://codeberg.org/calckey/calckey.git"
 	},
-	"packageManager": "pnpm@8.6.2",
+	"packageManager": "pnpm@8.6.3",
 	"private": true,
 	"scripts": {
 		"rebuild": "pnpm run clean && pnpm -r run build && pnpm run gulp",

packages/client/package.json

@@ -44,6 +44,7 @@
 		"cross-env": "7.0.3",
 		"cypress": "10.11.0",
 		"date-fns": "2.30.0",
+		"emojilib": "^3.0.10",
 		"escape-regexp": "0.0.1",
 		"eventemitter3": "4.0.7",
 		"focus-trap": "^7.4.3",
@@ -79,6 +80,7 @@
 		"tsconfig-paths": "4.2.0",
 		"twemoji-parser": "14.0.0",
 		"typescript": "5.1.3",
+		"unicode-emoji-json": "^0.4.0",
 		"uuid": "9.0.0",
 		"vanilla-tilt": "1.8.0",
 		"vite": "4.3.9",

packages/client/src/components/MkAutocomplete.vue

@@ -99,7 +99,7 @@ import { acct } from "@/filters/user";
 import * as os from "@/os";
 import { MFM_TAGS } from "@/scripts/mfm-tags";
 import { defaultStore } from "@/store";
-import { emojilist } from "@/scripts/emojilist";
+import { getEmojiData } from "@/scripts/emojilist";
 import { instance } from "@/instance";
 import { i18n } from "@/i18n";
 
@@ -111,22 +111,23 @@ type EmojiDef = {
 	isCustomEmoji?: boolean;
 };
 
+const emojilist = await getEmojiData();
 const lib = emojilist.filter((x) => x.category !== "flags");
 
 const emjdb: EmojiDef[] = lib.map((x) => ({
-	emoji: x.char,
-	name: x.name,
-	url: char2filePath(x.char),
+	emoji: x.emoji,
+	name: x.slug,
+	url: char2filePath(x.emoji),
 }));
 
 for (const x of lib) {
 	if (x.keywords) {
 		for (const k of x.keywords) {
 			emjdb.push({
-				emoji: x.char,
+				emoji: x.emoji,
 				name: k,
-				aliasOf: x.name,
-				url: char2filePath(x.char),
+				aliasOf: x.slug,
+				url: char2filePath(x.emoji),
 			});
 		}
 	}

packages/client/src/components/MkEmojiPicker.vue

@@ -42,13 +42,13 @@
 					<div v-if="searchResultUnicode.length > 0" class="body">
 						<button
 							v-for="emoji in searchResultUnicode"
-							:key="emoji.name"
+							:key="emoji.slug"
 							class="_button item"
-							:title="emoji.name"
+							:title="emoji.slug"
 							tabindex="0"
 							@click="chosen(emoji, $event)"
 						>
-							<MkEmoji class="emoji" :emoji="emoji.char" />
+							<MkEmoji class="emoji" :emoji="emoji.emoji" />
 						</button>
 					</div>
 				</section>
@@ -116,10 +116,12 @@
 						:emojis="
 							emojilist
 								.filter((e) => e.category === category)
-								.map((e) => e.char)
+								.map((e) => e.emoji)
 						"
 						@chosen="chosen"
-						>{{ category }}</XSection
+						>{{
+							getNicelyLabeledCategory(category) || category
+						}}</XSection
 					>
 				</div>
 			</div>
@@ -162,9 +164,10 @@ import { ref, computed, watch, onMounted } from "vue";
 import * as Misskey from "calckey-js";
 import XSection from "@/components/MkEmojiPicker.section.vue";
 import {
-	emojilist,
+	getEmojiData,
 	UnicodeEmojiDef,
 	unicodeEmojiCategories as categories,
+	getNicelyLabeledCategory,
 } from "@/scripts/emojilist";
 import { getStaticImageUrl } from "@/scripts/get-static-image-url";
 import Ripple from "@/components/MkRipple.vue";
@@ -216,6 +219,7 @@ const height = computed(() =>
 const customEmojiCategories = emojiCategories;
 const customEmojis = instance.emojis;
 const q = ref<string | null>(null);
+const emojilist = await getEmojiData();
 const searchResultCustom = ref<Misskey.entities.CustomEmoji[]>([]);
 const searchResultUnicode = ref<UnicodeEmojiDef[]>([]);
 const tab = ref<"index" | "custom" | "unicode" | "tags">("index");
@@ -232,7 +236,7 @@ watch(q, () => {
 	const newQ = q.value.replace(/:/g, "").toLowerCase();
 
 	const searchCustom = () => {
-		const max = 8;
+		const max = 16;
 		const emojis = customEmojis;
 		const matches = new Set<Misskey.entities.CustomEmoji>();
 
@@ -304,11 +308,11 @@ watch(q, () => {
 	};
 
 	const searchUnicode = () => {
-		const max = 8;
+		const max = 32;
 		const emojis = emojilist;
 		const matches = new Set<UnicodeEmojiDef>();
 
-		const exactMatch = emojis.find((emoji) => emoji.name === newQ);
+		const exactMatch = emojis.find((emoji) => emoji.slug === newQ);
 		if (exactMatch) matches.add(exactMatch);
 
 		if (newQ.includes(" ")) {
@@ -329,8 +333,8 @@ watch(q, () => {
 				if (
 					keywords.every(
 						(keyword) =>
-							emoji.name.includes(keyword) ||
-							emoji.keywords.some((alias) =>
+							emoji.slug.includes(keyword) ||
+							emoji.keywords?.some((alias) =>
 								alias.includes(keyword)
 							)
 					)
@@ -341,7 +345,7 @@ watch(q, () => {
 			}
 		} else {
 			for (const emoji of emojis) {
-				if (emoji.name.startsWith(newQ)) {
+				if (emoji.slug.startsWith(newQ)) {
 					matches.add(emoji);
 					if (matches.size >= max) break;
 				}
@@ -350,7 +354,7 @@ watch(q, () => {
 
 			for (const emoji of emojis) {
 				if (
-					emoji.keywords.some((keyword) => keyword.startsWith(newQ))
+					emoji.keywords?.some((keyword) => keyword.startsWith(newQ))
 				) {
 					matches.add(emoji);
 					if (matches.size >= max) break;
@@ -359,7 +363,7 @@ watch(q, () => {
 			if (matches.size >= max) return matches;
 
 			for (const emoji of emojis) {
-				if (emoji.name.includes(newQ)) {
+				if (emoji.slug.includes(newQ)) {
 					matches.add(emoji);
 					if (matches.size >= max) break;
 				}
@@ -367,7 +371,7 @@ watch(q, () => {
 			if (matches.size >= max) return matches;
 
 			for (const emoji of emojis) {
-				if (emoji.keywords.some((keyword) => keyword.includes(newQ))) {
+				if (emoji.keywords?.some((keyword) => keyword.includes(newQ))) {
 					matches.add(emoji);
 					if (matches.size >= max) break;
 				}
@@ -441,7 +445,7 @@ function done(query?: any): boolean | void {
 		return true;
 	}
 	const exactMatchUnicode = emojilist.find(
-		(emoji) => emoji.char === q2 || emoji.name === q2
+		(emoji) => emoji.emoji === q2 || emoji.slug === q2
 	);
 	if (exactMatchUnicode) {
 		chosen(exactMatchUnicode);

packages/client/src/scripts/emojilist.ts

@@ -1,5 +1,9 @@
+import data from "unicode-emoji-json/data-by-group.json";
+import components from "unicode-emoji-json/data-emoji-components.json";
+import keywordSet from "emojilib";
+
 export const unicodeEmojiCategories = [
-	"face",
+	"emotion",
 	"people",
 	"animals_and_nature",
 	"food_and_drink",
@@ -10,14 +14,125 @@ export const unicodeEmojiCategories = [
 	"flags",
 ] as const;
 
+export const categoryMapping = {
+	"Smileys & Emotion": "emotion",
+	"People & Body": "people",
+	"Animals & Nature": "animals_and_nature",
+	"Food & Drink": "food_and_drink",
+	"Activities": "activity",
+	"Travel & Places": "travel_and_places",
+	"Objects": "objects",
+	"Symbols": "symbols",
+	"Flags": "flags",
+} as const;
+
+const skinToneModifiers = [
+	"light_skin_tone",
+	"medium_light_skin_tone",
+	"medium_skin_tone",
+	"medium_dark_skin_tone",
+	"dark_skin_tone",
+];
+
+const newData = {};
+
+Object.keys(data).forEach((originalCategory) => {
+	const newCategory = categoryMapping[originalCategory];
+	if (newCategory) {
+		newData[newCategory] = newData[newCategory] || [];
+		Object.keys(data[originalCategory]).forEach((emojiIndex) => {
+			const emojiObj = { ...data[originalCategory][emojiIndex] };
+			emojiObj.keywords = keywordSet[emojiObj.emoji];
+			newData[newCategory].push(emojiObj);
+
+			if (emojiObj.skin_tone_support) {
+				skinToneModifiers.forEach((modifier) => {
+					const modifiedEmojiObj = { ...emojiObj };
+					modifiedEmojiObj.emoji += components[modifier];
+					modifiedEmojiObj.skin_tone = modifier;
+					newData[newCategory].push(modifiedEmojiObj);
+				});
+			}
+		});
+	}
+});
+
 export type UnicodeEmojiDef = {
-	name: string;
-	keywords: string[];
-	char: string;
+	emoji: string;
 	category: typeof unicodeEmojiCategories[number];
+	skin_tone_support: boolean;
+	name: string;
+	slug: string;
+	emoji_version: string;
+	skin_tone?: string;
+	keywords?: string[];
 };
 
-// initial converted from https://github.com/muan/emojilib/commit/242fe68be86ed6536843b83f7e32f376468b38fb
-import _emojilist from "../emojilist.json";
+export const emojilist = newData as UnicodeEmojiDef[];
 
-export const emojilist = _emojilist as UnicodeEmojiDef[];
+const storeName = "emojiList";
+
+function openDatabase() {
+	return new Promise<IDBDatabase>((resolve, reject) => {
+		const openRequest = indexedDB.open("emojiDatabase", 1);
+
+		openRequest.onupgradeneeded = () => {
+			const db = openRequest.result;
+			if (!db.objectStoreNames.contains(storeName)) {
+				db.createObjectStore(storeName);
+			}
+		};
+		openRequest.onsuccess = () => {
+			resolve(openRequest.result);
+		};
+		openRequest.onerror = () => {
+			reject(openRequest.error);
+		};
+	});
+}
+
+function storeData(db: IDBDatabase, data) {
+	return new Promise((resolve, reject) => {
+		const transaction = db.transaction(storeName, "readwrite");
+		const store = transaction.objectStore(storeName);
+		store.put(data, "emojiListKey");
+
+		transaction.oncomplete = resolve;
+		transaction.onerror = reject;
+	});
+}
+
+function getData(db: IDBDatabase): Promise<UnicodeEmojiDef[]> {
+	return new Promise((resolve, reject) => {
+		const transaction = db.transaction(storeName, "readonly");
+		const store = transaction.objectStore(storeName);
+		const getRequest = store.get("emojiListKey");
+
+		getRequest.onsuccess = () => resolve(getRequest.result);
+		getRequest.onerror = reject;
+	});
+}
+
+export async function getEmojiData(): Promise<UnicodeEmojiDef[]> {
+	try {
+		const db = await openDatabase();
+		const cachedData = await getData(db);
+
+		if (cachedData) {
+			return cachedData;
+		} else {
+			await storeData(db, emojilist);
+			console.log("Emoji data stored in IndexedDB");
+			return emojilist;
+		}
+	} catch (err) {
+		console.error("Error accessing IndexedDB:", err);
+		return emojilist;
+	}
+}
+
+export function getNicelyLabeledCategory(internalName) {
+	return Object.keys(categoryMapping).find(
+		(key) => categoryMapping[key] === internalName
+	) || internalName;
+}

pnpm-lock.yaml

@@ -777,6 +777,9 @@ importers:
       date-fns:
         specifier: 2.30.0
         version: 2.30.0
+      emojilib:
+        specifier: ^3.0.10
+        version: 3.0.10
       escape-regexp:
         specifier: 0.0.1
         version: 0.0.1
@@ -882,6 +885,9 @@ importers:
       typescript:
         specifier: 5.1.3
         version: 5.1.3
+      unicode-emoji-json:
+        specifier: ^0.4.0
+        version: 0.4.0
       uuid:
         specifier: 9.0.0
         version: 9.0.0
@@ -6777,6 +6783,10 @@ packages:
   /emoji-regex@9.2.2:
     resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
 
+  /emojilib@3.0.10:
+    resolution: {integrity: sha512-VQtCRroFykPTJaoEBEGFg5tI+rEluabjuaVDDbSftDtiRJ5GuqRG/LGV1mmDzkJP4bh5rzuEBOafMN68/YXQcQ==}
+    dev: true
+
   /emojis-list@3.0.0:
     resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==}
     engines: {node: '>= 4'}
@@ -14866,6 +14876,10 @@ packages:
       busboy: 1.6.0
     dev: false
 
+  /unicode-emoji-json@0.4.0:
+    resolution: {integrity: sha512-lVNOwh2AnmbwqtSrEVjAWKQoVzWgyWmXVqPuPkPfKb0tnA0+uYN/4ILCTdy9IRj/+3drAVhmjwjNJQr2dhCwnA==}
+    dev: true
+
   /union-value@1.0.1:
     resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==}
     engines: {node: '>=0.10.0'}

rome.json

@@ -14,7 +14,6 @@
 			"packages/backend/src/server/web/manifest.ts",
 			"packages/backend/built/",
 			"*/model.json",
-			"packages/client/src/emojilist.json",
 			"*.md",
 			"**/tsconfig.json",
 			"*/.yml"