rudeshark.net/packages/backend/src/db/meilisearch.ts

import { Health, Index, MeiliSearch, Stats } from "meilisearch";
import { dbLogger } from "./logger.js";

import config from "@/config/index.js";
import { Note } from "@/models/entities/note.js";
import * as url from "url";
import { ILocalUser } from "@/models/entities/user.js";
import { Followings, Users } from "@/models/index.js";

const logger = dbLogger.createSubLogger("meilisearch", "gray", false);

let posts: Index;
let client: MeiliSearch;

const hasConfig =
	config.meilisearch &&
	(config.meilisearch.host ||
		config.meilisearch.port ||
		config.meilisearch.apiKey);

if (hasConfig) {
	const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
	const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
	const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
	const ssl = hasConfig ? config.meilisearch.ssl ?? false : false;

	logger.info("Connecting to MeiliSearch");

	client = new MeiliSearch({
		host: `${ssl ? "https" : "http"}://${host}:${port}`,
		apiKey: auth,
	});

	posts = client.index("posts");

	posts
		.updateSearchableAttributes(["text"])
		.catch((e) =>
			logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
		);

	posts
		.updateFilterableAttributes([
			"userName",
			"userHost",
			"mediaAttachment",
			"createdAt",
			"userId",
		])
		.catch((e) =>
			logger.error(
				`Setting filterable attr failed, advanced searches won't work: ${e}`,
			),
		);

	posts
		.updateSortableAttributes(["createdAt"])
		.catch((e) =>
			logger.error(
				`Setting sortable attr failed, placeholder searches won't sort properly: ${e}`,
			),
		);

	posts
		.updateStopWords([
			"the",
			"a",
			"as",
			"be",
			"of",
			"they",
			"these",
			"is",
			"are",
			"これ",
			"それ",
			"あれ",
			"この",
			"その",
			"あの",
			"ここ",
			"そこ",
			"あそこ",
			"こちら",
			"どこ",
			"私",
			"僕",
			"俺",
			"君",
			"あなた",
			"我々",
			"私達",
			"彼女",
			"彼",
			"です",
			"ます",
			"は",
			"が",
			"の",
			"に",
			"を",
			"で",
			"へ",
			"から",
			"まで",
			"より",
			"も",
			"どの",
			"と",
			"それで",
			"しかし",
		])
		.catch((e) =>
			logger.error(
				`Failed to set Meilisearch stop words, database size will be larger: ${e}`,
			),
		);

	posts
		.updateRankingRules([
			"sort",
			"words",
			"typo",
			"proximity",
			"attribute",
			"exactness",
		])
		.catch((e) => {
			logger.error("Failed to set ranking rules, sorting won't work properly.");
		});

	logger.info("Connected to MeiliSearch");
}

export type MeilisearchNote = {
	id: string;
	text: string;
	userId: string;
	userHost: string;
	userName: string;
	channelId: string;
	mediaAttachment: string;
	createdAt: number;
};

function timestampToUnix(timestamp: string) {
	let unix = 0;

	// Only contains numbers => UNIX timestamp
	if (/^\d+$/.test(timestamp)) {
		unix = Number.parseInt(timestamp);
	}

	if (unix === 0) {
		// Try to parse the timestamp as JavaScript Date
		const date = Date.parse(timestamp);
		if (isNaN(date)) return 0;
		unix = date / 1000;
	}

	return unix;
}

export default hasConfig
	? {
			search: async (
				query: string,
				limit: number,
				offset: number,
				userCtx: ILocalUser | null,
				overrideSort: string | null,
			) => {
				/// Advanced search syntax
				/// from:user => filter by user + optional domain
				/// has:image/video/audio/text/file => filter by attachment types
				/// domain:domain.com => filter by domain
				/// before:Date => show posts made before Date
				/// after: Date => show posts made after Date
				/// "text" => get posts with exact text between quotes
				/// filter:following => show results only from users you follow
				/// filter:followers => show results only from followers
				/// order:desc/asc => order results ascending or descending

				const constructedFilters: string[] = [];
				let sortRules: string[] = [];

				const splitSearch = query.split(" ");

				// Detect search operators and remove them from the actual query
				const filteredSearchTerms = (
					await Promise.all(
						splitSearch.map(async (term) => {
							if (term.startsWith("has:")) {
								const fileType = term.slice(4);
								constructedFilters.push(`mediaAttachment = "${fileType}"`);
								return null;
							} else if (term.startsWith("from:")) {
								let user = term.slice(5);

								if (user.length === 0) return null;

								// Cut off leading @, those aren't saved in the DB
								if (user.charAt(0) === "@") {
									user = user.slice(1);
								}

								// Determine if we got a webfinger address or a single username
								if (user.split("@").length > 1) {
									const splitUser = user.split("@");

									const domain = splitUser.pop();
									user = splitUser.join("@");

									constructedFilters.push(
										`userName = ${user} AND userHost = ${domain}`,
									);
								} else {
									constructedFilters.push(`userName = ${user}`);
								}

								return null;
							} else if (term.startsWith("domain:")) {
								const domain = term.slice(7);
								if (
									domain.length === 0 ||
									domain === "local" ||
									domain === config.hostname
								) {
									constructedFilters.push("userHost NOT EXISTS");
									return null;
								}
								constructedFilters.push(`userHost = ${domain}`);
								return null;
							} else if (term.startsWith("after:")) {
								const timestamp = term.slice(6);

								const unix = timestampToUnix(timestamp);

								if (unix !== 0) constructedFilters.push(`createdAt > ${unix}`);

								return null;
							} else if (term.startsWith("before:")) {
								const timestamp = term.slice(7);

								const unix = timestampToUnix(timestamp);
								if (unix !== 0) constructedFilters.push(`createdAt < ${unix}`);

								return null;
							} else if (term.startsWith("filter:following")) {
								// Check if we got a context user
								if (userCtx) {
									// Fetch user follows from DB
									const followedUsers = await Followings.find({
										where: {
											followerId: userCtx.id,
										},
										select: {
											followeeId: true,
										},
									});
									const followIDs = followedUsers.map(
										(user) => user.followeeId,
									);

									if (followIDs.length === 0) return null;

									constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
								} else {
									logger.warn(
										"search filtered to follows called without user context",
									);
								}

								return null;
							} else if (term.startsWith("filter:followers")) {
								// Check if we got a context user
								if (userCtx) {
									// Fetch users follows from DB
									const followedUsers = await Followings.find({
										where: {
											followeeId: userCtx.id,
										},
										select: {
											followerId: true,
										},
									});
									const followIDs = followedUsers.map(
										(user) => user.followerId,
									);

									if (followIDs.length === 0) return null;

									constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
								} else {
									logger.warn(
										"search filtered to followers called without user context",
									);
								}

								return null;
							} else if (term.startsWith("order:desc")) {
								sortRules.push("createdAt:desc");

								return null;
							} else if (term.startsWith("order:asc")) {
								sortRules.push("createdAt:asc");

								return null;
							}

							return term;
						}),
					)
				).filter((term) => term !== null);

				// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
				// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
				// If the user has defined a sort rule, don't mess with it
				if (
					filteredSearchTerms.length === 0 &&
					constructedFilters.length > 0 &&
					sortRules.length === 0
				) {
					sortRules.push("createdAt:desc");
				}

				// More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen.
				if (sortRules.length > 1) {
					sortRules = [sortRules[0]];
				}

				// An override sort takes precedence, user sorting is ignored here
				if (overrideSort) {
					sortRules = [overrideSort];
				}

				logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
				logger.info(`Limit: ${limit}`);
				logger.info(`Offset: ${offset}`);
				logger.info(`Filters: ${constructedFilters}`);
				logger.info(`Ordering: ${sortRules}`);

				return posts.search(filteredSearchTerms.join(" "), {
					limit: limit,
					offset: offset,
					filter: constructedFilters,
					sort: sortRules,
				});
			},
			ingestNote: async (ingestNotes: Note | Note[]) => {
				if (ingestNotes instanceof Note) {
					ingestNotes = [ingestNotes];
				}

				const indexingBatch: MeilisearchNote[] = [];

				for (const note of ingestNotes) {
					if (note.user === undefined) {
						note.user = await Users.findOne({
							where: {
								id: note.userId,
							},
						});
					}

					let attachmentType = "";
					if (note.attachedFileTypes.length > 0) {
						attachmentType = note.attachedFileTypes[0].split("/")[0];
						switch (attachmentType) {
							case "image":
							case "video":
							case "audio":
							case "text":
								break;
							default:
								attachmentType = "file";
								break;
						}
					}

					indexingBatch.push(<MeilisearchNote>{
						id: note.id.toString(),
						text: note.text ? note.text : "",
						userId: note.userId,
						userHost:
							note.userHost !== ""
								? note.userHost
								: url.parse(config.host).host,
						channelId: note.channelId ? note.channelId : "",
						mediaAttachment: attachmentType,
						userName: note.user?.username ?? "UNKNOWN",
						createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
					});
				}

				return posts
					.addDocuments(indexingBatch, {
						primaryKey: "id",
					})
					.then(() =>
						logger.info(`sent ${indexingBatch.length} posts for indexing`),
					);
			},
			serverStats: async () => {
				const health: Health = await client.health();
				const stats: Stats = await client.getStats();

				return {
					health: health.status,
					size: stats.databaseSize,
					indexed_count: stats.indexes["posts"].numberOfDocuments,
				};
			},
			deleteNotes: async (note: Note | Note[] | string | string[]) => {
				if (note instanceof Note) {
					note = [note];
				}
				if (typeof note === "string") {
					note = [note];
				}

				const deletionBatch = note
					.map((n) => {
						if (n instanceof Note) {
							return n.id;
						}

						if (n.length > 0) return n;

						logger.error(
							`Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify(
								n,
							)}`,
						);

						throw new Error(
							`Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify(
								n,
							)}`,
						);
					})
					.filter((el) => el !== null);

				await posts.deleteDocuments(deletionBatch as string[]).then(() => {
					logger.info(
						`submitted ${deletionBatch.length} large batch for deletion`,
					);
				});
			},
	  }
	: null;