rudeshark.net/packages/backend/src/db/meilisearch.ts

import { Health, MeiliSearch, Stats } from "meilisearch";
import { dbLogger } from "./logger.js";

import config from "@/config/index.js";
import { Note } from "@/models/entities/note.js";
import * as url from "url";
import { ILocalUser, User } from "@/models/entities/user.js";
import { Followings, Users } from "@/models/index.js";

const logger = dbLogger.createSubLogger("meilisearch", "gray", false);

logger.info("Connecting to MeiliSearch");

const hasConfig =
	config.meilisearch &&
	(config.meilisearch.host ||
		config.meilisearch.port ||
		config.meilisearch.apiKey);

const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
const ssl = hasConfig ? config.meilisearch.ssl ?? false : false;

const client: MeiliSearch = new MeiliSearch({
	host: `${ssl ? "https" : "http"}://${host}:${port}`,
	apiKey: auth,
});

const posts = client.index("posts");

posts
	.updateSearchableAttributes(["text"])
	.catch((e) =>
		logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
	);

posts
	.updateFilterableAttributes([
		"userName",
		"userHost",
		"mediaAttachment",
		"createdAt",
		"userId",
	])
	.catch((e) =>
		logger.error(
			`Setting filterable attr failed, advanced searches won't work: ${e}`,
		),
	);

posts
	.updateSortableAttributes(["createdAt"])
	.catch((e) =>
		logger.error(
			`Setting sortable attr failed, placeholder searches won't sort properly: ${e}`,
		),
	);

logger.info("Connected to MeiliSearch");

export type MeilisearchNote = {
	id: string;
	text: string;
	userId: string;
	userHost: string;
	userName: string;
	channelId: string;
	mediaAttachment: string;
	createdAt: number;
};

export default hasConfig
	? {
			search: async (
				query: string,
				limit: number,
				offset: number,
				userCtx: ILocalUser | null,
			) => {
				/// Advanced search syntax
				/// from:user => filter by user + optional domain
				/// has:image/video/audio/text/file => filter by attachment types
				/// domain:domain.com => filter by domain
				/// before:Date => show posts made before Date
				/// after: Date => show posts made after Date
				/// "text" => get posts with exact text between quotes
				/// filter:following => show results only from users you follow
				/// filter:followers => show results only from followers

				let constructedFilters: string[] = [];

				let splitSearch = query.split(" ");

				// Detect search operators and remove them from the actual query
				let filteredSearchTerms = (
					await Promise.all(
						splitSearch.map(async (term) => {
							if (term.startsWith("has:")) {
								let fileType = term.slice(4);
								constructedFilters.push(`mediaAttachment = "${fileType}"`);
								return null;
							} else if (term.startsWith("from:")) {
								let user = term.slice(5);
								constructedFilters.push(`userName = ${user}`);
								return null;
							} else if (term.startsWith("domain:")) {
								let domain = term.slice(7);
								constructedFilters.push(`userHost = ${domain}`);
								return null;
							} else if (term.startsWith("after:")) {
								let timestamp = term.slice(6);
								// Try to parse the timestamp as JavaScript Date
								let date = Date.parse(timestamp);
								if (isNaN(date)) return null;
								constructedFilters.push(`createdAt > ${date / 1000}`);
								return null;
							} else if (term.startsWith("before:")) {
								let timestamp = term.slice(7);
								// Try to parse the timestamp as JavaScript Date
								let date = Date.parse(timestamp);
								if (isNaN(date)) return null;
								constructedFilters.push(`createdAt < ${date / 1000}`);
								return null;
							} else if (term.startsWith("filter:following")) {
								// Check if we got a context user
								if (userCtx) {
									// Fetch user follows from DB
									let followedUsers = await Followings.find({
										where: {
											followerId: userCtx.id,
										},
										select: {
											followeeId: true,
										},
									});
									let followIDs = followedUsers.map((user) => user.followeeId);

									if (followIDs.length === 0) return null;

									constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
								} else {
									logger.warn(
										"search filtered to follows called without user context",
									);
								}

								return null;
							} else if (term.startsWith("filter:followers")) {
								// Check if we got a context user
								if (userCtx) {
									// Fetch users follows from DB
									let followedUsers = await Followings.find({
										where: {
											followeeId: userCtx.id,
										},
										select: {
											followerId: true,
										},
									});
									let followIDs = followedUsers.map((user) => user.followerId);

									if (followIDs.length === 0) return null;

									constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
								} else {
									logger.warn(
										"search filtered to followers called without user context",
									);
								}

								return null;
							}

							return term;
						}),
					)
				).filter((term) => term !== null);

				let sortRules = [];

				// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
				// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
				if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
					sortRules.push("createdAt:desc");
				}

				logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
				logger.info(`Limit: ${limit}`);
				logger.info(`Offset: ${offset}`);
				logger.info(`Filters: ${constructedFilters}`);
				logger.info(`Ordering: ${sortRules}`);

				return posts.search(filteredSearchTerms.join(" "), {
					limit: limit,
					offset: offset,
					filter: constructedFilters,
					sort: sortRules,
				});
			},
			ingestNote: async (ingestNotes: Note | Note[]) => {
				if (ingestNotes instanceof Note) {
					ingestNotes = [ingestNotes];
				}

				let indexingBatch: MeilisearchNote[] = [];

				for (let note of ingestNotes) {
					if (note.user === undefined) {
						note.user = await Users.findOne({
							where: {
								id: note.userId,
							},
						});
					}

					let attachmentType = "";
					if (note.attachedFileTypes.length > 0) {
						attachmentType = note.attachedFileTypes[0].split("/")[0];
						switch (attachmentType) {
							case "image":
							case "video":
							case "audio":
							case "text":
								break;
							default:
								attachmentType = "file";
								break;
						}
					}

					indexingBatch.push(<MeilisearchNote>{
						id: note.id.toString(),
						text: note.text ? note.text : "",
						userId: note.userId,
						userHost:
							note.userHost !== ""
								? note.userHost
								: url.parse(config.host).host,
						channelId: note.channelId ? note.channelId : "",
						mediaAttachment: attachmentType,
						userName: note.user?.username ?? "UNKNOWN",
						createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
					});
				}

				return posts
					.addDocuments(indexingBatch, {
						primaryKey: "id",
					})
					.then(() =>
						console.log(`sent ${indexingBatch.length} posts for indexing`),
					);
			},
			serverStats: async () => {
				let health: Health = await client.health();
				let stats: Stats = await client.getStats();

				return {
					health: health.status,
					size: stats.databaseSize,
					indexed_count: stats.indexes["posts"].numberOfDocuments,
				};
			},
	  }
	: null;