From 274bfcd898231203671dd08457ad404878da48ac Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Fri, 26 May 2023 03:06:41 +0200 Subject: [PATCH] Lint + formatting --- packages/backend/src/daemons/server-stats.ts | 2 +- packages/backend/src/db/meilisearch.ts | 248 ++++++++++-------- .../processors/background/index-all-notes.ts | 8 +- .../src/server/api/endpoints/notes/search.ts | 3 +- .../src/server/api/endpoints/server-info.ts | 3 +- packages/client/src/scripts/search.ts | 5 +- .../src/widgets/server-metric/index.vue | 2 +- 7 files changed, 144 insertions(+), 127 deletions(-) diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index 01b9cdcb4..4227ce6ee 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -41,7 +41,7 @@ export default function () { r: round(Math.max(0, fsStats.rIO_sec ?? 0)), w: round(Math.max(0, fsStats.wIO_sec ?? 0)), }, - meilisearch: meilisearchStats + meilisearch: meilisearchStats, }; ev.emit("serverStats", stats); log.unshift(stats); diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index 2364ac7ba..75d584f8a 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -1,4 +1,4 @@ -import {Health, MeiliSearch, Stats} from 'meilisearch'; +import {Health, MeiliSearch, Stats} from "meilisearch"; import {dbLogger} from "./logger.js"; import config from "@/config/index.js"; @@ -7,13 +7,15 @@ import * as url from "url"; import {User} from "@/models/entities/user.js"; import {Users} from "@/models/index.js"; - const logger = dbLogger.createSubLogger("meilisearch", "gray", false); logger.info("Connecting to MeiliSearch"); const hasConfig = - config.meilisearch && (config.meilisearch.host || config.meilisearch.port || config.meilisearch.apiKey); + config.meilisearch && + (config.meilisearch.host || + config.meilisearch.port || + config.meilisearch.apiKey); const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; @@ -22,13 +24,28 @@ const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; const client: MeiliSearch = new MeiliSearch({ host: `http://${host}:${port}`, apiKey: auth, -}) +}); -const posts = client.index('posts'); +const posts = client.index("posts"); -posts.updateSearchableAttributes(['text']).catch((e) => logger.error(`Setting searchable attr failed, searches won't work: ${e}`)); +posts + .updateSearchableAttributes(["text"]) + .catch((e) => + logger.error(`Setting searchable attr failed, searches won't work: ${e}`), + ); -posts.updateFilterableAttributes(["userName", "userHost", "mediaAttachment", "createdAt"]).catch((e) => logger.error(`Setting filterable attr failed, advanced searches won't work: ${e}`)); +posts + .updateFilterableAttributes([ + "userName", + "userHost", + "mediaAttachment", + "createdAt", + ]) + .catch((e) => + logger.error( + `Setting filterable attr failed, advanced searches won't work: ${e}`, + ), + ); logger.info("Connected to MeiliSearch"); @@ -40,128 +57,129 @@ export type MeilisearchNote = { userName: string; channelId: string; mediaAttachment: string; - createdAt: number -} + createdAt: number; +}; -export default hasConfig ? { - search: (query : string, limit : number, offset : number) => { +export default hasConfig + ? { + search: (query: string, limit: number, offset: number) => { + /// Advanced search syntax + /// from:user => filter by user + optional domain + /// has:image/video/audio/text/file => filter by attachment types + /// domain:domain.com => filter by domain + /// before:Date => show posts made before Date + /// after: Date => show posts made after Date - /// Advanced search syntax - /// from:user => filter by user + optional domain - /// has:image/video/audio/text/file => filter by attachment types - /// domain:domain.com => filter by domain - /// before:Date => show posts made before Date - /// after: Date => show posts made after Date + let constructedFilters: string[] = []; + let splitSearch = query.split(" "); - let constructedFilters: string[] = []; - - let splitSearch = query.split(" "); - - // Detect search operators and remove them from the actual query - splitSearch = splitSearch.filter(term => { - if (term.startsWith("has:")) { - let fileType = term.slice(4); - constructedFilters.push(`mediaAttachment = "${fileType}"`) - return false; - } else if (term.startsWith("from:")) { - let user = term.slice(5); - constructedFilters.push(`userName = ${user}`) - return false; - } else if (term.startsWith("domain:")) { - let domain = term.slice(7); - constructedFilters.push(`userHost = ${domain}`) - return false; - } else if (term.startsWith("after:")) { - let timestamp = term.slice(6); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt > ${date}`) - return false; - } else if (term.startsWith("before:")) { - let timestamp = term.slice(7); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt < ${date}`) - return false; - } - - return true; - }) - - logger.info(`Searching for ${splitSearch.join(" ")}`); - logger.info(`Limit: ${limit}`); - logger.info(`Offset: ${offset}`); - logger.info(`Filters: ${constructedFilters}`) - - - return posts.search(splitSearch.join(" "), { - limit: limit, - offset: offset, - filter: constructedFilters - }); - }, - ingestNote: async (ingestNotes: Note | Note[]) => { - if (ingestNotes instanceof Note) { - ingestNotes = [ingestNotes]; - } - - let indexingBatch: MeilisearchNote[] = []; - - for (let note of ingestNotes) { - if (note.user === undefined) { - let user = await Users.findOne({ - where: { - id: note.userId - } - }); - note.user = user; - } - - let attachmentType = ""; - if (note.attachedFileTypes.length > 0) { - attachmentType = note.attachedFileTypes[0].split("/")[0]; - switch (attachmentType) { - case "image": - case "video": - case "audio": - case "text": - break; - default: - attachmentType = "file" - break + // Detect search operators and remove them from the actual query + splitSearch = splitSearch.filter((term) => { + if (term.startsWith("has:")) { + let fileType = term.slice(4); + constructedFilters.push(`mediaAttachment = "${fileType}"`); + return false; + } else if (term.startsWith("from:")) { + let user = term.slice(5); + constructedFilters.push(`userName = ${user}`); + return false; + } else if (term.startsWith("domain:")) { + let domain = term.slice(7); + constructedFilters.push(`userHost = ${domain}`); + return false; + } else if (term.startsWith("after:")) { + let timestamp = term.slice(6); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return false; + constructedFilters.push(`createdAt > ${date}`); + return false; + } else if (term.startsWith("before:")) { + let timestamp = term.slice(7); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return false; + constructedFilters.push(`createdAt < ${date}`); + return false; } + + return true; + }); + + logger.info(`Searching for ${splitSearch.join(" ")}`); + logger.info(`Limit: ${limit}`); + logger.info(`Offset: ${offset}`); + logger.info(`Filters: ${constructedFilters}`); + + return posts.search(splitSearch.join(" "), { + limit: limit, + offset: offset, + filter: constructedFilters, + }); + }, + ingestNote: async (ingestNotes: Note | Note[]) => { + if (ingestNotes instanceof Note) { + ingestNotes = [ingestNotes]; } - indexingBatch.push({ + let indexingBatch: MeilisearchNote[] = []; + + for (let note of ingestNotes) { + if (note.user === undefined) { + let user = await Users.findOne({ + where: { + id: note.userId, + }, + }); + note.user = user; + } + + let attachmentType = ""; + if (note.attachedFileTypes.length > 0) { + attachmentType = note.attachedFileTypes[0].split("/")[0]; + switch (attachmentType) { + case "image": + case "video": + case "audio": + case "text": + break; + default: + attachmentType = "file"; + break; + } + } + + indexingBatch.push({ id: note.id.toString(), text: note.text ? note.text : "", userId: note.userId, - userHost: note.userHost !== "" ? note.userHost : url.parse(config.host).host, + userHost: + note.userHost !== "" + ? note.userHost + : url.parse(config.host).host, channelId: note.channelId ? note.channelId : "", mediaAttachment: attachmentType, userName: note.user?.username ?? "UNKNOWN", - createdAt: note.createdAt.getTime() / 1000 // division by 1000 is necessary because Node returns in ms-accuracy - } - ) - } + createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy + }); + } - let indexingIDs = indexingBatch.map(note => note.id); + let indexingIDs = indexingBatch.map((note) => note.id); - return posts.addDocuments(indexingBatch, { - primaryKey: "id" - }); - }, - serverStats: async () => { - let health : Health = await client.health(); - let stats: Stats = await client.getStats(); + return posts.addDocuments(indexingBatch, { + primaryKey: "id", + }); + }, + serverStats: async () => { + let health: Health = await client.health(); + let stats: Stats = await client.getStats(); - return { - health: health.status, - size: stats.databaseSize, - indexed_count: stats.indexes["posts"].numberOfDocuments - } + return { + health: health.status, + size: stats.databaseSize, + indexed_count: stats.indexes["posts"].numberOfDocuments, + }; + }, } -} : null; + : null; diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts index 919c29ff7..9bc53d2d3 100644 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ b/packages/backend/src/queue/processors/background/index-all-notes.ts @@ -3,7 +3,7 @@ import type Bull from "bull"; import { queueLogger } from "../../logger.js"; import { Notes } from "@/models/index.js"; import { MoreThan } from "typeorm"; -import { index } from "@/services/note/create.js"; +import {index} from "@/services/note/create.js"; import {Note} from "@/models/entities/note.js"; import meilisearch from "../../../db/meilisearch.js"; @@ -33,13 +33,13 @@ export default async function indexAllNotes( try { notes = await Notes.find({ where: { - ...(cursor ? { id: MoreThan(cursor) } : {}), + ...(cursor ? {id: MoreThan(cursor)} : {}), }, take: take, order: { id: 1, }, - relations: ["user"] + relations: ["user"], }); } catch (e) { logger.error(`Failed to query notes ${e}`); @@ -62,7 +62,7 @@ export default async function indexAllNotes( const chunk = notes.slice(i, i + batch); if (meilisearch) { - await meilisearch.ingestNote(chunk) + await meilisearch.ingestNote(chunk); } await Promise.all(chunk.map((note) => index(note, true))); diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 425414561..60f264708 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -172,7 +172,7 @@ export default define(meta, paramDef, async (ps, me) => { } return found; - } else if(meilisearch) { + } else if (meilisearch) { let start = 0; const chunkSize = 100; @@ -236,7 +236,6 @@ export default define(meta, paramDef, async (ps, me) => { } return found; - } else { const userQuery = ps.userId != null diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index 3411ba416..5ba920301 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -34,8 +34,7 @@ export default define(meta, paramDef, async () => { total: fsStats[0].size, used: fsStats[0].used, }, - meilisearch: meilisearchStats - + meilisearch: meilisearchStats, }; }); diff --git a/packages/client/src/scripts/search.ts b/packages/client/src/scripts/search.ts index 54b300e1d..15f916cfc 100644 --- a/packages/client/src/scripts/search.ts +++ b/packages/client/src/scripts/search.ts @@ -6,12 +6,13 @@ export async function search() { const { canceled, result: query } = await os.inputText({ title: i18n.ts.search, placeholder: "Enter search terms...", - text: "Advanced search operators\n" + + text: + "Advanced search operators\n" + "from:user => filter by user\n" + "has:image/video/audio/text/file => filter by attachment types\n" + "domain:domain.com => filter by domain\n" + "before:Date => show posts made before Date\n" + - "after:Date => show posts made after Date" + "after:Date => show posts made after Date", }); if (canceled || query == null || query === "") return; diff --git a/packages/client/src/widgets/server-metric/index.vue b/packages/client/src/widgets/server-metric/index.vue index 8d36819b5..1eb9f56b8 100644 --- a/packages/client/src/widgets/server-metric/index.vue +++ b/packages/client/src/widgets/server-metric/index.vue @@ -61,7 +61,7 @@ import XNet from "./net.vue"; import XCpu from "./cpu.vue"; import XMemory from "./mem.vue"; import XDisk from "./disk.vue"; -import XMeili from "./meilisearch.vue" +import XMeili from "./meilisearch.vue"; import MkContainer from "@/components/MkContainer.vue"; import { GetFormResultType } from "@/scripts/form"; import * as os from "@/os";