Merge branch 'gh-fa55fa5e/10491/unknown/feat/relevance-search' into 'develop'

[PR]: Meilisearch relevancy search

See merge request firefish/firefish!10491
This commit is contained in:
Kainoa Kanter 2023-07-29 03:03:29 +00:00
commit 3824083446
2 changed files with 87 additions and 25 deletions

View File

@ -121,6 +121,19 @@ if (hasConfig) {
), ),
); );
posts
.updateRankingRules([
"sort",
"words",
"typo",
"proximity",
"attribute",
"exactness",
])
.catch((e) => {
logger.error("Failed to set ranking rules, sorting won't work properly.");
});
logger.info("Connected to MeiliSearch"); logger.info("Connected to MeiliSearch");
} }
@ -160,6 +173,7 @@ export default hasConfig
limit: number, limit: number,
offset: number, offset: number,
userCtx: ILocalUser | null, userCtx: ILocalUser | null,
overrideSort: string | null,
) => { ) => {
/// Advanced search syntax /// Advanced search syntax
/// from:user => filter by user + optional domain /// from:user => filter by user + optional domain
@ -170,8 +184,10 @@ export default hasConfig
/// "text" => get posts with exact text between quotes /// "text" => get posts with exact text between quotes
/// filter:following => show results only from users you follow /// filter:following => show results only from users you follow
/// filter:followers => show results only from followers /// filter:followers => show results only from followers
/// order:desc/asc => order results ascending or descending
const constructedFilters: string[] = []; const constructedFilters: string[] = [];
let sortRules: string[] = [];
const splitSearch = query.split(" "); const splitSearch = query.split(" ");
@ -278,6 +294,14 @@ export default hasConfig
); );
} }
return null;
} else if (term.startsWith("order:desc")) {
sortRules.push("createdAt:desc");
return null;
} else if (term.startsWith("order:asc")) {
sortRules.push("createdAt:asc");
return null; return null;
} }
@ -286,14 +310,27 @@ export default hasConfig
) )
).filter((term) => term !== null); ).filter((term) => term !== null);
const sortRules = [];
// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) { // If the user has defined a sort rule, don't mess with it
if (
filteredSearchTerms.length === 0 &&
constructedFilters.length > 0 &&
sortRules.length === 0
) {
sortRules.push("createdAt:desc"); sortRules.push("createdAt:desc");
} }
// More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen.
if (sortRules.length > 1) {
sortRules = [sortRules[0]];
}
// An override sort takes precedence, user sorting is ignored here
if (overrideSort) {
sortRules = [overrideSort];
}
logger.info(`Searching for ${filteredSearchTerms.join(" ")}`); logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
logger.info(`Limit: ${limit}`); logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`); logger.info(`Offset: ${offset}`);

View File

@ -1,4 +1,4 @@
import { In } from "typeorm"; import { FindManyOptions, In } from "typeorm";
import { Notes } from "@/models/index.js"; import { Notes } from "@/models/index.js";
import { Note } from "@/models/entities/note.js"; import { Note } from "@/models/entities/note.js";
import config from "@/config/index.js"; import config from "@/config/index.js";
@ -58,6 +58,11 @@ export const paramDef = {
nullable: true, nullable: true,
default: null, default: null,
}, },
order: {
type: "string",
default: "chronological",
nullable: true,
},
}, },
required: ["query"], required: ["query"],
} as const; } as const;
@ -156,9 +161,6 @@ export default define(meta, paramDef, async (ps, me) => {
where: { where: {
id: In(chunk), id: In(chunk),
}, },
order: {
id: "DESC",
},
}); });
// The notes are checked for visibility and muted/blocked users when packed // The notes are checked for visibility and muted/blocked users when packed
@ -175,19 +177,31 @@ export default define(meta, paramDef, async (ps, me) => {
} else if (meilisearch) { } else if (meilisearch) {
let start = 0; let start = 0;
const chunkSize = 100; const chunkSize = 100;
const sortByDate = ps.order !== "relevancy";
type NoteResult = {
id: string;
createdAt: number;
};
const extractedNotes: NoteResult[] = [];
// Use meilisearch to fetch and step through all search results that could match the requirements
const ids = [];
while (true) { while (true) {
const results = await meilisearch.search(ps.query, chunkSize, start, me); const searchRes = await meilisearch.search(
ps.query,
chunkSize,
start,
me,
sortByDate ? "createdAt:desc" : null,
);
const results: MeilisearchNote[] = searchRes.hits as MeilisearchNote[];
start += chunkSize; start += chunkSize;
if (results.hits.length === 0) { if (results.length === 0) {
break; break;
} }
const res = results.hits const res = results
.filter((key: MeilisearchNote) => { .filter((key: MeilisearchNote) => {
if (ps.userId && key.userId !== ps.userId) { if (ps.userId && key.userId !== ps.userId) {
return false; return false;
@ -203,34 +217,45 @@ export default define(meta, paramDef, async (ps, me) => {
} }
return true; return true;
}) })
.map((key) => key.id); .map((key) => {
return {
id: key.id,
createdAt: key.createdAt,
};
});
ids.push(...res); extractedNotes.push(...res);
} }
// Sort all the results by note id DESC (newest first)
ids.sort((a, b) => b - a);
// Fetch the notes from the database until we have enough to satisfy the limit // Fetch the notes from the database until we have enough to satisfy the limit
start = 0; start = 0;
const found = []; const found = [];
while (found.length < ps.limit && start < ids.length) { const noteIDs = extractedNotes.map((note) => note.id);
const chunk = ids.slice(start, start + chunkSize);
const notes: Note[] = await Notes.find({ // Index the ID => index number into a map, so we can restore the array ordering efficiently later
const idIndexMap = new Map(noteIDs.map((id, index) => [id, index]));
while (found.length < ps.limit && start < noteIDs.length) {
const chunk = noteIDs.slice(start, start + chunkSize);
let query: FindManyOptions = {
where: { where: {
id: In(chunk), id: In(chunk),
}, },
order: { };
id: "DESC",
}, const notes: Note[] = await Notes.find(query);
});
// Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier)
// @ts-ignore
notes.sort((a, b) => idIndexMap.get(a.id) - idIndexMap.get(b.id));
// The notes are checked for visibility and muted/blocked users when packed // The notes are checked for visibility and muted/blocked users when packed
found.push(...(await Notes.packMany(notes, me))); found.push(...(await Notes.packMany(notes, me)));
start += chunkSize; start += chunkSize;
} }
// If we have more results than the limit, trim them // If we have more results than the limit, trim the results down
if (found.length > ps.limit) { if (found.length > ps.limit) {
found.length = ps.limit; found.length = ps.limit;
} }