Madeorsk/rudeshark.net
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'gh-fa55fa5e/10491/unknown/feat/relevance-search' into 'develop'

Browse Source 
[PR]: Meilisearch relevancy search

See merge request firefish/firefish!10491
This commit is contained in:
Kainoa Kanter 2023-07-29 03:03:29 +00:00
commit 3824083446
2 changed files with 87 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
packages/backend/src/db/meilisearch.ts
View File

@ -121,6 +121,19 @@ if (hasConfig) {
),
);
posts
.updateRankingRules([
"sort",
"words",
"typo",
"proximity",
"attribute",
"exactness",
])
.catch((e) => {
logger.error("Failed to set ranking rules, sorting won't work properly.");
});
logger.info("Connected to MeiliSearch");
}
@ -160,6 +173,7 @@ export default hasConfig
limit: number,
offset: number,
userCtx: ILocalUser | null,
overrideSort: string | null,
) => {
/// Advanced search syntax
/// from:user => filter by user + optional domain
@ -170,8 +184,10 @@ export default hasConfig
/// "text" => get posts with exact text between quotes
/// filter:following => show results only from users you follow
/// filter:followers => show results only from followers
/// order:desc/asc => order results ascending or descending
const constructedFilters: string[] = [];
let sortRules: string[] = [];
const splitSearch = query.split(" ");
@ -278,6 +294,14 @@ export default hasConfig
);
}
return null;
} else if (term.startsWith("order:desc")) {
sortRules.push("createdAt:desc");
return null;
} else if (term.startsWith("order:asc")) {
sortRules.push("createdAt:asc");
return null;
}
@ -286,14 +310,27 @@ export default hasConfig
)
).filter((term) => term !== null);
const sortRules = [];
// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
// If the user has defined a sort rule, don't mess with it
if (
filteredSearchTerms.length === 0 &&
constructedFilters.length > 0 &&
sortRules.length === 0
) {
sortRules.push("createdAt:desc");
}
// More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen.
if (sortRules.length > 1) {
sortRules = [sortRules[0]];
}
// An override sort takes precedence, user sorting is ignored here
if (overrideSort) {
sortRules = [overrideSort];
}
logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`);

69
packages/backend/src/server/api/endpoints/notes/search.ts
View File

@ -1,4 +1,4 @@
import { In } from "typeorm";
import { FindManyOptions, In } from "typeorm";
import { Notes } from "@/models/index.js";
import { Note } from "@/models/entities/note.js";
import config from "@/config/index.js";
@ -58,6 +58,11 @@ export const paramDef = {
nullable: true,
default: null,
},
order: {
type: "string",
default: "chronological",
nullable: true,
},
},
required: ["query"],
} as const;
@ -156,9 +161,6 @@ export default define(meta, paramDef, async (ps, me) => {
where: {
id: In(chunk),
},
order: {
id: "DESC",
},
});
// The notes are checked for visibility and muted/blocked users when packed
@ -175,19 +177,31 @@ export default define(meta, paramDef, async (ps, me) => {
} else if (meilisearch) {
let start = 0;
const chunkSize = 100;
const sortByDate = ps.order !== "relevancy";
type NoteResult = {
id: string;
createdAt: number;
};
const extractedNotes: NoteResult[] = [];
// Use meilisearch to fetch and step through all search results that could match the requirements
const ids = [];
while (true) {
const results = await meilisearch.search(ps.query, chunkSize, start, me);
const searchRes = await meilisearch.search(
ps.query,
chunkSize,
start,
me,
sortByDate ? "createdAt:desc" : null,
);
const results: MeilisearchNote[] = searchRes.hits as MeilisearchNote[];
start += chunkSize;
if (results.hits.length === 0) {
if (results.length === 0) {
break;
}
const res = results.hits
const res = results
.filter((key: MeilisearchNote) => {
if (ps.userId && key.userId !== ps.userId) {
return false;
@ -203,34 +217,45 @@ export default define(meta, paramDef, async (ps, me) => {
}
return true;
})
.map((key) => key.id);
.map((key) => {
return {
id: key.id,
createdAt: key.createdAt,
};
});
ids.push(...res);
extractedNotes.push(...res);
}
// Sort all the results by note id DESC (newest first)
ids.sort((a, b) => b - a);
// Fetch the notes from the database until we have enough to satisfy the limit
start = 0;
const found = [];
while (found.length < ps.limit && start < ids.length) {
const chunk = ids.slice(start, start + chunkSize);
const notes: Note[] = await Notes.find({
const noteIDs = extractedNotes.map((note) => note.id);
// Index the ID => index number into a map, so we can restore the array ordering efficiently later
const idIndexMap = new Map(noteIDs.map((id, index) => [id, index]));
while (found.length < ps.limit && start < noteIDs.length) {
const chunk = noteIDs.slice(start, start + chunkSize);
let query: FindManyOptions = {
where: {
id: In(chunk),
},
order: {
id: "DESC",
},
});
};
const notes: Note[] = await Notes.find(query);
// Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier)
// @ts-ignore
notes.sort((a, b) => idIndexMap.get(a.id) - idIndexMap.get(b.id));
// The notes are checked for visibility and muted/blocked users when packed
found.push(...(await Notes.packMany(notes, me)));
start += chunkSize;
}
// If we have more results than the limit, trim them
// If we have more results than the limit, trim the results down
if (found.length > ps.limit) {
found.length = ps.limit;
}