Adding language filter feature

Co-authored-by: CGsama <CGsama@outlook.com>
This commit is contained in:
cg sama 2023-09-20 00:07:41 +00:00 committed by Kainoa Kanter
parent bf6b480f49
commit 90f01edddc
13 changed files with 155 additions and 4 deletions

View File

@ -1375,14 +1375,19 @@ _menuDisplay:
hide: "Hide" hide: "Hide"
_wordMute: _wordMute:
muteWords: "Muted words" muteWords: "Muted words"
muteLangs: "Muted Languages"
muteWordsDescription: "Separate with spaces for an AND condition or with line breaks muteWordsDescription: "Separate with spaces for an AND condition or with line breaks
for an OR condition." for an OR condition."
muteWordsDescription2: "Surround keywords with slashes to use regular expressions." muteWordsDescription2: "Surround keywords with slashes to use regular expressions."
muteLangsDescription: "Separate with spaces or line breaks for an OR condition."
muteLangsDescription2: "Use language code e.g. en, fr, ja, zh."
softDescription: "Hide posts that fulfil the set conditions from the timeline." softDescription: "Hide posts that fulfil the set conditions from the timeline."
langDescription: "Hide posts that match set language from the timeline."
hardDescription: "Prevents posts fulfilling the set conditions from being added hardDescription: "Prevents posts fulfilling the set conditions from being added
to the timeline. In addition, these posts will not be added to the timeline even to the timeline. In addition, these posts will not be added to the timeline even
if the conditions are changed." if the conditions are changed."
soft: "Soft" soft: "Soft"
lang: "Language"
hard: "Hard" hard: "Hard"
mutedNotes: "Muted posts" mutedNotes: "Muted posts"
_instanceMute: _instanceMute:

View File

@ -1200,11 +1200,16 @@ _menuDisplay:
hide: "隠す" hide: "隠す"
_wordMute: _wordMute:
muteWords: "ミュートするワード" muteWords: "ミュートするワード"
muteLangs: "ミュートされた言語"
muteWordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。" muteWordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。"
muteWordsDescription2: "キーワードをスラッシュで囲むと正規表現になります。" muteWordsDescription2: "キーワードをスラッシュで囲むと正規表現になります。"
muteLangsDescription: "OR 条件の場合はスペースまたは改行で区切ります。"
muteLangsDescription2: "言語コードを使用します。例: en, fr, ja, zh."
softDescription: "指定した条件の投稿をタイムラインから隠します。" softDescription: "指定した条件の投稿をタイムラインから隠します。"
langDescription: "設定した言語に一致する投稿をタイムラインから非表示にします。"
hardDescription: "指定した条件の投稿をタイムラインに追加しないようにします。追加されなかった投稿は、条件を変更しても除外されたままになります。" hardDescription: "指定した条件の投稿をタイムラインに追加しないようにします。追加されなかった投稿は、条件を変更しても除外されたままになります。"
soft: "ソフト" soft: "ソフト"
lang: "言語"
hard: "ハード" hard: "ハード"
mutedNotes: "ミュートされた投稿" mutedNotes: "ミュートされた投稿"
_instanceMute: _instanceMute:

View File

@ -1110,11 +1110,16 @@ _menuDisplay:
hide: "隐藏" hide: "隐藏"
_wordMute: _wordMute:
muteWords: "过滤词" muteWords: "过滤词"
muteLangs: "过滤语言"
muteWordsDescription: "AND 条件用空格分隔OR 条件用换行符分隔。" muteWordsDescription: "AND 条件用空格分隔OR 条件用换行符分隔。"
muteWordsDescription2: "将关键字用斜线括起来表示正则表达式。" muteWordsDescription2: "将关键字用斜线括起来表示正则表达式。"
muteLangsDescription: "OR 条件用空格,换行符分隔"
muteLangsDescription2: "使用语言代码。例: en, fr, ja, zh."
softDescription: "隐藏时间线中指定条件的帖子。" softDescription: "隐藏时间线中指定条件的帖子。"
langDescription: "从时间线中隐藏与设置语言匹配的帖子。"
hardDescription: "防止将具有指定条件的帖子添加到时间线。 即使您更改条件,原先未添加的帖文也会被排除在外。" hardDescription: "防止将具有指定条件的帖子添加到时间线。 即使您更改条件,原先未添加的帖文也会被排除在外。"
soft: "软过滤" soft: "软过滤"
lang: "语言"
hard: "硬过滤" hard: "硬过滤"
mutedNotes: "已过滤的帖子" mutedNotes: "已过滤的帖子"
_instanceMute: _instanceMute:

View File

@ -54,6 +54,7 @@
"chalk": "5.3.0", "chalk": "5.3.0",
"chalk-template": "0.4.0", "chalk-template": "0.4.0",
"chokidar": "^3.5.3", "chokidar": "^3.5.3",
"cld": "^2.9.0",
"cli-highlight": "2.1.11", "cli-highlight": "2.1.11",
"color-convert": "2.0.1", "color-convert": "2.0.1",
"content-disposition": "0.5.4", "content-disposition": "0.5.4",
@ -87,6 +88,7 @@
"koa-send": "5.0.1", "koa-send": "5.0.1",
"koa-slow": "2.1.0", "koa-slow": "2.1.0",
"koa-views": "7.0.2", "koa-views": "7.0.2",
"langdetect": "0.2.1",
"megalodon": "workspace:*", "megalodon": "workspace:*",
"meilisearch": "0.34.1", "meilisearch": "0.34.1",
"mfm-js": "0.23.3", "mfm-js": "0.23.3",

41
packages/backend/src/@types/cld.d.ts vendored Normal file
View File

@ -0,0 +1,41 @@
interface Language {
readonly name: string;
readonly code: string;
readonly percent: number;
readonly score: number;
}
interface Chunk {
readonly name: string;
readonly code: string;
readonly offset: number;
readonly bytes: number;
}
interface Options {
readonly isHTML: false;
readonly languageHint: string;
readonly encodingHint: string;
readonly tldHint: string;
readonly httpHint: string;
}
interface DetectLanguage {
readonly reliable: boolean;
readonly textBytes: number;
readonly languages: Language[];
readonly chunks: Chunk[];
}
export declare module "cld" {
declare function detect(
text: string,
options: Options,
callback: (err: string, result: DetectLanguage) => void,
): void;
declare function detect(
text: string,
callback: (err: string, result: DetectLanguage) => void,
): void;
declare function detect(
text: string,
options: Options,
): Promise<DetectLanguage>;
declare function detect(text: string): Promise<DetectLanguage>;
}

View File

@ -0,0 +1,7 @@
declare module "langdetect" {
interface DetectResult {
lang: string;
prob: number;
}
export function detect(words: string): DetectResult[];
}

View File

@ -27,6 +27,8 @@ import {
} from "@/misc/populate-emojis.js"; } from "@/misc/populate-emojis.js";
import { db } from "@/db/postgre.js"; import { db } from "@/db/postgre.js";
import { IdentifiableError } from "@/misc/identifiable-error.js"; import { IdentifiableError } from "@/misc/identifiable-error.js";
import cld from "cld";
import { detect } from "langdetect";
export async function populatePoll(note: Note, meId: User["id"] | null) { export async function populatePoll(note: Note, meId: User["id"] | null) {
const poll = await Polls.findOneByOrFail({ noteId: note.id }); const poll = await Polls.findOneByOrFail({ noteId: note.id });
@ -201,6 +203,15 @@ export const NoteRepository = db.getRepository(Note).extend({
note.emojis.concat(reactionEmojiNames), note.emojis.concat(reactionEmojiNames),
host, host,
); );
let lang;
try {
lang = (await cld.detect((note.text || "") + (note.cw || "")))
.languages[0].code;
} catch (e) {
lang =
detect((note.text || "") + (note.cw || ""))?.[0]?.lang || "unknown";
}
const reactionEmoji = await populateEmojis(reactionEmojiNames, host); const reactionEmoji = await populateEmojis(reactionEmojiNames, host);
const packed: Packed<"Note"> = await awaitAll({ const packed: Packed<"Note"> = await awaitAll({
id: note.id, id: note.id,
@ -260,6 +271,7 @@ export const NoteRepository = db.getRepository(Note).extend({
: undefined, : undefined,
} }
: {}), : {}),
lang: lang,
}); });
if (packed.user.isCat && packed.user.speakAsCat && packed.text) { if (packed.user.isCat && packed.user.speakAsCat && packed.text) {

View File

@ -354,7 +354,12 @@ const isMyRenote = $i && $i.id === note.value.userId;
const showContent = ref(false); const showContent = ref(false);
const isDeleted = ref(false); const isDeleted = ref(false);
const muted = ref( const muted = ref(
getWordSoftMute(note.value, $i, defaultStore.state.mutedWords), getWordSoftMute(
note.value,
$i,
defaultStore.state.mutedWords,
defaultStore.state.mutedLangs,
),
); );
const translation = ref(null); const translation = ref(null);
const translating = ref(false); const translating = ref(false);

View File

@ -210,7 +210,12 @@ const reactButton = ref<HTMLElement>();
const showContent = ref(false); const showContent = ref(false);
const isDeleted = ref(false); const isDeleted = ref(false);
const muted = ref( const muted = ref(
getWordSoftMute(note.value, $i, defaultStore.state.mutedWords), getWordSoftMute(
note.value,
$i,
defaultStore.state.mutedWords,
defaultStore.state.mutedLangs,
),
); );
const translation = ref(null); const translation = ref(null);
const translating = ref(false); const translating = ref(false);

View File

@ -266,7 +266,12 @@ const appearNote = computed(() =>
); );
const isDeleted = ref(false); const isDeleted = ref(false);
const muted = ref( const muted = ref(
getWordSoftMute(note.value, $i, defaultStore.state.mutedWords), getWordSoftMute(
note.value,
$i,
defaultStore.state.mutedWords,
defaultStore.state.mutedLangs,
),
); );
const translation = ref(null); const translation = ref(null);
const translating = ref(false); const translating = ref(false);

View File

@ -17,6 +17,17 @@
}}</template }}</template
> >
</FormTextarea> </FormTextarea>
<MkInfo class="_formBlock">{{
i18n.ts._wordMute.langDescription
}}</MkInfo>
<FormTextarea v-model="softMutedLangs" class="_formBlock">
<span>{{ i18n.ts._wordMute.muteLangs }}</span>
<template #caption
>{{ i18n.ts._wordMute.muteLangsDescription }}<br />{{
i18n.ts._wordMute.muteLangsDescription2
}}</template
>
</FormTextarea>
</div> </div>
<div v-show="tab === 'hard'"> <div v-show="tab === 'hard'">
<MkInfo class="_formBlock" <MkInfo class="_formBlock"
@ -76,6 +87,7 @@ const render = (mutedWords) =>
const tab = ref("soft"); const tab = ref("soft");
const softMutedWords = ref(render(defaultStore.state.mutedWords)); const softMutedWords = ref(render(defaultStore.state.mutedWords));
const softMutedLangs = ref(render(defaultStore.state.mutedLangs));
const hardMutedWords = ref(render($i!.mutedWords)); const hardMutedWords = ref(render($i!.mutedWords));
const hardWordMutedNotesCount = ref(null); const hardWordMutedNotesCount = ref(null);
const changed = ref(false); const changed = ref(false);
@ -88,6 +100,10 @@ watch(softMutedWords, () => {
changed.value = true; changed.value = true;
}); });
watch(softMutedLangs, () => {
changed.value = true;
});
watch(hardMutedWords, () => { watch(hardMutedWords, () => {
changed.value = true; changed.value = true;
}); });
@ -134,9 +150,10 @@ async function save() {
return lines; return lines;
}; };
let softMutes, hardMutes; let softMutes, softMLangs, hardMutes;
try { try {
softMutes = parseMutes(softMutedWords.value, i18n.ts._wordMute.soft); softMutes = parseMutes(softMutedWords.value, i18n.ts._wordMute.soft);
softMLangs = parseMutes(softMutedLangs.value, i18n.ts._wordMute.lang);
hardMutes = parseMutes(hardMutedWords.value, i18n.ts._wordMute.hard); hardMutes = parseMutes(hardMutedWords.value, i18n.ts._wordMute.hard);
} catch (err) { } catch (err) {
// already displayed error message in parseMutes // already displayed error message in parseMutes
@ -144,6 +161,7 @@ async function save() {
} }
defaultStore.set("mutedWords", softMutes); defaultStore.set("mutedWords", softMutes);
defaultStore.set("mutedLangs", softMLangs);
await os.api("i/update", { await os.api("i/update", {
mutedWords: hardMutes, mutedWords: hardMutes,
}); });

View File

@ -6,6 +6,19 @@ export interface Muted {
const NotMuted = { muted: false, matched: [] }; const NotMuted = { muted: false, matched: [] };
function checkLangMute(
note: NoteLike,
mutedLangs: Array<string | string[]>,
): Muted {
const mutedLangList = new Set(
mutedLangs.reduce((arr, x) => [...arr, ...(Array.isArray(x) ? x : [x])]),
);
if (mutedLangList.has((note.lang?.[0]?.lang || "").split("-")[0])) {
return { muted: true, matched: [note.lang?.[0]?.lang] };
}
return NotMuted;
}
function checkWordMute( function checkWordMute(
note: NoteLike, note: NoteLike,
mutedWords: Array<string | string[]>, mutedWords: Array<string | string[]>,
@ -62,6 +75,7 @@ export function getWordSoftMute(
note: Record<string, any>, note: Record<string, any>,
me: Record<string, any> | null | undefined, me: Record<string, any> | null | undefined,
mutedWords: Array<string | string[]>, mutedWords: Array<string | string[]>,
mutedLangs: Array<string | string[]>,
): Muted { ): Muted {
// 自分自身 // 自分自身
if (me && note.userId === me.id) { if (me && note.userId === me.id) {
@ -91,6 +105,29 @@ export function getWordSoftMute(
} }
} }
} }
if (mutedLangs.length > 0) {
let noteLangMuted = checkLangMute(note, mutedLangs);
if (noteLangMuted.muted) {
noteLangMuted.what = "note";
return noteLangMuted;
}
if (note.renote) {
let renoteLangMuted = checkLangMute(note, mutedLangs);
if (renoteLangMuted.muted) {
renoteLangMuted.what = note.text == null ? "renote" : "quote";
return renoteLangMuted;
}
}
if (note.reply) {
let replyLangMuted = checkLangMute(note, mutedLangs);
if (replyLangMuted.muted) {
replyLangMuted.what = "reply";
return replyLangMuted;
}
}
}
return NotMuted; return NotMuted;
} }

View File

@ -101,6 +101,10 @@ export const defaultStore = markRaw(
where: "account", where: "account",
default: [], default: [],
}, },
mutedLangs: {
where: "account",
default: [],
},
mutedAds: { mutedAds: {
where: "account", where: "account",
default: [] as string[], default: [] as string[],