Madeorsk/rudeshark.net
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

filter out MFM syntaxes before detecting the post language

Browse Source
This commit is contained in:
naskya 2023-09-24 17:50:23 +09:00
parent 3633dd4e7f
commit 87c54575c1
No known key found for this signature in database
GPG Key ID: 164DFF24E2D40139
4 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/backend/src/misc/detect-language.ts Normal file
View File

@ -0,0 +1,11 @@
import { detect } from "tinyld";
import * as mfm from "mfm-js";
export default function detectLanguage(text: string) {
const nodes = mfm.parse(text);
const filtered = mfm.extract(nodes, (node) => {
return node.type === "text" || node.type === "quote";
});
const purified = mfm.toString(filtered);
return detect(purified);
}

4
packages/backend/src/remote/activitypub/renderer/note.ts
View File

@ -1,12 +1,12 @@
import { In, IsNull } from "typeorm";
import { detect as detectLanguage } from "tinyld";
import config from "@/config/index.js";
import type { Note, IMentionedRemoteUsers } from "@/models/entities/note.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { DriveFiles, Notes, Users, Emojis, Polls } from "@/models/index.js";
import type { Emoji } from "@/models/entities/emoji.js";
import type { Poll } from "@/models/entities/poll.js";
import toHtml from "../misc/get-note-html.js";
import toHtml from "@/remote/activitypub/misc/get-note-html.js";
import detectLanguage from "@/misc/detect-language.js";
import renderEmoji from "./emoji.js";
import renderMention from "./mention.js";
import renderHashtag from "./hashtag.js";

2
packages/backend/src/server/api/endpoints/notes/edit.ts
View File

@ -35,8 +35,8 @@ import renderUpdate from "@/remote/activitypub/renderer/update.js";
import { deliverToRelays } from "@/services/relay.js";
// import { deliverQuestionUpdate } from "@/services/note/polls/update.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { detect as detectLanguage } from "tinyld";
import { langmap } from "@/misc/langmap.js";
import detectLanguage from "@/misc/detect-language.js";
export const meta = {
tags: ["notes"],

4
packages/backend/src/services/note/create.ts
View File

@ -64,11 +64,11 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
import { db } from "@/db/postgre.js";
import { getActiveWebhooks } from "@/misc/webhook-cache.js";
import { shouldSilenceInstance } from "@/misc/should-block-instance.js";
import meilisearch from "../../db/meilisearch.js";
import meilisearch from "@/db/meilisearch.js";
import { redisClient } from "@/db/redis.js";
import { Mutex } from "redis-semaphore";
import { detect as detectLanguage } from "tinyld";
import { langmap } from "@/misc/langmap.js";
import detectLanguage from "@/misc/detect-language.js";
const mutedWordsCache = new Cache<
{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]