From 4a7bad11c697b53712dfc162f87941dd16199c9c Mon Sep 17 00:00:00 2001 From: Essem Date: Fri, 22 Sep 2023 01:29:33 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20post=20language=20dat?= =?UTF-8?q?a=20to=20backend=20and=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api-change.md | 2 ++ .../migration/1695334243217-add-post-lang.js | 11 +++++++++ packages/backend/src/models/entities/note.ts | 6 +++++ .../backend/src/models/repositories/note.ts | 4 +--- .../src/remote/activitypub/models/note.ts | 23 +++++++++++++++++++ .../src/remote/activitypub/renderer/note.ts | 2 +- .../src/server/api/endpoints/notes/create.ts | 2 ++ .../src/server/api/endpoints/notes/edit.ts | 15 ++++++++++++ packages/backend/src/services/note/create.ts | 13 +++++++++++ 9 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 packages/backend/migration/1695334243217-add-post-lang.js diff --git a/docs/api-change.md b/docs/api-change.md index 4dc631f07..3dcdfc251 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -4,6 +4,8 @@ Breaking changes are indecated by the :warning: icon. ## v1.0.5 (unreleased) +- Added `lang` parameter to `notes/create` and `notes/edit`. + ### dev11 - :warning: `notes/translate` now requires credentials. diff --git a/packages/backend/migration/1695334243217-add-post-lang.js b/packages/backend/migration/1695334243217-add-post-lang.js new file mode 100644 index 000000000..f41426b4e --- /dev/null +++ b/packages/backend/migration/1695334243217-add-post-lang.js @@ -0,0 +1,11 @@ +export class AddPostLang1695334243217 { + name = 'AddPostLang1695334243217' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" ADD "lang" character varying(10)`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "lang"`); + } +} diff --git a/packages/backend/src/models/entities/note.ts b/packages/backend/src/models/entities/note.ts index 21fe64e90..2b9de38cb 100644 --- a/packages/backend/src/models/entities/note.ts +++ b/packages/backend/src/models/entities/note.ts @@ -66,6 +66,12 @@ export class Note { }) public text: string | null; + @Column("varchar", { + length: 10, + nullable: true, + }) + public lang: string | null; + @Column("varchar", { length: 256, nullable: true, diff --git a/packages/backend/src/models/repositories/note.ts b/packages/backend/src/models/repositories/note.ts index c8686a9df..7e90b7dee 100644 --- a/packages/backend/src/models/repositories/note.ts +++ b/packages/backend/src/models/repositories/note.ts @@ -199,8 +199,6 @@ export const NoteRepository = db.getRepository(Note).extend({ host, ); - const lang = - detectLanguage(`${note.cw ?? ""}\n${note.text ?? ""}`) ?? "unknown"; const reactionEmoji = await populateEmojis(reactionEmojiNames, host); const packed: Packed<"Note"> = await awaitAll({ id: note.id, @@ -260,7 +258,7 @@ export const NoteRepository = db.getRepository(Note).extend({ : undefined, } : {}), - lang: lang, + lang: note.lang, }); if (packed.user.isCat && packed.user.speakAsCat && packed.text) { diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts index 3fcaf9dc2..5c8807c29 100644 --- a/packages/backend/src/remote/activitypub/models/note.ts +++ b/packages/backend/src/remote/activitypub/models/note.ts @@ -53,6 +53,7 @@ import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; import { truncate } from "@/misc/truncate.js"; import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; +import { langmap } from "@/misc/langmap.js"; const logger = apLogger; @@ -305,11 +306,20 @@ export async function createNote( // Text parsing let text: string | null = null; + let lang: string | null = null; if ( note.source?.mediaType === "text/x.misskeymarkdown" && typeof note.source?.content === "string" ) { text = note.source.content; + if (note.contentMap != null) { + const key = Object.keys(note.contentMap)[0]; + lang = Object.keys(langmap).includes(key) ? key : null; + } + } else if (note.contentMap != null) { + const entry = Object.entries(note.contentMap)[0]; + lang = Object.keys(langmap).includes(entry[0]) ? entry[0] : null; + text = htmlToMfm(entry[1], note.tag); } else if (typeof note.content === "string") { text = htmlToMfm(note.content, note.tag); } @@ -378,6 +388,7 @@ export async function createNote( name: note.name, cw, text, + lang, localOnly: false, visibility, visibleUsers, @@ -565,11 +576,20 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) { // Text parsing let text: string | null = null; + let lang: string | null = null; if ( post.source?.mediaType === "text/x.misskeymarkdown" && typeof post.source?.content === "string" ) { text = post.source.content; + if (post.contentMap != null) { + const key = Object.keys(post.contentMap)[0]; + lang = Object.keys(langmap).includes(key) ? key : null; + } + } else if (post.contentMap != null) { + const entry = Object.entries(post.contentMap)[0]; + lang = Object.keys(langmap).includes(entry[0]) ? entry[0] : null; + text = htmlToMfm(entry[1], post.tag); } else if (typeof post.content === "string") { text = htmlToMfm(post.content, post.tag); } @@ -663,6 +683,9 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) { if (text && text !== note.text) { update.text = text; } + if (lang && lang !== note.lang) { + update.lang = lang; + } if (cw !== note.cw) { update.cw = cw ? cw : null; } diff --git a/packages/backend/src/remote/activitypub/renderer/note.ts b/packages/backend/src/remote/activitypub/renderer/note.ts index edeeb4222..19db792f8 100644 --- a/packages/backend/src/remote/activitypub/renderer/note.ts +++ b/packages/backend/src/remote/activitypub/renderer/note.ts @@ -115,7 +115,7 @@ export default async function renderNote( }), ); - const lang = detectLanguage(text); + const lang = note.lang ?? detectLanguage(text); const contentMap = lang ? { [lang]: content } : null; diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 542f617b3..150356811 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -108,6 +108,7 @@ export const paramDef = { }, }, text: { type: "string", maxLength: MAX_NOTE_TEXT_LENGTH, nullable: true }, + lang: { type: "string", nullable: true, maxLength: 10 }, cw: { type: "string", nullable: true, maxLength: 100 }, localOnly: { type: "boolean", default: false }, noExtractMentions: { type: "boolean", default: false }, @@ -294,6 +295,7 @@ export default define(meta, paramDef, async (ps, user) => { } : undefined, text: ps.text || undefined, + lang: ps.lang, reply, renote, cw: ps.cw, diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 8daf44b48..9e58021c2 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -35,6 +35,8 @@ import renderUpdate from "@/remote/activitypub/renderer/update.js"; import { deliverToRelays } from "@/services/relay.js"; // import { deliverQuestionUpdate } from "@/services/note/polls/update.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; +import { detect as detectLanguage } from "tinyld"; +import { langmap } from "@/misc/langmap.js"; export const meta = { tags: ["notes"], @@ -169,6 +171,7 @@ export const paramDef = { }, }, text: { type: "string", maxLength: MAX_NOTE_TEXT_LENGTH, nullable: true }, + lang: { type: "string", nullable: true, maxLength: 10 }, cw: { type: "string", nullable: true, maxLength: 250 }, localOnly: { type: "boolean", default: false }, noExtractMentions: { type: "boolean", default: false }, @@ -375,6 +378,15 @@ export default define(meta, paramDef, async (ps, user) => { ps.text = null; } + if (ps.lang) { + ps.lang = ps.lang.trim(); + if (!Object.keys(langmap).includes(ps.lang.trim())) throw new Error("invalid param"); + } else if (ps.text) { + ps.lang = detectLanguage(ps.text); + } else { + ps.lang = null; + } + let tags = []; let emojis = []; let mentionedUsers = []; @@ -532,6 +544,9 @@ export default define(meta, paramDef, async (ps, user) => { if (ps.text !== note.text) { update.text = ps.text; } + if (ps.lang !== note.lang) { + update.lang = ps.lang; + } if (ps.cw !== note.cw || (ps.cw && !note.cw)) { update.cw = ps.cw; } diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 18b524ecb..80ea6255e 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -67,6 +67,8 @@ import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; import meilisearch from "../../db/meilisearch.js"; import { redisClient } from "@/db/redis.js"; import { Mutex } from "redis-semaphore"; +import { detect as detectLanguage } from "tinyld"; +import { langmap } from "@/misc/langmap.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] @@ -139,6 +141,7 @@ type Option = { createdAt?: Date | null; name?: string | null; text?: string | null; + lang?: string | null; reply?: Note | null; renote?: Note | null; files?: DriveFile[] | null; @@ -276,6 +279,15 @@ export default async ( data.text = null; } + if (data.lang) { + data.lang = data.lang.trim(); + if (!Object.keys(langmap).includes(data.lang.trim())) throw new Error("invalid param"); + } else if (data.text) { + data.lang = detectLanguage(data.text); + } else { + data.lang = null; + } + let tags = data.apHashtags; let emojis = data.apEmojis; let mentionedUsers = data.apMentions; @@ -712,6 +724,7 @@ async function insertNote( : null, name: data.name, text: data.text, + lang: data.lang, hasPoll: data.poll != null, cw: data.cw == null ? null : data.cw, tags: tags.map((tag) => normalizeForSearch(tag)),