feat: ✨ verify links with rel=me (#10506)

Adds Mastodon-style `rel=me` link verification, and creates a background job to verify said links Closes #9341 ![image](/attachments/861e01eb-660f-4c62-8d83-d824cb79da48) Co-authored-by: ThatOneCalculator <kainoa@t1c.dev> Co-authored-by: Namekuji <nmkj@waah.day> Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10506
2023-07-17 05:31:34 +00:00 · 2023-07-17 05:31:34 +00:00 · 15ffb8cf40
commit 15ffb8cf40
parent 5d4af6b69e
31 changed files with 176 additions and 33 deletions
--- a/.gitignore
+++ b/.gitignore
@ -46,6 +46,7 @@ files
 ormconfig.json
 packages/backend/assets/instance.css
 packages/backend/assets/sounds/None.mp3
+packages/backend/assets/LICENSE

 !packages/backend/src/db

--- a/CALCKEY.md
+++ b/CALCKEY.md
@ -16,7 +16,6 @@

 ## Work in progress

- Link verification
 - Better Messaging UI
 - Better API Documentation
 - Remote follow button
@ -118,6 +117,7 @@
 - Non-mangled unicode emojis
  - Skin tone selection support
 - [DragonflyDB](https://dragonflydb.io/) support as a Redis alternative
+- Link verification

 ## Implemented (remote)

--- a/locales/ar-SA.yml
+++ b/locales/ar-SA.yml
@ -1179,7 +1179,6 @@ _profile:
  youCanIncludeHashtags: "يمكنك أيضًا إضافة وسوم إلى سيرتك التعريفية."
  metadata: "معلومات إضافية"
  metadataEdit: "عدّل المعلومات الإضافية"
-  metadataDescription: "يُمكنك عرض 4 حقول معلومات في ملفك الشخصي"
  metadataLabel: "التسمية"
  metadataContent: "المحتوى"
  changeAvatar: "غيّر الصورة الرمزية"
--- a/locales/bn-BD.yml
+++ b/locales/bn-BD.yml
@ -1268,7 +1268,7 @@ _profile:
  youCanIncludeHashtags: "হ্যাশট্যাগ অন্তর্ভুক্ত করা যেতে পারে।"
  metadata: "অতিরিক্ত তথ্য"
  metadataEdit: "অতিরিক্ত তথ্য সম্পাদনা করুন"
-  metadataDescription: "আপনি আপনার প্রোফাইলে একটি টেবিল হিসাবে চারটি অতিরিক্ত তথ্য দেখাতে পারেন।"
+  metadataDescription: "আপনি আপনার প্রোফাইলে একটি টেবিল হিসাবে চারটি অতিরিক্ত তথ্য দেখাতে পারেন।. আপনি আপনার প্রোফাইলে লিঙ্কটি যাচাই করতে {rel} এর সাথে একটি {a} ট্যাগ বা {l} ট্যাগ যোগ করতে পারেন!"
  metadataLabel: "লেবেল"
  metadataContent: "বিষয়বস্তু"
  changeAvatar: "অ্যাভাটার পরিবর্তন করুন"
--- a/locales/ca-ES.yml
+++ b/locales/ca-ES.yml
@ -409,8 +409,8 @@ _profile:
  locationDescription: Si primer introduïu la vostra ciutat, es mostrarà l'hora local
    a altres usuaris.
  name: Nom
-  metadataDescription: Fent servir això, podràs mostrar camps d'informació addicionals
-    al vostre perfil.
+  metadataDescription: "Fent servir això, podràs mostrar camps d'informació addicionals
+    al vostre perfil. Podeu afegir una etiqueta {a} o una etiqueta {l} amb {rel} per verificar l'enllaç al vostre perfil."
 _exportOrImport:
  followingList: "Usuaris que segueixes"
  muteList: "Silencia"
--- a/locales/de-DE.yml
+++ b/locales/de-DE.yml
@ -1551,7 +1551,7 @@ _profile:
  metadata: "Zusätzliche Informationen"
  metadataEdit: "Zusätzliche Informationen bearbeiten"
  metadataDescription: "Hierdurch kannst du auf deinem Profil zusätzliche Informationsblöcke
-    anzeigen lassen."
+    anzeigen lassen. Sie können ein {a}-Tag oder ein {l}-Tag mit {rel} hinzufügen, um den Link in Ihrem Profil zu überprüfen!"
  metadataLabel: "Beschriftung"
  metadataContent: "Inhalt"
  changeAvatar: "Profilbild ändern"
--- a/locales/en-US.yml
+++ b/locales/en-US.yml
@ -1124,6 +1124,7 @@ remindMeLater: "Maybe later"
 removeQuote: "Remove quote"
 removeRecipient: "Remove recipient"
 removeMember: "Remove member"
+verifiedLink: "Verified link"

 _sensitiveMediaDetection:
  description: "Reduces the effort of server moderation through automatically recognizing
@ -1676,8 +1677,10 @@ _profile:
  youCanIncludeHashtags: "You can also include hashtags in your bio."
  metadata: "Additional Information"
  metadataEdit: "Edit additional Information"
-  metadataDescription: "Using these, you can display additional information fields
-    in your profile."
+  metadataDescription:
+    "Using these, you can display additional information fields
+    in your profile. You can add an {a} tag or {l} tag with {rel}
+    to verify the link on your profile!"
  metadataLabel: "Label"
  metadataContent: "Content"
  changeAvatar: "Change avatar"
--- a/locales/es-ES.yml
+++ b/locales/es-ES.yml
@ -1475,7 +1475,7 @@ _profile:
  youCanIncludeHashtags: "Puedes añadir hashtags"
  metadata: "información adicional"
  metadataEdit: "Editar información adicional"
-  metadataDescription: "Muestra la información adicional en el perfil"
+  metadataDescription: "Muestra la información adicional en el perfil. ¡Puede agregar una etiqueta {a} o una etiqueta {l} con {rel} para verificar el enlace en su perfil!"
  metadataLabel: "Etiqueta"
  metadataContent: "Contenido"
  changeAvatar: "Cambiar avatar"
--- a/locales/fr-FR.yml
+++ b/locales/fr-FR.yml
@ -1413,7 +1413,7 @@ _profile:
  metadata: "Informations supplémentaires"
  metadataEdit: "Éditer les informations supplémentaires"
  metadataDescription: "Vous pouvez afficher jusqu'à quatre informations supplémentaires
-    dans votre profil."
+    dans votre profil. Vous pouvez ajouter une balise {a} ou une balise {l} avec {rel} pour vérifier le lien sur votre profil!"
  metadataLabel: "Étiquette"
  metadataContent: "Contenu"
  changeAvatar: "Changer l'image de profil"
--- a/locales/id-ID.yml
+++ b/locales/id-ID.yml
@ -1399,7 +1399,7 @@ _profile:
  metadata: "Informasi tambahan"
  metadataEdit: "Sunting informasi tambahan"
  metadataDescription: "Kamu dapat menampilkan hingga 4 bagian informasi tambahan\
-    \ ke dalam profilmu."
+    \ ke dalam profilmu. Anda dapat menambahkan tag {a} atau tag {l} dengan {rel} untuk memverifikasi tautan di profil Anda!"
  metadataLabel: "Label"
  metadataContent: "Isi"
  changeAvatar: "Ubah avatar"
--- a/locales/it-IT.yml
+++ b/locales/it-IT.yml
@ -1266,7 +1266,7 @@ _profile:
  metadata: "Informazioni aggiuntive"
  metadataEdit: "Modifica informazioni aggiuntive"
  metadataDescription: "Puoi pubblicare fino a quattro informazioni aggiuntive sul
-    profilo."
+    profilo. Puoi aggiungere un tag {a} o {l} con {rel} per verificare il link sul tuo profilo!"
  metadataLabel: "Etichetta"
  metadataContent: "Contenuto"
  changeAvatar: "Modifica immagine profilo"
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@ -1491,7 +1491,7 @@ _profile:
  youCanIncludeHashtags: "ハッシュタグを含められます。"
  metadata: "追加情報"
  metadataEdit: "追加情報を編集"
-  metadataDescription: "プロフィールに表として追加情報を表示できます。"
+  metadataDescription: "プロフィールに表として追加情報を表示できます。{a}タグまたは{l}タグを{rel}とともに追加すると、プロフィールのリンクを確認できます。"
  metadataLabel: "ラベル"
  metadataContent: "内容"
  changeAvatar: "アバター画像を変更"
--- a/locales/ko-KR.yml
+++ b/locales/ko-KR.yml
@ -1319,7 +1319,7 @@ _profile:
  youCanIncludeHashtags: "해시 태그를 포함할 수 있습니다."
  metadata: "추가 정보"
  metadataEdit: "추가 정보 편집"
-  metadataDescription: "프로필에 추가 정보를 표시할 수 있어요"
+  metadataDescription: "프로필에 추가 정보를 표시할 수 있어요. {rel}과 함께 {a} 태그 또는 {l} 태그를 추가하여 프로필의 링크를 확인할 수 있습니다!"
  metadataLabel: "라벨"
  metadataContent: "내용"
  changeAvatar: "아바타 이미지 변경"
--- a/locales/pl-PL.yml
+++ b/locales/pl-PL.yml
@ -1404,7 +1404,7 @@ _profile:
  metadata: "Dodatkowe informacje"
  metadataEdit: "Edytuj dodatkowe informacje"
  metadataDescription: "Możesz wyświetlać do czterech sekcji dodatkowych informacji
-    na swoim profilu."
+    na swoim profilu. Możesz dodać tag {a} lub tag {l} z {rel}, aby zweryfikować link w swoim profilu!"
  metadataLabel: "Etykieta"
  metadataContent: "Treść"
  changeAvatar: "Zmień awatar"
--- a/locales/ru-RU.yml
+++ b/locales/ru-RU.yml
@ -1398,7 +1398,7 @@ _profile:
  youCanIncludeHashtags: "Можете использовать здесь хэштеги."
  metadata: "Дополнительные сведения"
  metadataEdit: "Редактировать дополнительные сведения"
-  metadataDescription: "Можно добавить до четырёх дополнительных граф в профиль."
+  metadataDescription: "Можно добавить до четырёх дополнительных граф в профиль. Вы можете добавить тег {a} или тег {l} с {rel}, чтобы подтвердить ссылку в своем профиле!"
  metadataLabel: "Метка"
  metadataContent: "Содержимое"
  changeAvatar: "Поменять аватар"
--- a/locales/sk-SK.yml
+++ b/locales/sk-SK.yml
@ -1337,7 +1337,7 @@ _profile:
  youCanIncludeHashtags: "Vo svojom bio môžete mať aj hashtagy."
  metadata: "Dodatočné informácie"
  metadataEdit: "Upraviť dodatočné informácie"
-  metadataDescription: "Vo svojom profile môžete uviesť až štyri dodatočné informačné polia."
+  metadataDescription: "Vo svojom profile môžete uviesť až štyri dodatočné informačné polia. Dodate lahko oznako {a} ali oznako {l} z {rel}, da preverite povezavo v svojem profile!"
  metadataLabel: "Popisok"
  metadataContent: "Obsah"
  changeAvatar: "Zmeniť avatara"
--- a/locales/tr-TR.yml
+++ b/locales/tr-TR.yml
@ -182,7 +182,7 @@ _profile:
    gösterecektir.
  youCanIncludeHashtags: Hakkımdan'da etiket kullanabilirsin.
  description: Hakkımda
-  metadataDescription: Bunları kullanarak profilinizde ek bilgi alanları görüntüleyebilirsiniz.
+  metadataDescription: 'Bunları kullanarak profilinizde ek bilgi alanları görüntüleyebilirsiniz. Profilinizdeki bağlantıyı doğrulamak için {rel} ile bir {a} etiketi veya {l} etiketi ekleyebilirsiniz!'
  metadata: Ek Bilgi
  metadataContent: İçerik
  metadataLabel: Etiket
--- a/locales/uk-UA.yml
+++ b/locales/uk-UA.yml
@ -1277,7 +1277,7 @@ _profile:
  metadata: "Додаткова інформація"
  metadataEdit: "Редагувати додаткову інформацію"
  metadataDescription: "Ви можете вказати до чотирьох пунктів додаткової інформації
-    у своєму профілі."
+    у своєму профілі. Ви можете додати тег {a} або {l} за допомогою {rel}, щоб підтвердити посилання у своєму профілі!"
  metadataLabel: "Назва"
  metadataContent: "Вміст"
  changeAvatar: "Змінити аватар"
--- a/locales/vi-VN.yml
+++ b/locales/vi-VN.yml
@ -1342,7 +1342,7 @@ _profile:
  youCanIncludeHashtags: "Bạn có thể dùng hashtag trong tiểu sử."
  metadata: "Thông tin bổ sung"
  metadataEdit: "Sửa thông tin bổ sung"
-  metadataDescription: "Sử dụng phần này, bạn có thể hiển thị các mục thông tin bổ sung trong hồ sơ của mình."
+  metadataDescription: "Sử dụng phần này, bạn có thể hiển thị các mục thông tin bổ sung trong hồ sơ của mình. Bạn có thể thêm thẻ {a} hoặc thẻ {l} với {rel} để xác minh liên kết trên tiểu sử của mình!"
  metadataLabel: "Nhãn"
  metadataContent: "Nội dung"
  changeAvatar: "Đổi ảnh đại diện"
--- a/locales/zh-CN.yml
+++ b/locales/zh-CN.yml
@ -1402,7 +1402,7 @@ _profile:
  youCanIncludeHashtags: "您可以包含一个话题标签。"
  metadata: "附加信息"
  metadataEdit: "附加信息编辑"
-  metadataDescription: "使用这些，您可以在您的个人资料中显示其它信息字段。"
+  metadataDescription: "使用这些，您可以在您的个人资料中显示其它信息字段。您可以添加带有 {rel} 的 {a} 标签或 {l} 标签来验证您个人资料上的链接！"
  metadataLabel: "标签"
  metadataContent: "内容"
  changeAvatar: "修改头像"
--- a/locales/zh-TW.yml
+++ b/locales/zh-TW.yml
@ -1361,7 +1361,7 @@ _profile:
  youCanIncludeHashtags: "你也可以在「關於我」中加上 #tag。"
  metadata: "進階資訊"
  metadataEdit: "編輯進階資訊"
-  metadataDescription: "可以在個人資料中以表格形式顯示其他資訊。"
+  metadataDescription: "可以在個人資料中以表格形式顯示其他資訊。您可以添加帶有 {rel} 的 {a} 標籤或 {l} 標籤來驗證您個人資料上的鏈接！"
  metadataLabel: "標籤"
  metadataContent: "内容"
  changeAvatar: "更換大頭貼"
--- a/packages/backend/src/models/entities/user-profile.ts
+++ b/packages/backend/src/models/entities/user-profile.ts
@ -51,6 +51,7 @@ export class UserProfile {
 	public fields: {
 		name: string;
 		value: string;
+		verified?: boolean;
 	}[];

 	@Column("varchar", {
--- a/packages/backend/src/queue/index.ts
+++ b/packages/backend/src/queue/index.ts
@ -576,6 +576,16 @@ export default function () {
 		{ removeOnComplete: true, removeOnFail: true },
 	);

+	systemQueue.add(
+		"verifyLinks",
+		{},
+		{
+			repeat: { cron: "0 0 * * 0" },
+			removeOnComplete: true,
+			removeOnFail: true,
+		},
+	);
+
 	processSystemQueue(systemQueue);
 }

--- a/packages/backend/src/queue/processors/system/index.ts
+++ b/packages/backend/src/queue/processors/system/index.ts
@ -5,6 +5,7 @@ import { cleanCharts } from "./clean-charts.js";
 import { checkExpiredMutings } from "./check-expired-mutings.js";
 import { clean } from "./clean.js";
 import { setLocalEmojiSizes } from "./local-emoji-size.js";
+import { verifyLinks } from "./verify-links.js";

 const jobs = {
 	tickCharts,
@ -13,6 +14,7 @@ const jobs = {
 	checkExpiredMutings,
 	clean,
 	setLocalEmojiSizes,
+	verifyLinks,
 } as Record<
 	string,
 	| Bull.ProcessCallbackFunction<Record<string, unknown>>
--- a/packages/backend/src/queue/processors/system/verify-links.ts
+++ b/packages/backend/src/queue/processors/system/verify-links.ts
@ -0,0 +1,44 @@
+import type Bull from "bull";
+
+import { UserProfiles } from "@/models/index.js";
+import { Not } from "typeorm";
+import { queueLogger } from "../../logger.js";
+import { verifyLink } from "@/services/fetch-rel-me.js";
+import config from "@/config/index.js";
+
+const logger = queueLogger.createSubLogger("verify-links");
+
+export async function verifyLinks(
+	job: Bull.Job<Record<string, unknown>>,
+	done: any,
+): Promise<void> {
+	logger.info("Verifying links...");
+
+	const usersToVerify = await UserProfiles.findBy({
+		fields: Not(null),
+		userHost: "",
+	});
+	for (const user of usersToVerify) {
+		for (const field of user.fields) {
+			if (!field || field.name === "" || field.value === "") {
+				continue;
+			}
+			if (field.value.startsWith("http") && user.user?.username) {
+				field.verified = await verifyLink(field.value, user.user.username);
+			}
+		}
+		if (user.fields.length > 0) {
+			try {
+				await UserProfiles.update(user.userId, {
+					fields: user.fields,
+				});
+			} catch (e) {
+				logger.error(`Failed to update user ${user.userId} ${e}`);
+				done(e);
+			}
+		}
+	}
+
+	logger.succ("All links successfully verified.");
+	done();
+}
--- a/packages/backend/src/server/api/endpoints/i/update.ts
+++ b/packages/backend/src/server/api/endpoints/i/update.ts
@ -12,7 +12,9 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
 import { notificationTypes } from "@/types.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
 import { langmap } from "@/misc/langmap.js";
+import { verifyLink } from "@/services/fetch-rel-me.js";
 import { ApiError } from "../../error.js";
+import config from "@/config/index.js";
 import define from "../../define.js";

 export const meta = {
@ -58,6 +60,18 @@ export const meta = {
 			code: "INVALID_REGEXP",
 			id: "0d786918-10df-41cd-8f33-8dec7d9a89a5",
 		},
+
+		invalidFieldName: {
+			message: "Invalid field name.",
+			code: "INVALID_FIELD_NAME",
+			id: "8f81972e-8b53-4d30-b0d2-efb026dda673",
+		},
+
+		invalidFieldValue: {
+			message: "Invalid field value.",
+			code: "INVALID_FIELD_VALUE",
+			id: "aede7444-244b-11ee-be56-0242ac120002",
+		},
 	},

 	res: {
@ -234,16 +248,29 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 	}

 	if (ps.fields) {
+		for (const field of ps.fields) {
+			if (!field || field.name === "" || field.value === "") {
+				continue;
+			}
+			if (typeof field.name !== "string" || field.name === "") {
+				throw new ApiError(meta.errors.invalidFieldName);
+			}
+			if (typeof field.value !== "string" || field.value === "") {
+				throw new ApiError(meta.errors.invalidFieldValue);
+			}
+			if (field.value.startsWith("http")) {
+				field.verified = await verifyLink(field.value, user.username);
+			}
+		}
+
 		profileUpdates.fields = ps.fields
-			.filter(
-				(x) =>
-					typeof x.name === "string" &&
-					x.name !== "" &&
-					typeof x.value === "string" &&
-					x.value !== "",
-			)
+			.filter((x) => Object.keys(x).length !== 0)
 			.map((x) => {
-				return { name: x.name, value: x.value };
+				return {
+					name: x.name,
+					value: x.value,
+					verified: x.verified,
+				};
 			});
 	}

--- a/packages/backend/src/services/fetch-rel-me.ts
+++ b/packages/backend/src/services/fetch-rel-me.ts
@ -0,0 +1,33 @@
+import { getHtml } from "@/misc/fetch.js";
+import { JSDOM } from "jsdom";
+import config from "@/config/index.js";
+
+async function getRelMeLinks(url: string): Promise<string[]> {
+	try {
+		const html = await getHtml(url);
+		const dom = new JSDOM(html);
+		const relMeLinks = [
+			...dom.window.document.querySelectorAll("a[rel='me']"),
+			...dom.window.document.querySelectorAll("link[rel='me']"),
+		].map((a) => (a as HTMLAnchorElement | HTMLLinkElement).href);
+		return relMeLinks;
+	} catch {
+		return [];
+	}
+}
+
+export async function verifyLink(link: string, username: string): Promise<boolean> {
+	let verified = false;
+	if (link.startsWith("http")) {
+		const relMeLinks = await getRelMeLinks(link);
+		verified = relMeLinks.some((href) =>
+			new RegExp(
+				`^https?:\/\/${config.host.replace(
+					/[.*+\-?^${}()|[\]\\]/g,
+					"\\$&",
+				)}\/@${username.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&")}$`,
+			).test(href),
+		);
+	}
+	return verified;
+}
--- a/packages/calckey-js/src/entities.ts
+++ b/packages/calckey-js/src/entities.ts
@ -38,7 +38,11 @@ export type UserDetailed = UserLite & {
 	createdAt: DateString;
 	description: string | null;
 	ffVisibility: "public" | "followers" | "private";
-	fields: { name: string; value: string }[];
+	fields: {
+		name: string;
+		value: string;
+		verified?: boolean;
+	}[];
 	followersCount: number;
 	followingCount: number;
 	hasPendingFollowRequestFromYou: boolean;
--- a/packages/client/src/pages/settings/profile.vue
+++ b/packages/client/src/pages/settings/profile.vue
@ -126,7 +126,11 @@
 				</div>
 			</FormFolder>
 			<template #caption>{{
-				i18n.ts._profile.metadataDescription
+				i18n.t("_profile.metadataDescription", {
+					a: '<code><a></code>',
+					l: '<code><a></code>',
+					rel: `rel="me" href="https://${host}/@${$i.username}"`
+				})
 			}}</template>
 		</FormSlot>

@ -173,6 +177,7 @@ import { i18n } from "@/i18n";
 import { $i } from "@/account";
 import { langmap } from "@/scripts/langmap";
 import { definePageMetadata } from "@/scripts/page-metadata";
+import { host } from "@/config";

 const profile = reactive({
 	name: $i?.name,
--- a/packages/client/src/pages/user/home.vue
+++ b/packages/client/src/pages/user/home.vue
@ -288,10 +288,17 @@
 						<div v-if="user.fields.length > 0" class="fields">
 							<dl
 								v-for="(field, i) in user.fields"
+								:class="field.verified ? 'verified' : ''"
 								:key="i"
 								class="field"
 							>
 								<dt class="name">
+									<i
+										v-if="field.verified"
+										class="ph-bold ph-seal-check ph-lg ph-fw"
+										style="padding: 5px"
+										v-tooltip="i18n.ts.verifiedLink"
+									></i>
 									<Mfm
 										:text="field.name"
 										:plain="true"
@ -748,6 +755,12 @@ onUnmounted(() => {
 							margin-bottom: 8px;
 						}

+					 &.verified {
+							background-color: var(--hover);
+							border-radius: 10px;
+							color: var(--badge) !important;
+						}
+
 						> .name {
 							width: 30%;
 							overflow: hidden;
--- a/packages/megalodon/src/misskey/entities/field.ts
+++ b/packages/megalodon/src/misskey/entities/field.ts
@ -2,5 +2,6 @@ namespace MisskeyEntity {
  export type Field = {
    name: string
    value: string
+		verified?: string
  }
 }