From 86cad1056c358cd9744f729dd6d18f2962ab2881 Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Sat, 2 Sep 2023 11:19:04 -0700
Subject: [PATCH] docs: :lock: robots.txt asset

Closes #10640, #10608
---
 README.md                                      |  1 +
 custom/assets/robots.txt                       | 14 ++++++++++++++
 packages/backend/src/server/web/views/clip.pug |  2 ++
 3 files changed, 17 insertions(+)
 create mode 100644 custom/assets/robots.txt

diff --git a/README.md b/README.md
index 42af4aa9e..aad5b4d8b 100644
--- a/README.md
+++ b/README.md
@@ -209,6 +209,7 @@ Please don't use ElasticSearch unless you already have an ElasticSearch setup an
 - To add custom error images, place them in the `./custom/assets/badges` directory, replacing the files already there.
 - To add custom sounds, place only mp3 files in the `./custom/assets/sounds` directory.
 - To update custom assets without rebuilding, just run `pnpm run gulp`.
+- To block ChatGPT, CommonCrawl, or other crawlers from indexing your instance, uncomment the respective rules in `./custom/robots.txt`.
 
 ## 🧑‍🔬 Configuring a new server
 
diff --git a/custom/assets/robots.txt b/custom/assets/robots.txt
new file mode 100644
index 000000000..cedb5b6cf
--- /dev/null
+++ b/custom/assets/robots.txt
@@ -0,0 +1,14 @@
+User-agent: *
+Allow: /
+
+# Uncomment the following to block CommonCrawl
+#
+# User-agent: CCBot
+# User-agent: CCBot/2.0
+# User-agent: CCBot/3.1
+# Disallow: /
+
+# Uncomment the following to block ChatGPT
+#
+# User-agent: GPTBot
+# Disallow: /
diff --git a/packages/backend/src/server/web/views/clip.pug b/packages/backend/src/server/web/views/clip.pug
index 79d629380..b3285dbe2 100644
--- a/packages/backend/src/server/web/views/clip.pug
+++ b/packages/backend/src/server/web/views/clip.pug
@@ -24,9 +24,11 @@ block meta
 	unless privateMode
 		if profile.noCrawle
 			meta(name='robots' content='noindex')
+
 		if profile.preventAiLearning
 			meta(name='robots' content='noai')
 			meta(name='robots' content='noimageai')
+			meta(name='GPTBot' content='noindex')
 
 		meta(name='misskey:user-username' content=user.username)
 		meta(name='misskey:user-id' content=user.id)