From 1101784230082351e48e0317a047805af619eaf1 Mon Sep 17 00:00:00 2001
From: Umar Getagazov <umar@handlerug.me>
Date: Wed, 29 Sep 2021 22:20:30 +0700
Subject: [PATCH] Replace Disallow: / with a more specific list

---
 static/robots.txt | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/static/robots.txt b/static/robots.txt
index 9d4c6dc..98cffe6 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -3,5 +3,24 @@ Allow: /help/
 Allow: /page/
 Allow: /hypha/
 Allow: /recent-changes
-Disallow: /
+Disallow: /lock
+Disallow: /text/
+Disallow: /binary/
+Disallow: /rev/
+Disallow: /rev-text/
+Disallow: /primitive-diff/
+Disallow: /attachment/
+Disallow: /edit/
+Disallow: /delete-ask/
+Disallow: /rename-ask/
+Disallow: /unattach-ask/
+Disallow: /delete-confirm/
+Disallow: /rename-confirm/
+Disallow: /unattach-confirm/
+Disallow: /history/
+Disallow: /list
+Disallow: /random
+Disallow: /title-search/
+Disallow: /backlinks/
+Disallow: /user-list
 Crawl-delay: 5