diff --git a/static/robots.txt b/static/robots.txt index 9d4c6dc..98cffe6 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -3,5 +3,24 @@ Allow: /help/ Allow: /page/ Allow: /hypha/ Allow: /recent-changes -Disallow: / +Disallow: /lock +Disallow: /text/ +Disallow: /binary/ +Disallow: /rev/ +Disallow: /rev-text/ +Disallow: /primitive-diff/ +Disallow: /attachment/ +Disallow: /edit/ +Disallow: /delete-ask/ +Disallow: /rename-ask/ +Disallow: /unattach-ask/ +Disallow: /delete-confirm/ +Disallow: /rename-confirm/ +Disallow: /unattach-confirm/ +Disallow: /history/ +Disallow: /list +Disallow: /random +Disallow: /title-search/ +Disallow: /backlinks/ +Disallow: /user-list Crawl-delay: 5