diff options
Diffstat (limited to 'internal/web')
| -rw-r--r-- | internal/web/robots.go | 30 |
1 files changed, 22 insertions, 8 deletions
diff --git a/internal/web/robots.go b/internal/web/robots.go index b843203b5..a79c31034 100644 --- a/internal/web/robots.go +++ b/internal/web/robots.go @@ -27,29 +27,43 @@ const ( robotsPath = "/robots.txt" robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta robotsTxt = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go -# more info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro +# More info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro + +# Before we commence, a giant fuck you to ChatGPT in particular. +# https://platform.openai.com/docs/gptbot +User-agent: GPTBot +Disallow: / + +# Rules for everything else. User-agent: * Crawl-delay: 500 -# api stuff + +# API endpoints. Disallow: /api/ -# auth/login stuff + +# Auth/login endpoints. Disallow: /auth/ Disallow: /oauth/ Disallow: /check_your_email Disallow: /wait_for_approval Disallow: /account_disabled -# well known stuff + +# Well-known endpoints. Disallow: /.well-known/ -# files + +# Fileserver/media. Disallow: /fileserver/ -# s2s AP stuff + +# Fedi S2S API endpoints. Disallow: /users/ Disallow: /emoji/ -# panels + +# Settings panels. Disallow: /admin Disallow: /user Disallow: /settings/ -# domain blocklist + +# Domain blocklist. Disallow: /about/suspended` ) |