diff options
Diffstat (limited to 'internal/web/robots.go')
| -rw-r--r-- | internal/web/robots.go | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/internal/web/robots.go b/internal/web/robots.go index 92bd33014..cfd1758a4 100644 --- a/internal/web/robots.go +++ b/internal/web/robots.go @@ -64,6 +64,11 @@ Disallow: / User-agent: WellKnownBot Disallow: / +# Block Amazonbot, because Amazon. +# https://developer.amazon.com/amazonbot +User-agent: Amazonbot +Disallow: / + # Rules for everything else. User-agent: * Crawl-delay: 500 |