diff options
| author |  Daenney <daenney@users.noreply.github.com> | 2024-08-29 17:42:48 +0200 |
|---|---|---|
| committer |  GitHub <noreply@github.com> | 2024-08-29 17:42:48 +0200 |
| commit | 277b0436332c3e8a2fa9639b9f5ca2c3eeef6fac (patch) | |
| tree | 39b7c555aecb0f7fae66025eda66acb7d5b67559 /internal/web/robots.go | |
| parent | [bugfix/frontend] Fix `TypeError: gtsError is undefined` (#3245) (diff) | |
| download | gotosocial-277b0436332c3e8a2fa9639b9f5ca2c3eeef6fac.tar.xz | |
[chore] Update robots.txt with more AI scrapers (#3248)
Diffstat (limited to 'internal/web/robots.go')
| -rw-r--r-- | internal/web/robots.go | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/internal/web/robots.go b/internal/web/robots.go index 3309de97c..be0126109 100644 --- a/internal/web/robots.go +++ b/internal/web/robots.go @@ -31,6 +31,8 @@ const ( # AI scrapers and the like. # https://github.com/ai-robots-txt/ai.robots.txt/ +User-agent: AI2Bot +User-agent: Ai2Bot-Dolma User-agent: AdsBot-Google User-agent: Amazonbot User-agent: anthropic-ai @@ -50,9 +52,11 @@ User-agent: GoogleOther User-agent: GoogleOther-Image User-agent: GoogleOther-Video User-agent: GPTBot +User-agent: ICC-Crawler User-agent: ImagesiftBot User-agent: img2dataset User-agent: Meta-ExternalAgent +User-agent: Meta-ExternalFetcher User-agent: OAI-SearchBot User-agent: omgili User-agent: omgilibot @@ -61,6 +65,7 @@ User-agent: PetalBot User-agent: Scrapy User-agent: Timpibot User-agent: VelenPublicWebCrawler +User-agent: Webzio-Extended User-agent: YouBot Disallow: / |