diff options
author | 2025-02-05 12:47:13 +0100 | |
---|---|---|
committer | 2025-02-05 12:47:13 +0100 | |
commit | baed591a1d19942ec553baed41a8048ab9dd18ca (patch) | |
tree | b8c91d4f193ab2a80e71f222fb1bda4bb775805b /internal/middleware/extraheaders.go | |
parent | [bugfix] wrong nodeinfo version (tobi is a boob) (#3735) (diff) | |
download | gotosocial-baed591a1d19942ec553baed41a8048ab9dd18ca.tar.xz |
[feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers (#3737)
* [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers
* use switch for RobotsHeaders
Diffstat (limited to 'internal/middleware/extraheaders.go')
-rw-r--r-- | internal/middleware/extraheaders.go | 7 |
1 files changed, 0 insertions, 7 deletions
diff --git a/internal/middleware/extraheaders.go b/internal/middleware/extraheaders.go index fb91bcc93..c75b65551 100644 --- a/internal/middleware/extraheaders.go +++ b/internal/middleware/extraheaders.go @@ -44,12 +44,5 @@ func ExtraHeaders() gin.HandlerFunc { // // See: https://github.com/patcg-individual-drafts/topics c.Header("Permissions-Policy", "browsing-topics=()") - - // Some AI scrapers respect the following tags to opt-out - // of their crawling and datasets. - c.Header("X-Robots-Tag", "noimageai") - // c.Header calls .Set(), but we want to emit the header - // twice, not override it. - c.Writer.Header().Add("X-Robots-Tag", "noai") } } |