From baed591a1d19942ec553baed41a8048ab9dd18ca Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Wed, 5 Feb 2025 12:47:13 +0100
Subject: [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers
 (#3737)

* [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers

* use switch for RobotsHeaders
---
 internal/middleware/extraheaders.go | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'internal/middleware/extraheaders.go')

diff --git a/internal/middleware/extraheaders.go b/internal/middleware/extraheaders.go
index fb91bcc93..c75b65551 100644
--- a/internal/middleware/extraheaders.go
+++ b/internal/middleware/extraheaders.go
@@ -44,12 +44,5 @@ func ExtraHeaders() gin.HandlerFunc {
 		//
 		// See: https://github.com/patcg-individual-drafts/topics
 		c.Header("Permissions-Policy", "browsing-topics=()")
-
-		// Some AI scrapers respect the following tags to opt-out
-		// of their crawling and datasets.
-		c.Header("X-Robots-Tag", "noimageai")
-		// c.Header calls .Set(), but we want to emit the header
-		// twice, not override it.
-		c.Writer.Header().Add("X-Robots-Tag", "noai")
 	}
 }
-- 
cgit v1.2.3