From baed591a1d19942ec553baed41a8048ab9dd18ca Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Wed, 5 Feb 2025 12:47:13 +0100
Subject: [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers
 (#3737)

* [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers

* use switch for RobotsHeaders
---
 internal/api/robots.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 internal/api/robots.go

(limited to 'internal/api/robots.go')

diff --git a/internal/api/robots.go b/internal/api/robots.go
new file mode 100644
index 000000000..3ed8282f5
--- /dev/null
+++ b/internal/api/robots.go
@@ -0,0 +1,52 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/api/robots"
+	"github.com/superseriousbusiness/gotosocial/internal/middleware"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+)
+
+type Robots struct {
+	robots *robots.Module
+}
+
+func (rb *Robots) Route(r *router.Router, m ...gin.HandlerFunc) {
+	// Create a group so we can attach middlewares.
+	robotsGroup := r.AttachGroup("robots.txt")
+
+	// Use passed-in middlewares.
+	robotsGroup.Use(m...)
+
+	// Allow caching for 24 hrs.
+	// https://www.rfc-editor.org/rfc/rfc9309.html#section-2.4
+	robotsGroup.Use(
+		middleware.CacheControl(middleware.CacheControlConfig{
+			Directives: []string{"public", "max-age=86400"},
+			Vary:       []string{"Accept-Encoding"},
+		}),
+	)
+
+	rb.robots.Route(robotsGroup.Handle)
+}
+
+func NewRobots() *Robots {
+	return &Robots{}
+}
-- 
cgit v1.2.3