[feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers (#3737)

* [feature] Use `X-Robots-Tag` headers to instruct scrapers/crawlers

* use switch for RobotsHeaders

1 files changed, 11 insertions, 0 deletions

diff --git a/internal/api/nodeinfo.go b/internal/api/nodeinfo.go
index 29942aba4..2f0c234fd 100644
--- a/internal/api/nodeinfo.go
+++ b/internal/api/nodeinfo.go
@@ -20,6 +20,7 @@ package api
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/nodeinfo"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/router"
@@ -43,6 +44,16 @@ func (w *NodeInfo) Route(r *router.Router, m ...gin.HandlerFunc) {
 		}),
 	)
 
+	// If instance is configured to serve instance stats
+	// faithfully at nodeinfo, we should allow robots to
+	// crawl nodeinfo endpoints in a limited capacity.
+	// In all other cases, disallow everything.
+	if config.GetInstanceStatsMode() == config.InstanceStatsModeServe {
+		nodeInfoGroup.Use(middleware.RobotsHeaders("allowSome"))
+	} else {
+		nodeInfoGroup.Use(middleware.RobotsHeaders(""))
+	}
+
 	w.nodeInfo.Route(nodeInfoGroup.Handle)
 }