summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLibravatar tobi <31960611+tsmethurst@users.noreply.github.com>2025-02-24 11:17:18 +0100
committerLibravatar GitHub <noreply@github.com>2025-02-24 11:17:18 +0100
commitfd670c6a279e2aa54822546536dbf88b45a93051 (patch)
tree108c9b8ff3f01bc845a33f628ff302accde00211
parent[docs] Update `interactionPolicy` (#3703) (diff)
downloadgotosocial-fd670c6a279e2aa54822546536dbf88b45a93051.tar.xz
[feature] Use ETag for robots.txt to prevent mishaps (#3829)v0.18.1
* [feature] Use ETag for robots.txt to prevent mishaps * check incoming if-none-match header
-rw-r--r--internal/api/robots.go2
-rw-r--r--internal/api/robots/robots.go20
-rw-r--r--internal/api/util/robots.go5
3 files changed, 26 insertions, 1 deletions
diff --git a/internal/api/robots.go b/internal/api/robots.go
index 3ed8282f5..29e2823ef 100644
--- a/internal/api/robots.go
+++ b/internal/api/robots.go
@@ -39,7 +39,7 @@ func (rb *Robots) Route(r *router.Router, m ...gin.HandlerFunc) {
// https://www.rfc-editor.org/rfc/rfc9309.html#section-2.4
robotsGroup.Use(
middleware.CacheControl(middleware.CacheControlConfig{
- Directives: []string{"public", "max-age=86400"},
+ Directives: []string{"public", "no-cache"},
Vary: []string{"Accept-Encoding"},
}),
)
diff --git a/internal/api/robots/robots.go b/internal/api/robots/robots.go
index 98db4682d..b72a01ed6 100644
--- a/internal/api/robots/robots.go
+++ b/internal/api/robots/robots.go
@@ -49,9 +49,29 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
}
func (m *Module) robotsGETHandler(c *gin.Context) {
+ const ETag = "\"" + apiutil.RobotsTxtETag + "\""
+ c.Header("ETag", ETag)
+
+ if c.Request.Header.Get("If-None-Match") == ETag {
+ // Cached.
+ c.AbortWithStatus(http.StatusNotModified)
+ return
+ }
+
+ // Not cached, serve.
c.String(http.StatusOK, apiutil.RobotsTxt)
}
func (m *Module) robotsGETHandlerDisallowNodeInfo(c *gin.Context) {
+ const ETag = "\"" + apiutil.RobotsTxtDisallowNodeInfoETag + "\""
+ c.Header("ETag", ETag)
+
+ if c.Request.Header.Get("If-None-Match") == ETag {
+ // Cached.
+ c.AbortWithStatus(http.StatusNotModified)
+ return
+ }
+
+ // Not cached, serve.
c.String(http.StatusOK, apiutil.RobotsTxtDisallowNodeInfo)
}
diff --git a/internal/api/util/robots.go b/internal/api/util/robots.go
index 49fb04561..32d510edd 100644
--- a/internal/api/util/robots.go
+++ b/internal/api/util/robots.go
@@ -130,4 +130,9 @@ Disallow: /.well-known/webfinger
Disallow: /.well-known/nodeinfo
Disallow: /nodeinfo/
`
+
+ // MD5 hash of basic robots.txt.
+ RobotsTxtETag = `ce6729aacbb16fae3628210c04b462b7`
+ // MD5 hash of robots.txt with NodeInfo disallowed.
+ RobotsTxtDisallowNodeInfoETag = `a1e4ce6342978bc8d6c3e3dfab07cab4`
)