Timeline manager (#40)

* start messing about with timeline manager

* i have no idea what i'm doing

* i continue to not know what i'm doing

* it's coming along

* bit more progress

* update timeline with new posts as they come in

* lint and fmt

* Select accounts where empty string

* restructure a bunch, get unfaves working

* moving stuff around

* federate status deletes properly

* mention regex better but not 100% there

* fix regex

* some more hacking away at the timeline code phew

* fix up some little things

* i can't even

* more timeline stuff

* move to ulid

* fiddley

* some lil fixes for kibou compatibility

* timelines working pretty alright!

* tidy + lint

3 files changed, 30 insertions, 4 deletions

diff --git a/internal/api/security/robots.go b/internal/api/security/robots.go
new file mode 100644
index 000000000..65056072a
--- /dev/null
+++ b/internal/api/security/robots.go
@@ -0,0 +1,17 @@
+package security
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+const robotsString = `User-agent: *
+Disallow: /
+`
+
+// RobotsGETHandler returns the most restrictive possible robots.txt file in response to a call to /robots.txt.
+// The response instructs bots with *any* user agent not to index the instance at all.
+func (m *Module) RobotsGETHandler(c *gin.Context) {
+	c.String(http.StatusOK, robotsString)
+}
diff --git a/internal/api/security/security.go b/internal/api/security/security.go
index 523b5dd55..7298bc7cb 100644
--- a/internal/api/security/security.go
+++ b/internal/api/security/security.go
@@ -19,12 +19,16 @@
 package security
 
 import (
+	"net/http"
+
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 )
 
+const robotsPath = "/robots.txt"
+
 // Module implements the ClientAPIModule interface for security middleware
 type Module struct {
 	config *config.Config
@@ -44,5 +48,6 @@ func (m *Module) Route(s router.Router) error {
 	s.AttachMiddleware(m.FlocBlock)
 	s.AttachMiddleware(m.ExtraHeaders)
 	s.AttachMiddleware(m.UserAgentBlock)
+	s.AttachHandler(http.MethodGet, robotsPath, m.RobotsGETHandler)
 	return nil
 }
diff --git a/internal/api/security/useragentblock.go b/internal/api/security/useragentblock.go
index f7d3a4ffc..82d65742a 100644
--- a/internal/api/security/useragentblock.go
+++ b/internal/api/security/useragentblock.go
@@ -23,20 +23,24 @@ import (
 	"strings"
 
 	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
 )
 
-// UserAgentBlock is a middleware that prevents google chrome cohort tracking by
-// writing the Permissions-Policy header after all other parts of the request have been completed.
-// See: https://plausible.io/blog/google-floc
+// UserAgentBlock blocks requests with undesired, empty, or invalid user-agent strings.
 func (m *Module) UserAgentBlock(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func": "UserAgentBlock",
+	})
 
 	ua := c.Request.UserAgent()
 	if ua == "" {
+		l.Debug("aborting request because there's no user-agent set")
 		c.AbortWithStatus(http.StatusTeapot)
 		return
 	}
 
-	if strings.Contains(strings.ToLower(c.Request.UserAgent()), strings.ToLower("friendica")) {
+	if strings.Contains(strings.ToLower(ua), strings.ToLower("friendica")) {
+		l.Debugf("aborting request with user-agent %s because it contains 'friendica'", ua)
 		c.AbortWithStatus(http.StatusTeapot)
 		return
 	}