[feature] Hashtag federation (in/out), hashtag client API endpoints (#2032)

* update go-fed

* do the things

* remove unused columns from tags

* update to latest lingo from main

* further tag shenanigans

* serve stub page at tag endpoint

* we did it lads

* tests, oh tests, ohhh tests, oh tests (doo doo doo doo)

* swagger docs

* document hashtag usage + federation

* instanceGet

* don't bother parsing tag href

* rename whereStartsWith -> whereStartsLike

* remove GetOrCreateTag

* dont cache status tag timelineability

4 files changed, 115 insertions, 56 deletions

diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go
index 9121564fb..cb69cba5d 100644
--- a/internal/typeutils/converter.go
+++ b/internal/typeutils/converter.go
@@ -71,7 +71,8 @@ type TypeConverter interface {
 	// EmojiCategoryToAPIEmojiCategory converts a gts model emoji category into its api (frontend) representation.
 	EmojiCategoryToAPIEmojiCategory(ctx context.Context, category *gtsmodel.EmojiCategory) (*apimodel.EmojiCategory, error)
 	// TagToAPITag converts a gts model tag into its api (frontend) representation for serialization on the API.
-	TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error)
+	// If stubHistory is set to 'true', then the 'history' field of the tag will be populated with a pointer to an empty slice, for API compatibility reasons.
+	TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error)
 	// StatusToAPIStatus converts a gts model status into its api (frontend) representation for serialization on the API.
 	//
 	// Requesting account can be nil.
@@ -160,6 +161,8 @@ type TypeConverter interface {
 	MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab.ActivityStreamsMention, error)
 	// EmojiToAS converts a gts emoji into a mastodon ns Emoji, suitable for federation
 	EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.TootEmoji, error)
+	// TagToAS converts a gts model tag into a toot Hashtag, suitable for federation.
+	TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error)
 	// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
 	AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error)
 	// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.
diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go
index 3c1615cfb..60ab24383 100644
--- a/internal/typeutils/internaltoas.go
+++ b/internal/typeutils/internaltoas.go
@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams"
@@ -33,6 +34,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
 // Converts a gts model account into an Activity Streams person type.
@@ -407,7 +409,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	if s.Account == nil {
 		a, err := c.db.GetAccountByID(ctx, s.AccountID)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error retrieving author account from db: %s", err)
+			return nil, gtserror.Newf("error retrieving author account from db: %w", err)
 		}
 		s.Account = a
 	}
@@ -418,7 +420,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	// id
 	statusURI, err := url.Parse(s.URI)
 	if err != nil {
-		return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URI, err)
+		return nil, gtserror.Newf("error parsing url %s: %w", s.URI, err)
 	}
 	statusIDProp := streams.NewJSONLDIdProperty()
 	statusIDProp.SetIRI(statusURI)
@@ -436,7 +438,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	if s.InReplyToURI != "" {
 		rURI, err := url.Parse(s.InReplyToURI)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.InReplyToURI, err)
+			return nil, gtserror.Newf("error parsing url %s: %w", s.InReplyToURI, err)
 		}
 
 		inReplyToProp := streams.NewActivityStreamsInReplyToProperty()
@@ -453,7 +455,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	if s.URL != "" {
 		sURL, err := url.Parse(s.URL)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URL, err)
+			return nil, gtserror.Newf("error parsing url %s: %w", s.URL, err)
 		}
 
 		urlProp := streams.NewActivityStreamsUrlProperty()
@@ -464,7 +466,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	// attributedTo
 	authorAccountURI, err := url.Parse(s.Account.URI)
 	if err != nil {
-		return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.URI, err)
+		return nil, gtserror.Newf("error parsing url %s: %w", s.Account.URI, err)
 	}
 	attributedToProp := streams.NewActivityStreamsAttributedToProperty()
 	attributedToProp.AppendIRI(authorAccountURI)
@@ -478,13 +480,13 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	if len(s.MentionIDs) > len(mentions) {
 		mentions, err = c.db.GetMentions(ctx, s.MentionIDs)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error getting mentions: %w", err)
+			return nil, gtserror.Newf("error getting mentions: %w", err)
 		}
 	}
 	for _, m := range mentions {
 		asMention, err := c.MentionToAS(ctx, m)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error converting mention to AS mention: %s", err)
+			return nil, gtserror.Newf("error converting mention to AS mention: %w", err)
 		}
 		tagProp.AppendActivityStreamsMention(asMention)
 	}
@@ -496,7 +498,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, emojiID := range s.EmojiIDs {
 			emoji, err := c.db.GetEmojiByID(ctx, emojiID)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error getting emoji %s from database: %s", emojiID, err)
+				return nil, gtserror.Newf("error getting emoji %s from database: %w", emojiID, err)
 			}
 			emojis = append(emojis, emoji)
 		}
@@ -504,25 +506,38 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	for _, emoji := range emojis {
 		asEmoji, err := c.EmojiToAS(ctx, emoji)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error converting emoji to AS emoji: %s", err)
+			return nil, gtserror.Newf("error converting emoji to AS emoji: %w", err)
 		}
 		tagProp.AppendTootEmoji(asEmoji)
 	}
 
 	// tag -- hashtags
-	// TODO
+	hashtags := s.Tags
+	if len(s.TagIDs) > len(hashtags) {
+		hashtags, err = c.db.GetTags(ctx, s.TagIDs)
+		if err != nil {
+			return nil, gtserror.Newf("error getting tags: %w", err)
+		}
+	}
+	for _, ht := range hashtags {
+		asHashtag, err := c.TagToAS(ctx, ht)
+		if err != nil {
+			return nil, gtserror.Newf("error converting tag to AS tag: %w", err)
+		}
+		tagProp.AppendTootHashtag(asHashtag)
+	}
 
 	status.SetActivityStreamsTag(tagProp)
 
 	// parse out some URIs we need here
 	authorFollowersURI, err := url.Parse(s.Account.FollowersURI)
 	if err != nil {
-		return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.FollowersURI, err)
+		return nil, gtserror.Newf("error parsing url %s: %w", s.Account.FollowersURI, err)
 	}
 
 	publicURI, err := url.Parse(pub.PublicActivityPubIRI)
 	if err != nil {
-		return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", pub.PublicActivityPubIRI, err)
+		return nil, gtserror.Newf("error parsing url %s: %w", pub.PublicActivityPubIRI, err)
 	}
 
 	// to and cc
@@ -534,7 +549,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, m := range mentions {
 			iri, err := url.Parse(m.TargetAccount.URI)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
+				return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
 			}
 			toProp.AppendIRI(iri)
 		}
@@ -546,7 +561,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, m := range mentions {
 			iri, err := url.Parse(m.TargetAccount.URI)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
+				return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
 			}
 			ccProp.AppendIRI(iri)
 		}
@@ -557,7 +572,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, m := range mentions {
 			iri, err := url.Parse(m.TargetAccount.URI)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
+				return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
 			}
 			ccProp.AppendIRI(iri)
 		}
@@ -568,7 +583,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, m := range mentions {
 			iri, err := url.Parse(m.TargetAccount.URI)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
+				return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
 			}
 			ccProp.AppendIRI(iri)
 		}
@@ -592,7 +607,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 		for _, attachmentID := range s.AttachmentIDs {
 			attachment, err := c.db.GetAttachmentByID(ctx, attachmentID)
 			if err != nil {
-				return nil, fmt.Errorf("StatusToAS: error getting attachment %s from database: %s", attachmentID, err)
+				return nil, gtserror.Newf("error getting attachment %s from database: %w", attachmentID, err)
 			}
 			attachments = append(attachments, attachment)
 		}
@@ -600,7 +615,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	for _, a := range attachments {
 		doc, err := c.AttachmentToAS(ctx, a)
 		if err != nil {
-			return nil, fmt.Errorf("StatusToAS: error converting attachment: %s", err)
+			return nil, gtserror.Newf("error converting attachment: %w", err)
 		}
 		attachmentProp.AppendActivityStreamsDocument(doc)
 	}
@@ -609,7 +624,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
 	// replies
 	repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
 	if err != nil {
-		return nil, fmt.Errorf("error creating repliesCollection: %s", err)
+		return nil, fmt.Errorf("error creating repliesCollection: %w", err)
 	}
 
 	repliesProp := streams.NewActivityStreamsRepliesProperty()
@@ -846,6 +861,32 @@ func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab
 	return mention, nil
 }
 
+func (c *converter) TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error) {
+	// This is probably already lowercase,
+	// but let's err on the safe side.
+	nameLower := strings.ToLower(t.Name)
+	tagURLString := uris.GenerateURIForTag(nameLower)
+
+	// Create the tag.
+	tag := streams.NewTootHashtag()
+
+	// `href` should be the URL of the tag.
+	hrefProp := streams.NewActivityStreamsHrefProperty()
+	tagURL, err := url.Parse(tagURLString)
+	if err != nil {
+		return nil, gtserror.Newf("error parsing url %s: %w", tagURLString, err)
+	}
+	hrefProp.SetIRI(tagURL)
+	tag.SetActivityStreamsHref(hrefProp)
+
+	// `name` should be the name of the tag with the # prefix.
+	nameProp := streams.NewActivityStreamsNameProperty()
+	nameProp.AppendXMLSchemaString("#" + nameLower)
+	tag.SetActivityStreamsName(nameProp)
+
+	return tag, nil
+}
+
 /*
 	 we're making something like this:
 		{
diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go
index 60c59326c..30e4f2135 100644
--- a/internal/typeutils/internaltoas_test.go
+++ b/internal/typeutils/internaltoas_test.go
@@ -403,17 +403,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
   },
   "sensitive": false,
   "summary": "",
-  "tag": {
-    "icon": {
-      "mediaType": "image/png",
-      "type": "Image",
-      "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
+  "tag": [
+    {
+      "icon": {
+        "mediaType": "image/png",
+        "type": "Image",
+        "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
+      },
+      "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
+      "name": ":rainbow:",
+      "type": "Emoji",
+      "updated": "2021-09-20T10:40:37Z"
     },
-    "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
-    "name": ":rainbow:",
-    "type": "Emoji",
-    "updated": "2021-09-20T10:40:37Z"
-  },
+    {
+      "href": "http://localhost:8080/tags/welcome",
+      "name": "#welcome",
+      "type": "Hashtag"
+    }
+  ],
   "to": "https://www.w3.org/ns/activitystreams#Public",
   "type": "Note",
   "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"
@@ -463,17 +470,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
   },
   "sensitive": false,
   "summary": "",
-  "tag": {
-    "icon": {
-      "mediaType": "image/png",
-      "type": "Image",
-      "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
+  "tag": [
+    {
+      "icon": {
+        "mediaType": "image/png",
+        "type": "Image",
+        "url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
+      },
+      "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
+      "name": ":rainbow:",
+      "type": "Emoji",
+      "updated": "2021-09-20T10:40:37Z"
     },
-    "id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
-    "name": ":rainbow:",
-    "type": "Emoji",
-    "updated": "2021-09-20T10:40:37Z"
-  },
+    {
+      "href": "http://localhost:8080/tags/welcome",
+      "name": "#welcome",
+      "type": "Hashtag"
+    }
+  ],
   "to": "https://www.w3.org/ns/activitystreams#Public",
   "type": "Note",
   "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"
diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go
index 975214da7..8ad1681d0 100644
--- a/internal/typeutils/internaltofrontend.go
+++ b/internal/typeutils/internaltofrontend.go
@@ -32,6 +32,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/uris"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -568,10 +569,18 @@ func (c *converter) EmojiCategoryToAPIEmojiCategory(ctx context.Context, categor
 	}, nil
 }
 
-func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error) {
+func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error) {
 	return apimodel.Tag{
-		Name: t.Name,
-		URL:  t.URL,
+		Name: strings.ToLower(t.Name),
+		URL:  uris.GenerateURIForTag(t.Name),
+		History: func() *[]any {
+			if !stubHistory {
+				return nil
+			}
+
+			h := make([]any, 0)
+			return &h
+		}(),
 	}, nil
 }
 
@@ -1297,19 +1306,11 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
 	var errs gtserror.MultiError
 
 	if len(tags) == 0 {
-		// GTS model tags were not populated
-
-		// Preallocate expected GTS slice
-		tags = make([]*gtsmodel.Tag, 0, len(tagIDs))
+		var err error
 
-		// Fetch GTS models for tag IDs
-		for _, id := range tagIDs {
-			tag := new(gtsmodel.Tag)
-			if err := c.db.GetByID(ctx, id, tag); err != nil {
-				errs.Appendf("error fetching tag %s from database: %v", id, err)
-				continue
-			}
-			tags = append(tags, tag)
+		tags, err = c.db.GetTags(ctx, tagIDs)
+		if err != nil {
+			errs.Appendf("error fetching tags from database: %v", err)
 		}
 	}
 
@@ -1318,7 +1319,7 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
 
 	// Convert GTS models to frontend models
 	for _, tag := range tags {
-		apiTag, err := c.TagToAPITag(ctx, tag)
+		apiTag, err := c.TagToAPITag(ctx, tag, false)
 		if err != nil {
 			errs.Appendf("error converting tag %s to api tag: %v", tag.ID, err)
 			continue