[feature] Hashtag federation (in/out), hashtag client API endpoints (#2032)

* update go-fed

* do the things

* remove unused columns from tags

* update to latest lingo from main

* further tag shenanigans

* serve stub page at tag endpoint

* we did it lads

* tests, oh tests, ohhh tests, oh tests (doo doo doo doo)

* swagger docs

* document hashtag usage + federation

* instanceGet

* don't bother parsing tag href

* rename whereStartsWith -> whereStartsLike

* remove GetOrCreateTag

* dont cache status tag timelineability

3 files changed, 323 insertions, 4 deletions

diff --git a/internal/processing/search/get.go b/internal/processing/search/get.go
index aaade8908..8e1881ab1 100644
--- a/internal/processing/search/get.go
+++ b/internal/processing/search/get.go
@@ -33,6 +33,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -108,14 +109,16 @@ func (p *Processor) Get(
 	// supply an offset greater than 0, return nothing as
 	// though there were no additional results.
 	if req.Offset > 0 {
-		return p.packageSearchResult(ctx, account, nil, nil)
+		return p.packageSearchResult(ctx, account, nil, nil, nil, req.APIv1)
 	}
 
 	var (
 		foundStatuses = make([]*gtsmodel.Status, 0, limit)
 		foundAccounts = make([]*gtsmodel.Account, 0, limit)
-		appendStatus  = func(foundStatus *gtsmodel.Status) { foundStatuses = append(foundStatuses, foundStatus) }
-		appendAccount = func(foundAccount *gtsmodel.Account) { foundAccounts = append(foundAccounts, foundAccount) }
+		foundTags     = make([]*gtsmodel.Tag, 0, limit)
+		appendStatus  = func(s *gtsmodel.Status) { foundStatuses = append(foundStatuses, s) }
+		appendAccount = func(a *gtsmodel.Account) { foundAccounts = append(foundAccounts, a) }
+		appendTag     = func(t *gtsmodel.Tag) { foundTags = append(foundTags, t) }
 		keepLooking   bool
 		err           error
 	)
@@ -162,6 +165,8 @@ func (p *Processor) Get(
 				account,
 				foundAccounts,
 				foundStatuses,
+				foundTags,
+				req.APIv1,
 			)
 		}
 	}
@@ -189,6 +194,48 @@ func (p *Processor) Get(
 			account,
 			foundAccounts,
 			foundStatuses,
+			foundTags,
+			req.APIv1,
+		)
+	}
+
+	// If query looks like a hashtag (ie., starts
+	// with '#'), then search for tags.
+	//
+	// Since '#' is a very unique prefix and isn't
+	// shared among account or status searches, we
+	// can save a bit of time by searching for this
+	// now, and bailing quickly if we get no results,
+	// or we're not allowed to include hashtags in
+	// search results.
+	//
+	// We know that none of the subsequent searches
+	// would show any good results either, and those
+	// searches are *much* more expensive.
+	keepLooking, err = p.hashtag(
+		ctx,
+		maxID,
+		minID,
+		limit,
+		offset,
+		query,
+		queryType,
+		appendTag,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err = gtserror.Newf("error searching for hashtag: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if !keepLooking {
+		// Return whatever we have.
+		return p.packageSearchResult(
+			ctx,
+			account,
+			foundAccounts,
+			foundStatuses,
+			foundTags,
+			req.APIv1,
 		)
 	}
 
@@ -218,6 +265,8 @@ func (p *Processor) Get(
 		account,
 		foundAccounts,
 		foundStatuses,
+		foundTags,
+		req.APIv1,
 	)
 }
 
@@ -559,6 +608,80 @@ func (p *Processor) statusByURI(
 	return nil, gtserror.SetUnretrievable(err)
 }
 
+func (p *Processor) hashtag(
+	ctx context.Context,
+	maxID string,
+	minID string,
+	limit int,
+	offset int,
+	query string,
+	queryType string,
+	appendTag func(*gtsmodel.Tag),
+) (bool, error) {
+	if query[0] != '#' {
+		// Query doesn't look like a hashtag,
+		// but if we're being instructed to
+		// look explicitly *only* for hashtags,
+		// let's be generous and assume caller
+		// just left out the hash prefix.
+
+		if queryType != queryTypeHashtags {
+			// Nope, search isn't explicitly
+			// for hashtags, keep looking.
+			return true, nil
+		}
+
+		// Search is explicitly for
+		// tags, let this one through.
+	} else if !includeHashtags(queryType) {
+		// Query looks like a hashtag,
+		// but we're not meant to include
+		// hashtags in the results.
+		//
+		// Indicate to caller they should
+		// stop looking, since they're not
+		// going to get results for this by
+		// looking in any other way.
+		return false, nil
+	}
+
+	// Query looks like a hashtag, and we're allowed
+	// to search for hashtags.
+	//
+	// Ensure this is a valid tag for our instance.
+	normalized, ok := text.NormalizeHashtag(query)
+	if !ok {
+		// Couldn't normalize/not a
+		// valid hashtag after all.
+		// Caller should stop looking.
+		return false, nil
+	}
+
+	// Search for tags starting with the normalized string.
+	tags, err := p.state.DB.SearchForTags(
+		ctx,
+		normalized,
+		maxID,
+		minID,
+		limit,
+		offset,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err := gtserror.Newf(
+			"error checking database for tags using text %s: %w",
+			normalized, err,
+		)
+		return false, err
+	}
+
+	// Return whatever we got.
+	for _, tag := range tags {
+		appendTag(tag)
+	}
+
+	return false, nil
+}
+
 // byText searches in the database for accounts and/or
 // statuses containing the given query string, using
 // the provided parameters.
diff --git a/internal/processing/search/util.go b/internal/processing/search/util.go
index 4172e4e1a..171d0e570 100644
--- a/internal/processing/search/util.go
+++ b/internal/processing/search/util.go
@@ -36,6 +36,11 @@ func includeStatuses(queryType string) bool {
 	return queryType == queryTypeAny || queryType == queryTypeStatuses
 }
 
+// return true if given queryType should include hashtags.
+func includeHashtags(queryType string) bool {
+	return queryType == queryTypeAny || queryType == queryTypeHashtags
+}
+
 // packageAccounts is a util function that just
 // converts the given accounts into an apimodel
 // account slice, or errors appropriately.
@@ -111,14 +116,59 @@ func (p *Processor) packageStatuses(
 	return apiStatuses, nil
 }
 
+// packageHashtags is a util function that just
+// converts the given hashtags into an apimodel
+// hashtag slice, or errors appropriately.
+func (p *Processor) packageHashtags(
+	ctx context.Context,
+	requestingAccount *gtsmodel.Account,
+	tags []*gtsmodel.Tag,
+	v1 bool,
+) ([]any, gtserror.WithCode) {
+	apiTags := make([]any, 0, len(tags))
+
+	var rangeF func(*gtsmodel.Tag)
+	if v1 {
+		// If API version 1, just provide slice of tag names.
+		rangeF = func(tag *gtsmodel.Tag) {
+			apiTags = append(apiTags, tag.Name)
+		}
+	} else {
+		// If API not version 1, provide slice of full tags.
+		rangeF = func(tag *gtsmodel.Tag) {
+			apiTag, err := p.tc.TagToAPITag(ctx, tag, true)
+			if err != nil {
+				log.Debugf(
+					ctx,
+					"skipping tag %s because it couldn't be converted to its api representation: %s",
+					tag.Name, err,
+				)
+				return
+			}
+
+			apiTags = append(apiTags, &apiTag)
+		}
+	}
+
+	for _, tag := range tags {
+		rangeF(tag)
+	}
+
+	return apiTags, nil
+}
+
 // packageSearchResult wraps up the given accounts
 // and statuses into an apimodel SearchResult that
 // can be serialized to an API caller as JSON.
+//
+// Set v1 to 'true' if the search is using v1 of the API.
 func (p *Processor) packageSearchResult(
 	ctx context.Context,
 	requestingAccount *gtsmodel.Account,
 	accounts []*gtsmodel.Account,
 	statuses []*gtsmodel.Status,
+	tags []*gtsmodel.Tag,
+	v1 bool,
 ) (*apimodel.SearchResult, gtserror.WithCode) {
 	apiAccounts, errWithCode := p.packageAccounts(ctx, requestingAccount, accounts)
 	if errWithCode != nil {
@@ -130,9 +180,14 @@ func (p *Processor) packageSearchResult(
 		return nil, errWithCode
 	}
 
+	apiTags, errWithCode := p.packageHashtags(ctx, requestingAccount, tags, v1)
+	if errWithCode != nil {
+		return nil, errWithCode
+	}
+
 	return &apimodel.SearchResult{
 		Accounts: apiAccounts,
 		Statuses: apiStatuses,
-		Hashtags: make([]*apimodel.Tag, 0),
+		Hashtags: apiTags,
 	}, nil
 }
diff --git a/internal/processing/timeline/tag.go b/internal/processing/timeline/tag.go
new file mode 100644
index 000000000..943aa1722
--- /dev/null
+++ b/internal/processing/timeline/tag.go
@@ -0,0 +1,141 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package timeline
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/text"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// TagTimelineGet gets a pageable timeline for the given
+// tagName and given paging parameters. It will ensure
+// that each status in the timeline is actually visible
+// to requestingAcct before returning it.
+func (p *Processor) TagTimelineGet(
+	ctx context.Context,
+	requestingAcct *gtsmodel.Account,
+	tagName string,
+	maxID string,
+	sinceID string,
+	minID string,
+	limit int,
+) (*apimodel.PageableResponse, gtserror.WithCode) {
+	tag, errWithCode := p.getTag(ctx, tagName)
+	if errWithCode != nil {
+		return nil, errWithCode
+	}
+
+	if tag == nil || !*tag.Useable || !*tag.Listable {
+		// Obey mastodon API by returning 404 for this.
+		err := fmt.Errorf("tag was not found, or not useable/listable on this instance")
+		return nil, gtserror.NewErrorNotFound(err, err.Error())
+	}
+
+	statuses, err := p.state.DB.GetTagTimeline(ctx, tag.ID, maxID, sinceID, minID, limit)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err = gtserror.Newf("db error getting statuses: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	return p.packageTagResponse(
+		ctx,
+		requestingAcct,
+		statuses,
+		limit,
+		// Use API URL for tag.
+		"/api/v1/timelines/tag/"+tagName,
+	)
+}
+
+func (p *Processor) getTag(ctx context.Context, tagName string) (*gtsmodel.Tag, gtserror.WithCode) {
+	// Normalize + validate tag name.
+	tagNameNormal, ok := text.NormalizeHashtag(tagName)
+	if !ok {
+		err := gtserror.Newf("string '%s' could not be normalized to a valid hashtag", tagName)
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	// Ensure we have tag with this name in the db.
+	tag, err := p.state.DB.GetTagByName(ctx, tagNameNormal)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		// Real db error.
+		err = gtserror.Newf("db error getting tag by name: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	return tag, nil
+}
+
+func (p *Processor) packageTagResponse(
+	ctx context.Context,
+	requestingAcct *gtsmodel.Account,
+	statuses []*gtsmodel.Status,
+	limit int,
+	requestPath string,
+) (*apimodel.PageableResponse, gtserror.WithCode) {
+	count := len(statuses)
+	if count == 0 {
+		return util.EmptyPageableResponse(), nil
+	}
+
+	var (
+		items = make([]interface{}, 0, count)
+
+		// Set next + prev values before filtering and API
+		// converting, so caller can still page properly.
+		nextMaxIDValue = statuses[count-1].ID
+		prevMinIDValue = statuses[0].ID
+	)
+
+	for _, s := range statuses {
+		timelineable, err := p.filter.StatusTagTimelineable(ctx, requestingAcct, s)
+		if err != nil {
+			log.Errorf(ctx, "error checking status visibility: %v", err)
+			continue
+		}
+
+		if !timelineable {
+			continue
+		}
+
+		apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, requestingAcct)
+		if err != nil {
+			log.Errorf(ctx, "error converting to api status: %v", err)
+			continue
+		}
+
+		items = append(items, apiStatus)
+	}
+
+	return util.PackagePageableResponse(util.PageableResponseParams{
+		Items:          items,
+		Path:           requestPath,
+		NextMaxIDValue: nextMaxIDValue,
+		PrevMinIDValue: prevMinIDValue,
+		Limit:          limit,
+	})
+}