[bugfix] 2643 bug search for account url doesnt always work when redirected (#2673)

* update activity library so dereferencer returns full response and checks *final* link to allow for redirects

* temporarily add bodged fixed library

* remove unused code

* update getAccountFeatured() to use dereferenceCollectionPage()

* make sure to release map

* perform a 2nd decode to ensure reader is empty after primary decode

* add comment explaining choice of using Decode() instead of Unmarshal()

* update embedded activity library to latest matching https://github.com/superseriousbusiness/activity/pull/21

* add checks to look for changed URI and re-check database if redirected

* update max iteration count to 512, add checks during dereferenceAncestors() for indirect URLs

* remove doubled-up code

* fix use of status instead of current

* use URIs for checking equality for security

* use the latest known URI for boost_of_uri in case original was an indirect

* add dereferenceCollection() function for dereferenceAccountFeatured()

* pull in latest github.com/superseriousbusiness/activity version (and remove the bodge!!)

* fix typo in code comments

* update decodeType() to accept a readcloser and handle body closing

* switch to checking using BoostOfID and add note why not using BoostOfURI

* ensure InReplyTo gets unset when deleting status parent in case currently stubbed

* add tests for Collection and CollectionPage iterators

1 files changed, 154 insertions, 0 deletions

diff --git a/internal/federation/dereferencing/collection.go b/internal/federation/dereferencing/collection.go
new file mode 100644
index 000000000..07f56c952
--- /dev/null
+++ b/internal/federation/dereferencing/collection.go
@@ -0,0 +1,154 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dereferencing
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+)
+
+// dereferenceCollectionPage returns the activitystreams Collection at the specified IRI, or an error if something goes wrong.
+func (d *Dereferencer) dereferenceCollection(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionIterator, error) {
+	if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
+		return nil, gtserror.Newf("domain %s is blocked", pageIRI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(ctx, username)
+	if err != nil {
+		return nil, gtserror.Newf("error creating transport: %w", err)
+	}
+
+	rsp, err := transport.Dereference(ctx, pageIRI)
+	if err != nil {
+		return nil, gtserror.Newf("error deferencing %s: %w", pageIRI.String(), err)
+	}
+
+	collect, err := ap.ResolveCollection(ctx, rsp.Body)
+
+	// Tidy up rsp body.
+	_ = rsp.Body.Close()
+
+	if err != nil {
+		return nil, gtserror.Newf("error resolving collection %s: %w", pageIRI.String(), err)
+	}
+
+	return collect, nil
+}
+
+// dereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
+func (d *Dereferencer) dereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageIterator, error) {
+	if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
+		return nil, gtserror.Newf("domain %s is blocked", pageIRI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(ctx, username)
+	if err != nil {
+		return nil, gtserror.Newf("error creating transport: %w", err)
+	}
+
+	rsp, err := transport.Dereference(ctx, pageIRI)
+	if err != nil {
+		return nil, gtserror.Newf("error deferencing %s: %w", pageIRI.String(), err)
+	}
+
+	page, err := ap.ResolveCollectionPage(ctx, rsp.Body)
+
+	// Tidy up rsp body.
+	_ = rsp.Body.Close()
+
+	if err != nil {
+		return nil, gtserror.Newf("error resolving collection page %s: %w", pageIRI.String(), err)
+	}
+
+	return page, nil
+}
+
+// getAttachedStatusCollection is a small utility function to fetch the first page of an
+// attached activity streams collection from a provided statusable object, along with a URI.
+func getAttachedStatusCollectionPage(status ap.Statusable) (ap.CollectionPageIterator, string) { //nolint:gocritic
+	// Look for an attached status replies (as collection)
+	replies := status.GetActivityStreamsReplies()
+	if replies == nil {
+		return nil, ""
+	}
+
+	// Look for an attached collection page, wrap and return.
+	if page := getRepliesCollectionPage(replies); page != nil {
+		return ap.WrapCollectionPage(page), getIDString(page)
+	}
+
+	// Look for an attached ordered collection page, wrap and return.
+	if page := getRepliesOrderedCollectionPage(replies); page != nil {
+		return ap.WrapOrderedCollectionPage(page), getIDString(page)
+	}
+
+	log.Warnf(nil, "replies without collection page: %s", getIDString(status))
+	return nil, ""
+}
+
+func getRepliesCollectionPage(replies vocab.ActivityStreamsRepliesProperty) vocab.ActivityStreamsCollectionPage {
+	// Get the status replies collection
+	collection := replies.GetActivityStreamsCollection()
+	if collection == nil {
+		return nil
+	}
+
+	// Get the "first" property of the replies collection
+	first := collection.GetActivityStreamsFirst()
+	if first == nil {
+		return nil
+	}
+
+	// Return the first activity stream collection page
+	return first.GetActivityStreamsCollectionPage()
+}
+
+func getRepliesOrderedCollectionPage(replies vocab.ActivityStreamsRepliesProperty) vocab.ActivityStreamsOrderedCollectionPage {
+	// Get the status replies collection
+	collection := replies.GetActivityStreamsOrderedCollection()
+	if collection == nil {
+		return nil
+	}
+
+	// Get the "first" property of the replies collection
+	first := collection.GetActivityStreamsFirst()
+	if first == nil {
+		return nil
+	}
+
+	// Return the first activity stream collection page
+	return first.GetActivityStreamsOrderedCollectionPage()
+}
+
+// getIDString is shorthand to fetch an ID URI string from AP type with attached JSONLDId.
+func getIDString(a ap.WithJSONLDId) string {
+	id := a.GetJSONLDId()
+	if id == nil {
+		return ""
+	}
+	uri := id.Get()
+	if uri == nil {
+		return ""
+	}
+	return uri.String()
+}