[bugfix] 2643 bug search for account url doesnt always work when redirected (#2673)

* update activity library so dereferencer returns full response and checks *final* link to allow for redirects

* temporarily add bodged fixed library

* remove unused code

* update getAccountFeatured() to use dereferenceCollectionPage()

* make sure to release map

* perform a 2nd decode to ensure reader is empty after primary decode

* add comment explaining choice of using Decode() instead of Unmarshal()

* update embedded activity library to latest matching https://github.com/superseriousbusiness/activity/pull/21

* add checks to look for changed URI and re-check database if redirected

* update max iteration count to 512, add checks during dereferenceAncestors() for indirect URLs

* remove doubled-up code

* fix use of status instead of current

* use URIs for checking equality for security

* use the latest known URI for boost_of_uri in case original was an indirect

* add dereferenceCollection() function for dereferenceAccountFeatured()

* pull in latest github.com/superseriousbusiness/activity version (and remove the bodge!!)

* fix typo in code comments

* update decodeType() to accept a readcloser and handle body closing

* switch to checking using BoostOfID and add note why not using BoostOfURI

* ensure InReplyTo gets unset when deleting status parent in case currently stubbed

* add tests for Collection and CollectionPage iterators

3 files changed, 55 insertions, 12 deletions

diff --git a/internal/transport/controller.go b/internal/transport/controller.go
index 32ef6d7c2..81022596a 100644
--- a/internal/transport/controller.go
+++ b/internal/transport/controller.go
@@ -18,11 +18,15 @@
 package transport
 
 import (
+	"bytes"
 	"context"
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"net/http"
 	"net/url"
 	"runtime"
 
@@ -31,6 +35,7 @@ import (
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/httpclient"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
@@ -150,36 +155,73 @@ func (c *controller) NewTransportForUsername(ctx context.Context, username strin
 // account on this instance, without making any external api/http calls.
 //
 // It is passed to new transports, and should only be invoked when the iri.Host == this host.
-func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) (*http.Response, error) {
 	followers, err := c.fedDB.Followers(ctx, iri)
-	if err != nil {
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return nil, err
 	}
 
+	if followers == nil {
+		// Return a generic 404 not found response.
+		rsp := craftResponse(iri, http.StatusNotFound)
+		return rsp, nil
+	}
+
 	i, err := ap.Serialize(followers)
 	if err != nil {
 		return nil, err
 	}
 
-	return json.Marshal(i)
+	b, err := json.Marshal(i)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return a response with AS data as body.
+	rsp := craftResponse(iri, http.StatusOK)
+	rsp.Body = io.NopCloser(bytes.NewReader(b))
+	return rsp, nil
 }
 
 // dereferenceLocalUser is a shortcut to dereference followers an account on
 // this instance, without making any external api/http calls.
 //
 // It is passed to new transports, and should only be invoked when the iri.Host == this host.
-func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) (*http.Response, error) {
 	user, err := c.fedDB.Get(ctx, iri)
-	if err != nil {
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return nil, err
 	}
 
+	if user == nil {
+		// Return a generic 404 not found response.
+		rsp := craftResponse(iri, http.StatusNotFound)
+		return rsp, nil
+	}
+
 	i, err := ap.Serialize(user)
 	if err != nil {
 		return nil, err
 	}
 
-	return json.Marshal(i)
+	b, err := json.Marshal(i)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return a response with AS data as body.
+	rsp := craftResponse(iri, http.StatusOK)
+	rsp.Body = io.NopCloser(bytes.NewReader(b))
+	return rsp, nil
+}
+
+func craftResponse(url *url.URL, code int) *http.Response {
+	rsp := new(http.Response)
+	rsp.Request = new(http.Request)
+	rsp.Request.URL = url
+	rsp.Status = http.StatusText(code)
+	rsp.StatusCode = code
+	return rsp
 }
 
 // privkeyToPublicStr will create a string representation of RSA public key from private.
diff --git a/internal/transport/dereference.go b/internal/transport/dereference.go
index 3a33a81ad..efd3f0fbf 100644
--- a/internal/transport/dereference.go
+++ b/internal/transport/dereference.go
@@ -19,7 +19,6 @@ package transport
 
 import (
 	"context"
-	"io"
 	"net/http"
 	"net/url"
 
@@ -29,7 +28,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
-func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) {
+func (t *transport) Dereference(ctx context.Context, iri *url.URL) (*http.Response, error) {
 	// if the request is to us, we can shortcut for certain URIs rather than going through
 	// the normal request flow, thereby saving time and energy
 	if iri.Host == config.GetHost() {
@@ -62,18 +61,20 @@ func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, erro
 	if err != nil {
 		return nil, err
 	}
-	defer rsp.Body.Close()
 
 	// Ensure a non-error status response.
 	if rsp.StatusCode != http.StatusOK {
-		return nil, gtserror.NewFromResponse(rsp)
+		err := gtserror.NewFromResponse(rsp)
+		_ = rsp.Body.Close() // done with body
+		return nil, err
 	}
 
 	// Ensure that the incoming request content-type is expected.
 	if ct := rsp.Header.Get("Content-Type"); !apiutil.ASContentType(ct) {
 		err := gtserror.Newf("non activity streams response: %s", ct)
+		_ = rsp.Body.Close() // done with body
 		return nil, gtserror.SetMalformed(err)
 	}
 
-	return io.ReadAll(rsp.Body)
+	return rsp, nil
 }
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index a2a9dc23c..99972fe25 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -65,7 +65,7 @@ type Transport interface {
 	GET(*http.Request) (*http.Response, error)
 
 	// Dereference fetches the ActivityStreams object located at this IRI with a GET request.
-	Dereference(ctx context.Context, iri *url.URL) ([]byte, error)
+	Dereference(ctx context.Context, iri *url.URL) (*http.Response, error)
 
 	// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
 	DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error)