diff options
Diffstat (limited to 'internal/ap')
-rw-r--r-- | internal/ap/ap_test.go | 7 | ||||
-rw-r--r-- | internal/ap/collections.go | 112 | ||||
-rw-r--r-- | internal/ap/collections_test.go | 148 | ||||
-rw-r--r-- | internal/ap/interfaces.go | 9 | ||||
-rw-r--r-- | internal/ap/resolve.go | 206 | ||||
-rw-r--r-- | internal/ap/resolve_test.go | 20 | ||||
-rw-r--r-- | internal/ap/util.go | 29 |
7 files changed, 445 insertions, 86 deletions
diff --git a/internal/ap/ap_test.go b/internal/ap/ap_test.go index 583a37c53..0a9f66ca6 100644 --- a/internal/ap/ap_test.go +++ b/internal/ap/ap_test.go @@ -18,8 +18,10 @@ package ap_test import ( + "bytes" "context" "encoding/json" + "io" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/pub" @@ -187,7 +189,10 @@ func (suite *APTestSuite) noteWithHashtags1() ap.Statusable { } }`) - statusable, err := ap.ResolveStatusable(context.Background(), noteJson) + statusable, err := ap.ResolveStatusable( + context.Background(), + io.NopCloser(bytes.NewReader(noteJson)), + ) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/ap/collections.go b/internal/ap/collections.go index ba3887a5b..da789e179 100644 --- a/internal/ap/collections.go +++ b/internal/ap/collections.go @@ -26,6 +26,24 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/paging" ) +// TODO: replace must of this logic with just +// using extractIRIs() on the iterator types. + +// ToCollectionIterator attempts to resolve the given vocab type as a Collection +// like object and wrap in a standardised interface in order to iterate its contents. +func ToCollectionIterator(t vocab.Type) (CollectionIterator, error) { + switch name := t.GetTypeName(); name { + case ObjectCollection: + t := t.(vocab.ActivityStreamsCollection) + return WrapCollection(t), nil + case ObjectOrderedCollection: + t := t.(vocab.ActivityStreamsOrderedCollection) + return WrapOrderedCollection(t), nil + default: + return nil, fmt.Errorf("%T(%s) was not Collection-like", t, name) + } +} + // ToCollectionPageIterator attempts to resolve the given vocab type as a CollectionPage // like object and wrap in a standardised interface in order to iterate its contents. func ToCollectionPageIterator(t vocab.Type) (CollectionPageIterator, error) { @@ -41,6 +59,16 @@ func ToCollectionPageIterator(t vocab.Type) (CollectionPageIterator, error) { } } +// WrapCollection wraps an ActivityStreamsCollection in a standardised collection interface. +func WrapCollection(collection vocab.ActivityStreamsCollection) CollectionIterator { + return ®ularCollectionIterator{ActivityStreamsCollection: collection} +} + +// WrapOrderedCollection wraps an ActivityStreamsOrderedCollection in a standardised collection interface. +func WrapOrderedCollection(collection vocab.ActivityStreamsOrderedCollection) CollectionIterator { + return &orderedCollectionIterator{ActivityStreamsOrderedCollection: collection} +} + // WrapCollectionPage wraps an ActivityStreamsCollectionPage in a standardised collection page interface. func WrapCollectionPage(page vocab.ActivityStreamsCollectionPage) CollectionPageIterator { return ®ularCollectionPageIterator{ActivityStreamsCollectionPage: page} @@ -51,6 +79,90 @@ func WrapOrderedCollectionPage(page vocab.ActivityStreamsOrderedCollectionPage) return &orderedCollectionPageIterator{ActivityStreamsOrderedCollectionPage: page} } +// regularCollectionIterator implements CollectionIterator +// for the vocab.ActivitiyStreamsCollection type. +type regularCollectionIterator struct { + vocab.ActivityStreamsCollection + items vocab.ActivityStreamsItemsPropertyIterator + once bool // only init items once +} + +func (iter *regularCollectionIterator) NextItem() TypeOrIRI { + if !iter.initItems() { + return nil + } + cur := iter.items + iter.items = iter.items.Next() + return cur +} + +func (iter *regularCollectionIterator) PrevItem() TypeOrIRI { + if !iter.initItems() { + return nil + } + cur := iter.items + iter.items = iter.items.Prev() + return cur +} + +func (iter *regularCollectionIterator) initItems() bool { + if iter.once { + return (iter.items != nil) + } + iter.once = true + if iter.ActivityStreamsCollection == nil { + return false // no page set + } + items := iter.GetActivityStreamsItems() + if items == nil { + return false // no items found + } + iter.items = items.Begin() + return (iter.items != nil) +} + +// orderedCollectionIterator implements CollectionIterator +// for the vocab.ActivitiyStreamsOrderedCollection type. +type orderedCollectionIterator struct { + vocab.ActivityStreamsOrderedCollection + items vocab.ActivityStreamsOrderedItemsPropertyIterator + once bool // only init items once +} + +func (iter *orderedCollectionIterator) NextItem() TypeOrIRI { + if !iter.initItems() { + return nil + } + cur := iter.items + iter.items = iter.items.Next() + return cur +} + +func (iter *orderedCollectionIterator) PrevItem() TypeOrIRI { + if !iter.initItems() { + return nil + } + cur := iter.items + iter.items = iter.items.Prev() + return cur +} + +func (iter *orderedCollectionIterator) initItems() bool { + if iter.once { + return (iter.items != nil) + } + iter.once = true + if iter.ActivityStreamsOrderedCollection == nil { + return false // no page set + } + items := iter.GetActivityStreamsOrderedItems() + if items == nil { + return false // no items found + } + iter.items = items.Begin() + return (iter.items != nil) +} + // regularCollectionPageIterator implements CollectionPageIterator // for the vocab.ActivitiyStreamsCollectionPage type. type regularCollectionPageIterator struct { diff --git a/internal/ap/collections_test.go b/internal/ap/collections_test.go new file mode 100644 index 000000000..87a5bb057 --- /dev/null +++ b/internal/ap/collections_test.go @@ -0,0 +1,148 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package ap_test + +import ( + "net/url" + "slices" + "testing" + + "github.com/superseriousbusiness/activity/pub" + "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" +) + +var testIteratorIRIs = [][]string{ + { + "https://google.com", + "https://mastodon.social", + "http://naughty.naughty.website/heres/the/porn", + "https://god.monarchies.suck?yes=they&really=do", + }, + { + // zero length + }, + { + "https://superseriousbusiness.org", + "http://gotosocial.tv/@slothsgonewild", + }, +} + +func TestToCollectionIterator(t *testing.T) { + for _, iris := range testIteratorIRIs { + testToCollectionIterator(t, toCollection(iris), "", iris) + testToCollectionIterator(t, toOrderedCollection(iris), "", iris) + } + testToCollectionIterator(t, streams.NewActivityStreamsAdd(), "*typeadd.ActivityStreamsAdd(Add) was not Collection-like", nil) + testToCollectionIterator(t, streams.NewActivityStreamsBlock(), "*typeblock.ActivityStreamsBlock(Block) was not Collection-like", nil) +} + +func TestToCollectionPageIterator(t *testing.T) { + for _, iris := range testIteratorIRIs { + testToCollectionPageIterator(t, toCollectionPage(iris), "", iris) + testToCollectionPageIterator(t, toOrderedCollectionPage(iris), "", iris) + } + testToCollectionPageIterator(t, streams.NewActivityStreamsAdd(), "*typeadd.ActivityStreamsAdd(Add) was not CollectionPage-like", nil) + testToCollectionPageIterator(t, streams.NewActivityStreamsBlock(), "*typeblock.ActivityStreamsBlock(Block) was not CollectionPage-like", nil) +} + +func testToCollectionIterator(t *testing.T, in vocab.Type, expectErr string, expectIRIs []string) { + collect, err := ap.ToCollectionIterator(in) + if !errCheck(err, expectErr) { + t.Fatalf("did not return expected error: expect=%v receive=%v", expectErr, err) + } + iris := gatherFromIterator(collect) + if !slices.Equal(iris, expectIRIs) { + t.Fatalf("did not return expected iris: expect=%v receive=%v", expectIRIs, iris) + } +} + +func testToCollectionPageIterator(t *testing.T, in vocab.Type, expectErr string, expectIRIs []string) { + page, err := ap.ToCollectionPageIterator(in) + if !errCheck(err, expectErr) { + t.Fatalf("did not return expected error: expect=%v receive=%v", expectErr, err) + } + iris := gatherFromIterator(page) + if !slices.Equal(iris, expectIRIs) { + t.Fatalf("did not return expected iris: expect=%v receive=%v", expectIRIs, iris) + } +} + +func toCollection(iris []string) vocab.ActivityStreamsCollection { + collect := streams.NewActivityStreamsCollection() + collect.SetActivityStreamsItems(toItems(iris)) + return collect +} + +func toOrderedCollection(iris []string) vocab.ActivityStreamsOrderedCollection { + collect := streams.NewActivityStreamsOrderedCollection() + collect.SetActivityStreamsOrderedItems(toOrderedItems(iris)) + return collect +} + +func toCollectionPage(iris []string) vocab.ActivityStreamsCollectionPage { + page := streams.NewActivityStreamsCollectionPage() + page.SetActivityStreamsItems(toItems(iris)) + return page +} + +func toOrderedCollectionPage(iris []string) vocab.ActivityStreamsOrderedCollectionPage { + page := streams.NewActivityStreamsOrderedCollectionPage() + page.SetActivityStreamsOrderedItems(toOrderedItems(iris)) + return page +} + +func toItems(iris []string) vocab.ActivityStreamsItemsProperty { + items := streams.NewActivityStreamsItemsProperty() + for _, iri := range iris { + u, _ := url.Parse(iri) + items.AppendIRI(u) + } + return items +} + +func toOrderedItems(iris []string) vocab.ActivityStreamsOrderedItemsProperty { + items := streams.NewActivityStreamsOrderedItemsProperty() + for _, iri := range iris { + u, _ := url.Parse(iri) + items.AppendIRI(u) + } + return items +} + +func gatherFromIterator(iter ap.CollectionIterator) []string { + var iris []string + if iter == nil { + return nil + } + for item := iter.NextItem(); item != nil; item = iter.NextItem() { + id, _ := pub.ToId(item) + if id != nil { + iris = append(iris, id.String()) + } + } + return iris +} + +func errCheck(err error, str string) bool { + if err == nil { + return str == "" + } + return err.Error() == str +} diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index 811e09125..fa8e8a338 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -300,6 +300,15 @@ type ReplyToable interface { WithInReplyTo } +// CollectionIterator represents the minimum interface for interacting with a +// wrapped Collection or OrderedCollection in order to access next / prev items. +type CollectionIterator interface { + vocab.Type + + NextItem() TypeOrIRI + PrevItem() TypeOrIRI +} + // CollectionPageIterator represents the minimum interface for interacting with a wrapped // CollectionPage or OrderedCollectionPage in order to access both next / prev pages and items. type CollectionPageIterator interface { diff --git a/internal/ap/resolve.go b/internal/ap/resolve.go index 20a858900..b2e866b6f 100644 --- a/internal/ap/resolve.go +++ b/internal/ap/resolve.go @@ -22,8 +22,8 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" - "sync" "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" @@ -31,61 +31,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) -// mapPool is a memory pool of maps for JSON decoding. -var mapPool = sync.Pool{ - New: func() any { - return make(map[string]any) - }, -} - -// getMap acquires a map from memory pool. -func getMap() map[string]any { - m := mapPool.Get().(map[string]any) //nolint - return m -} - -// putMap clears and places map back in pool. -func putMap(m map[string]any) { - if len(m) > int(^uint8(0)) { - // don't pool overly - // large maps. - return - } - for k := range m { - delete(m, k) - } - mapPool.Put(m) -} - -// bytesToType tries to parse the given bytes slice -// as a JSON ActivityPub type, failing if the input -// bytes are not parseable as JSON, or do not parse -// to an ActivityPub that we can understand. -// -// The given map pointer will also be populated with -// the parsed JSON, to allow further processing. -func bytesToType( - ctx context.Context, - b []byte, - raw map[string]any, -) (vocab.Type, error) { - // Unmarshal the raw JSON bytes into a "raw" map. - // This will fail if the input is not parseable - // as JSON; eg., a remote has returned HTML as a - // fallback response to an ActivityPub JSON request. - if err := json.Unmarshal(b, &raw); err != nil { - return nil, gtserror.NewfAt(3, "error unmarshalling bytes into json: %w", err) - } - - // Resolve an ActivityStreams type. - t, err := streams.ToType(ctx, raw) - if err != nil { - return nil, gtserror.NewfAt(3, "error resolving json into ap vocab type: %w", err) - } - - return t, nil -} - // ResolveActivity is a util function for pulling a pub.Activity type out of an incoming request body, // returning the resolved activity type, error and whether to accept activity (false = transient i.e. ignore). func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.WithCode) { @@ -93,25 +38,23 @@ func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.With // destination. raw := getMap() - // Tidy up when done. - defer r.Body.Close() + // Decode data as JSON into 'raw' map + // and get the resolved AS vocab.Type. + // (this handles close of request body). + t, err := decodeType(r.Context(), r.Body, raw) - // Decode the JSON body stream into "raw" map. - if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { - err := gtserror.Newf("error decoding json: %w", err) - return nil, false, gtserror.NewErrorInternalError(err) - } - - // Resolve "raw" JSON to vocab.Type. - t, err := streams.ToType(r.Context(), raw) if err != nil { + // NOTE: if the error here was due to the response body + // ending early, the connection will have broken so it + // doesn't matter if we try to return 400 or 500, the + // error is mainly for our logging. tl;dr there's not a + // huge need to differentiate between those error types. + if !streams.IsUnmatchedErr(err) { err := gtserror.Newf("error matching json to type: %w", err) return nil, false, gtserror.NewErrorInternalError(err) } - // Respond with bad request; we just couldn't - // match the type to one that we know about. const text = "body json not resolvable as ActivityStreams type" return nil, false, gtserror.NewErrorBadRequest(errors.New(text), text) } @@ -142,18 +85,19 @@ func ResolveIncomingActivity(r *http.Request) (pub.Activity, bool, gtserror.With return activity, true, nil } -// ResolveStatusable tries to resolve the given bytes into an ActivityPub Statusable representation. -// It will then perform normalization on the Statusable. +// ResolveStatusable tries to resolve the response data as an ActivityPub +// Statusable representation. It will then perform normalization on the Statusable. // // Works for: Article, Document, Image, Video, Note, Page, Event, Place, Profile, Question. -func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) { +func ResolveStatusable(ctx context.Context, body io.ReadCloser) (Statusable, error) { // Get "raw" map // destination. raw := getMap() - // Convert raw bytes to an AP type. - // This will also populate the map. - t, err := bytesToType(ctx, b, raw) + // Decode data as JSON into 'raw' map + // and get the resolved AS vocab.Type. + // (this handles close of given body). + t, err := decodeType(ctx, body, raw) if err != nil { return nil, gtserror.SetWrongType(err) } @@ -183,18 +127,19 @@ func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) { return statusable, nil } -// ResolveStatusable tries to resolve the given bytes into an ActivityPub Accountable representation. -// It will then perform normalization on the Accountable. +// ResolveAccountable tries to resolve the given reader into an ActivityPub +// Accountable representation. It will then perform normalization on the Accountable. // // Works for: Application, Group, Organization, Person, Service -func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) { +func ResolveAccountable(ctx context.Context, body io.ReadCloser) (Accountable, error) { // Get "raw" map // destination. raw := getMap() - // Convert raw bytes to an AP type. - // This will also populate the map. - t, err := bytesToType(ctx, b, raw) + // Decode data as JSON into 'raw' map + // and get the resolved AS vocab.Type. + // (this handles close of given body). + t, err := decodeType(ctx, body, raw) if err != nil { return nil, gtserror.SetWrongType(err) } @@ -213,3 +158,104 @@ func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) { return accountable, nil } + +// ResolveCollection tries to resolve the given reader into an ActivityPub Collection-like +// representation, then wrapping as abstracted iterator. Works for: Collection, OrderedCollection. +func ResolveCollection(ctx context.Context, body io.ReadCloser) (CollectionIterator, error) { + // Get "raw" map + // destination. + raw := getMap() + + // Decode data as JSON into 'raw' map + // and get the resolved AS vocab.Type. + // (this handles close of given body). + t, err := decodeType(ctx, body, raw) + if err != nil { + return nil, gtserror.SetWrongType(err) + } + + // Release. + putMap(raw) + + // Cast as as Collection-like. + return ToCollectionIterator(t) +} + +// ResolveCollectionPage tries to resolve the given reader into an ActivityPub CollectionPage-like +// representation, then wrapping as abstracted iterator. Works for: CollectionPage, OrderedCollectionPage. +func ResolveCollectionPage(ctx context.Context, body io.ReadCloser) (CollectionPageIterator, error) { + // Get "raw" map + // destination. + raw := getMap() + + // Decode data as JSON into 'raw' map + // and get the resolved AS vocab.Type. + // (this handles close of given body). + t, err := decodeType(ctx, body, raw) + if err != nil { + return nil, gtserror.SetWrongType(err) + } + + // Release. + putMap(raw) + + // Cast as as CollectionPage-like. + return ToCollectionPageIterator(t) +} + +// emptydest is an empty JSON decode +// destination useful for "noop" decodes +// to check underlying reader is empty. +var emptydest = &struct{}{} + +// decodeType tries to read and parse the data +// at provided io.ReadCloser as a JSON ActivityPub +// type, failing if not parseable as JSON or not +// resolveable as one of our known AS types. +// +// NOTE: this function handles closing +// given body when it is finished with. +// +// The given map pointer will also be populated with +// the 'raw' JSON data, for further processing. +func decodeType( + ctx context.Context, + body io.ReadCloser, + raw map[string]any, +) (vocab.Type, error) { + + // Wrap body in JSON decoder. + // + // We do this instead of using json.Unmarshal() + // so we can take advantage of the decoder's streamed + // check of input data as valid JSON. This means that + // in the cases of garbage input, or even just fallback + // HTML responses that were incorrectly content-type'd, + // we can error-out as soon as possible. + dec := json.NewDecoder(body) + + // Unmarshal JSON source data into "raw" map. + if err := dec.Decode(&raw); err != nil { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "error decoding into json: %w", err) + } + + // Perform a secondary decode just to ensure we drained the + // entirety of the data source. Error indicates either extra + // trailing garbage, or multiple JSON values (invalid data). + if err := dec.Decode(emptydest); err != io.EOF { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "data remaining after json") + } + + // Done with body. + _ = body.Close() + + // Resolve an ActivityStreams type. + t, err := streams.ToType(ctx, raw) + if err != nil { + return nil, gtserror.NewfAt(3, "error resolving json into ap vocab type: %w", err) + } + + return t, nil +} diff --git a/internal/ap/resolve_test.go b/internal/ap/resolve_test.go index bc32af7e4..aaf90ab0a 100644 --- a/internal/ap/resolve_test.go +++ b/internal/ap/resolve_test.go @@ -18,7 +18,9 @@ package ap_test import ( + "bytes" "context" + "io" "testing" "github.com/stretchr/testify/suite" @@ -33,7 +35,9 @@ type ResolveTestSuite struct { func (suite *ResolveTestSuite) TestResolveDocumentAsStatusable() { b := []byte(suite.typeToJson(suite.document1)) - statusable, err := ap.ResolveStatusable(context.Background(), b) + statusable, err := ap.ResolveStatusable( + context.Background(), io.NopCloser(bytes.NewReader(b)), + ) suite.NoError(err) suite.NotNil(statusable) } @@ -41,7 +45,9 @@ func (suite *ResolveTestSuite) TestResolveDocumentAsStatusable() { func (suite *ResolveTestSuite) TestResolveDocumentAsAccountable() { b := []byte(suite.typeToJson(suite.document1)) - accountable, err := ap.ResolveAccountable(context.Background(), b) + accountable, err := ap.ResolveAccountable( + context.Background(), io.NopCloser(bytes.NewReader(b)), + ) suite.True(gtserror.IsWrongType(err)) suite.EqualError(err, "ResolveAccountable: cannot resolve vocab type *typedocument.ActivityStreamsDocument as accountable") suite.Nil(accountable) @@ -51,9 +57,11 @@ func (suite *ResolveTestSuite) TestResolveHTMLAsAccountable() { b := []byte(`<!DOCTYPE html> <title>.</title>`) - accountable, err := ap.ResolveAccountable(context.Background(), b) + accountable, err := ap.ResolveAccountable( + context.Background(), io.NopCloser(bytes.NewReader(b)), + ) suite.True(gtserror.IsWrongType(err)) - suite.EqualError(err, "ResolveAccountable: error unmarshalling bytes into json: invalid character '<' looking for beginning of value") + suite.EqualError(err, "ResolveAccountable: error decoding into json: invalid character '<' looking for beginning of value") suite.Nil(accountable) } @@ -64,7 +72,9 @@ func (suite *ResolveTestSuite) TestResolveNonAPJSONAsAccountable() { "pee pee":"poo poo" }`) - accountable, err := ap.ResolveAccountable(context.Background(), b) + accountable, err := ap.ResolveAccountable( + context.Background(), io.NopCloser(bytes.NewReader(b)), + ) suite.True(gtserror.IsWrongType(err)) suite.EqualError(err, "ResolveAccountable: error resolving json into ap vocab type: activity stream did not match any known types") suite.Nil(accountable) diff --git a/internal/ap/util.go b/internal/ap/util.go index c810b7985..967a1659d 100644 --- a/internal/ap/util.go +++ b/internal/ap/util.go @@ -19,10 +19,39 @@ package ap import ( "net/url" + "sync" "github.com/superseriousbusiness/activity/streams/vocab" ) +const mapmax = 256 + +// mapPool is a memory pool +// of maps for JSON decoding. +var mapPool sync.Pool + +// getMap acquires a map from memory pool. +func getMap() map[string]any { + v := mapPool.Get() + if v == nil { + // preallocate map of max-size. + m := make(map[string]any, mapmax) + v = m + } + return v.(map[string]any) //nolint +} + +// putMap clears and places map back in pool. +func putMap(m map[string]any) { + if len(m) > mapmax { + // drop maps beyond + // our maximum size. + return + } + clear(m) + mapPool.Put(m) +} + // _TypeOrIRI wraps a vocab.Type to implement TypeOrIRI. type _TypeOrIRI struct { vocab.Type |