From 0f2de6394a1c52d47e326bb7d7d129a217ae4f6f Mon Sep 17 00:00:00 2001 From: Tobi Smethurst <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 10 Aug 2021 13:32:39 +0200 Subject: Dereference remote replies (#132) * decided where to put reply dereferencing * fiddling with dereferencing threads * further adventures * tidy up some stuff * move dereferencing functionality * a bunch of refactoring * go fmt * more refactoring * bleep bloop * docs and linting * start implementing replies collection on gts side * fiddling around * allow dereferencing our replies * lint, fmt --- internal/ap/extract.go | 614 +++++++++++++++++++++ internal/ap/interfaces.go | 321 +++++++++++ internal/api/client/account/accountupdate_test.go | 6 +- internal/api/client/fileserver/servefile_test.go | 4 +- internal/api/client/media/mediacreate_test.go | 4 +- internal/api/client/status/statusboost_test.go | 4 +- internal/api/client/status/statuscreate_test.go | 4 +- internal/api/client/status/statusfave_test.go | 4 +- internal/api/client/status/statusfavedby_test.go | 4 +- internal/api/client/status/statusget_test.go | 4 +- internal/api/client/status/statusunfave_test.go | 4 +- internal/api/s2s/user/repliesget.go | 186 +++++++ internal/api/s2s/user/repliesget_test.go | 241 ++++++++ internal/api/s2s/user/user.go | 10 + internal/api/s2s/user/user_test.go | 16 +- internal/api/s2s/user/userget_test.go | 66 +-- internal/cliactions/server/server.go | 2 +- internal/cliactions/testrig/testrig.go | 4 +- internal/db/db.go | 8 +- internal/db/pg/statuscontext.go | 31 +- internal/federation/authenticate.go | 26 +- internal/federation/dereference.go | 518 +---------------- internal/federation/dereferencing/account.go | 243 ++++++++ internal/federation/dereferencing/announce.go | 65 +++ internal/federation/dereferencing/blocked.go | 41 ++ .../federation/dereferencing/collectionpage.go | 70 +++ internal/federation/dereferencing/dereferencer.go | 73 +++ internal/federation/dereferencing/handshake.go | 98 ++++ internal/federation/dereferencing/instance.go | 40 ++ internal/federation/dereferencing/status.go | 369 +++++++++++++ internal/federation/dereferencing/thread.go | 250 +++++++++ internal/federation/federatingdb/update.go | 4 +- internal/federation/federatingprotocol.go | 51 +- internal/federation/federator.go | 41 +- internal/federation/federator_test.go | 6 +- internal/federation/finger.go | 2 +- internal/federation/handshake.go | 75 +-- internal/federation/transport.go | 32 +- internal/gtsmodel/activitystreams.go | 4 + internal/gtsmodel/status.go | 4 +- internal/processing/account/get.go | 9 - internal/processing/account/getfollowers.go | 6 - internal/processing/account/getfollowing.go | 6 - internal/processing/federation.go | 199 +++++-- internal/processing/fromfederator.go | 39 +- internal/processing/processor.go | 4 + internal/processing/search.go | 114 +--- internal/processing/status/context.go | 4 +- internal/text/link_test.go | 2 +- internal/text/plain_test.go | 2 +- internal/transport/controller.go | 35 +- internal/typeutils/asextractionutil.go | 570 ------------------- internal/typeutils/asinterfaces.go | 265 --------- internal/typeutils/astointernal.go | 83 +-- internal/typeutils/astointernal_test.go | 9 +- internal/typeutils/converter.go | 22 +- internal/typeutils/converter_test.go | 3 +- internal/typeutils/internaltoas.go | 145 ++++- internal/typeutils/internaltoas_test.go | 2 +- 59 files changed, 3175 insertions(+), 1893 deletions(-) create mode 100644 internal/ap/extract.go create mode 100644 internal/ap/interfaces.go create mode 100644 internal/api/s2s/user/repliesget.go create mode 100644 internal/api/s2s/user/repliesget_test.go create mode 100644 internal/federation/dereferencing/account.go create mode 100644 internal/federation/dereferencing/announce.go create mode 100644 internal/federation/dereferencing/blocked.go create mode 100644 internal/federation/dereferencing/collectionpage.go create mode 100644 internal/federation/dereferencing/dereferencer.go create mode 100644 internal/federation/dereferencing/handshake.go create mode 100644 internal/federation/dereferencing/instance.go create mode 100644 internal/federation/dereferencing/status.go create mode 100644 internal/federation/dereferencing/thread.go delete mode 100644 internal/typeutils/asextractionutil.go delete mode 100644 internal/typeutils/asinterfaces.go (limited to 'internal') diff --git a/internal/ap/extract.go b/internal/ap/extract.go new file mode 100644 index 000000000..baffd4bf2 --- /dev/null +++ b/internal/ap/extract.go @@ -0,0 +1,614 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +// Package ap contains models and utilities for working with activitypub/activitystreams representations. +// +// It is built on top of go-fed/activity. +package ap + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/go-fed/activity/pub" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// ExtractPreferredUsername returns a string representation of an interface's preferredUsername property. +func ExtractPreferredUsername(i WithPreferredUsername) (string, error) { + u := i.GetActivityStreamsPreferredUsername() + if u == nil || !u.IsXMLSchemaString() { + return "", errors.New("preferredUsername was not a string") + } + if u.GetXMLSchemaString() == "" { + return "", errors.New("preferredUsername was empty") + } + return u.GetXMLSchemaString(), nil +} + +// ExtractName returns a string representation of an interface's name property. +func ExtractName(i WithName) (string, error) { + nameProp := i.GetActivityStreamsName() + if nameProp == nil { + return "", errors.New("activityStreamsName not found") + } + + // take the first name string we can find + for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() { + if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { + return iter.GetXMLSchemaString(), nil + } + } + + return "", errors.New("activityStreamsName not found") +} + +// ExtractInReplyToURI extracts the inReplyToURI property (if present) from an interface. +func ExtractInReplyToURI(i WithInReplyTo) *url.URL { + inReplyToProp := i.GetActivityStreamsInReplyTo() + if inReplyToProp == nil { + // the property just wasn't set + return nil + } + for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() { + if iter.IsIRI() { + if iter.GetIRI() != nil { + return iter.GetIRI() + } + } + } + // couldn't find a URI + return nil +} + +// ExtractURLItems extracts a slice of URLs from a property that has withItems. +func ExtractURLItems(i WithItems) []*url.URL { + urls := []*url.URL{} + items := i.GetActivityStreamsItems() + if items == nil || items.Len() == 0 { + return urls + } + + for iter := items.Begin(); iter != items.End(); iter = iter.Next() { + if iter.IsIRI() { + urls = append(urls, iter.GetIRI()) + } + } + return urls +} + +// ExtractTos returns a list of URIs that the activity addresses as To. +func ExtractTos(i WithTo) ([]*url.URL, error) { + to := []*url.URL{} + toProp := i.GetActivityStreamsTo() + if toProp == nil { + return nil, errors.New("toProp was nil") + } + for iter := toProp.Begin(); iter != toProp.End(); iter = iter.Next() { + if iter.IsIRI() { + if iter.GetIRI() != nil { + to = append(to, iter.GetIRI()) + } + } + } + return to, nil +} + +// ExtractCCs returns a list of URIs that the activity addresses as CC. +func ExtractCCs(i WithCC) ([]*url.URL, error) { + cc := []*url.URL{} + ccProp := i.GetActivityStreamsCc() + if ccProp == nil { + return cc, nil + } + for iter := ccProp.Begin(); iter != ccProp.End(); iter = iter.Next() { + if iter.IsIRI() { + if iter.GetIRI() != nil { + cc = append(cc, iter.GetIRI()) + } + } + } + return cc, nil +} + +// ExtractAttributedTo returns the URL of the actor that the withAttributedTo is attributed to. +func ExtractAttributedTo(i WithAttributedTo) (*url.URL, error) { + attributedToProp := i.GetActivityStreamsAttributedTo() + if attributedToProp == nil { + return nil, errors.New("attributedToProp was nil") + } + for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() { + if iter.IsIRI() { + if iter.GetIRI() != nil { + return iter.GetIRI(), nil + } + } + } + return nil, errors.New("couldn't find iri for attributed to") +} + +// ExtractPublished extracts the publication time of an activity. +func ExtractPublished(i WithPublished) (time.Time, error) { + publishedProp := i.GetActivityStreamsPublished() + if publishedProp == nil { + return time.Time{}, errors.New("published prop was nil") + } + + if !publishedProp.IsXMLSchemaDateTime() { + return time.Time{}, errors.New("published prop was not date time") + } + + t := publishedProp.Get() + if t.IsZero() { + return time.Time{}, errors.New("published time was zero") + } + return t, nil +} + +// ExtractIconURL extracts a URL to a supported image file from something like: +// "icon": { +// "mediaType": "image/jpeg", +// "type": "Image", +// "url": "http://example.org/path/to/some/file.jpeg" +// }, +func ExtractIconURL(i WithIcon) (*url.URL, error) { + iconProp := i.GetActivityStreamsIcon() + if iconProp == nil { + return nil, errors.New("icon property was nil") + } + + // icon can potentially contain multiple entries, so we iterate through all of them + // here in order to find the first one that meets these criteria: + // 1. is an image + // 2. has a URL so we can grab it + for iter := iconProp.Begin(); iter != iconProp.End(); iter = iter.Next() { + // 1. is an image + if !iter.IsActivityStreamsImage() { + continue + } + imageValue := iter.GetActivityStreamsImage() + if imageValue == nil { + continue + } + + // 2. has a URL so we can grab it + url, err := ExtractURL(imageValue) + if err == nil && url != nil { + return url, nil + } + } + // if we get to this point we didn't find an icon meeting our criteria :'( + return nil, errors.New("could not extract valid image from icon") +} + +// ExtractImageURL extracts a URL to a supported image file from something like: +// "image": { +// "mediaType": "image/jpeg", +// "type": "Image", +// "url": "http://example.org/path/to/some/file.jpeg" +// }, +func ExtractImageURL(i WithImage) (*url.URL, error) { + imageProp := i.GetActivityStreamsImage() + if imageProp == nil { + return nil, errors.New("icon property was nil") + } + + // icon can potentially contain multiple entries, so we iterate through all of them + // here in order to find the first one that meets these criteria: + // 1. is an image + // 2. has a URL so we can grab it + for iter := imageProp.Begin(); iter != imageProp.End(); iter = iter.Next() { + // 1. is an image + if !iter.IsActivityStreamsImage() { + continue + } + imageValue := iter.GetActivityStreamsImage() + if imageValue == nil { + continue + } + + // 2. has a URL so we can grab it + url, err := ExtractURL(imageValue) + if err == nil && url != nil { + return url, nil + } + } + // if we get to this point we didn't find an image meeting our criteria :'( + return nil, errors.New("could not extract valid image from image property") +} + +// ExtractSummary extracts the summary/content warning of an interface. +func ExtractSummary(i WithSummary) (string, error) { + summaryProp := i.GetActivityStreamsSummary() + if summaryProp == nil { + return "", errors.New("summary property was nil") + } + + for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() { + if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { + return iter.GetXMLSchemaString(), nil + } + } + + return "", errors.New("could not extract summary") +} + +// ExtractDiscoverable extracts the Discoverable boolean of an interface. +func ExtractDiscoverable(i WithDiscoverable) (bool, error) { + if i.GetTootDiscoverable() == nil { + return false, errors.New("discoverable was nil") + } + return i.GetTootDiscoverable().Get(), nil +} + +// ExtractURL extracts the URL property of an interface. +func ExtractURL(i WithURL) (*url.URL, error) { + urlProp := i.GetActivityStreamsUrl() + if urlProp == nil { + return nil, errors.New("url property was nil") + } + + for iter := urlProp.Begin(); iter != urlProp.End(); iter = iter.Next() { + if iter.IsIRI() && iter.GetIRI() != nil { + return iter.GetIRI(), nil + } + } + + return nil, errors.New("could not extract url") +} + +// ExtractPublicKeyForOwner extracts the public key from an interface, as long as it belongs to the specified owner. +// It will return the public key itself, the id/URL of the public key, or an error if something goes wrong. +func ExtractPublicKeyForOwner(i WithPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { + publicKeyProp := i.GetW3IDSecurityV1PublicKey() + if publicKeyProp == nil { + return nil, nil, errors.New("public key property was nil") + } + + for iter := publicKeyProp.Begin(); iter != publicKeyProp.End(); iter = iter.Next() { + pkey := iter.Get() + if pkey == nil { + continue + } + + pkeyID, err := pub.GetId(pkey) + if err != nil || pkeyID == nil { + continue + } + + if pkey.GetW3IDSecurityV1Owner() == nil || pkey.GetW3IDSecurityV1Owner().Get() == nil || pkey.GetW3IDSecurityV1Owner().Get().String() != forOwner.String() { + continue + } + + if pkey.GetW3IDSecurityV1PublicKeyPem() == nil { + continue + } + + pkeyPem := pkey.GetW3IDSecurityV1PublicKeyPem().Get() + if pkeyPem == "" { + continue + } + + block, _ := pem.Decode([]byte(pkeyPem)) + if block == nil || block.Type != "PUBLIC KEY" { + return nil, nil, errors.New("could not decode publicKeyPem to PUBLIC KEY pem block type") + } + + p, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("could not parse public key from block bytes: %s", err) + } + if p == nil { + return nil, nil, errors.New("returned public key was empty") + } + + if publicKey, ok := p.(*rsa.PublicKey); ok { + return publicKey, pkeyID, nil + } + } + return nil, nil, errors.New("couldn't find public key") +} + +// ExtractContent returns a string representation of the interface's Content property. +func ExtractContent(i WithContent) (string, error) { + contentProperty := i.GetActivityStreamsContent() + if contentProperty == nil { + return "", errors.New("content property was nil") + } + for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() { + if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { + return iter.GetXMLSchemaString(), nil + } + } + return "", errors.New("no content found") +} + +// ExtractAttachments returns a slice of attachments on the interface. +func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) { + attachments := []*gtsmodel.MediaAttachment{} + attachmentProp := i.GetActivityStreamsAttachment() + if attachmentProp == nil { + return attachments, nil + } + for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() { + t := iter.GetType() + if t == nil { + continue + } + attachmentable, ok := t.(Attachmentable) + if !ok { + continue + } + attachment, err := ExtractAttachment(attachmentable) + if err != nil { + continue + } + attachments = append(attachments, attachment) + } + return attachments, nil +} + +// ExtractAttachment returns a gts model of an attachment from an attachmentable interface. +func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { + attachment := >smodel.MediaAttachment{ + File: gtsmodel.File{}, + } + + attachmentURL, err := ExtractURL(i) + if err != nil { + return nil, err + } + attachment.RemoteURL = attachmentURL.String() + + mediaType := i.GetActivityStreamsMediaType() + if mediaType == nil { + return nil, errors.New("no media type") + } + if mediaType.Get() == "" { + return nil, errors.New("no media type") + } + attachment.File.ContentType = mediaType.Get() + attachment.Type = gtsmodel.FileTypeImage + + name, err := ExtractName(i) + if err == nil { + attachment.Description = name + } + + attachment.Processing = gtsmodel.ProcessingStatusReceived + + return attachment, nil +} + +// func extractBlurhash(i withBlurhash) (string, error) { +// if i.GetTootBlurhashProperty() == nil { +// return "", errors.New("blurhash property was nil") +// } +// if i.GetTootBlurhashProperty().Get() == "" { +// return "", errors.New("empty blurhash string") +// } +// return i.GetTootBlurhashProperty().Get(), nil +// } + +// ExtractHashtags returns a slice of tags on the interface. +func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { + tags := []*gtsmodel.Tag{} + tagsProp := i.GetActivityStreamsTag() + if tagsProp == nil { + return tags, nil + } + for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { + t := iter.GetType() + if t == nil { + continue + } + + if t.GetTypeName() != "Hashtag" { + continue + } + + hashtaggable, ok := t.(Hashtaggable) + if !ok { + continue + } + + tag, err := ExtractHashtag(hashtaggable) + if err != nil { + continue + } + + tags = append(tags, tag) + } + return tags, nil +} + +// ExtractHashtag returns a gtsmodel tag from a hashtaggable. +func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { + tag := >smodel.Tag{} + + hrefProp := i.GetActivityStreamsHref() + if hrefProp == nil || !hrefProp.IsIRI() { + return nil, errors.New("no href prop") + } + tag.URL = hrefProp.GetIRI().String() + + name, err := ExtractName(i) + if err != nil { + return nil, err + } + tag.Name = strings.TrimPrefix(name, "#") + + return tag, nil +} + +// ExtractEmojis returns a slice of emojis on the interface. +func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) { + emojis := []*gtsmodel.Emoji{} + tagsProp := i.GetActivityStreamsTag() + if tagsProp == nil { + return emojis, nil + } + for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { + t := iter.GetType() + if t == nil { + continue + } + + if t.GetTypeName() != "Emoji" { + continue + } + + emojiable, ok := t.(Emojiable) + if !ok { + continue + } + + emoji, err := ExtractEmoji(emojiable) + if err != nil { + continue + } + + emojis = append(emojis, emoji) + } + return emojis, nil +} + +// ExtractEmoji ... +func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { + emoji := >smodel.Emoji{} + + idProp := i.GetJSONLDId() + if idProp == nil || !idProp.IsIRI() { + return nil, errors.New("no id for emoji") + } + uri := idProp.GetIRI() + emoji.URI = uri.String() + emoji.Domain = uri.Host + + name, err := ExtractName(i) + if err != nil { + return nil, err + } + emoji.Shortcode = strings.Trim(name, ":") + + if i.GetActivityStreamsIcon() == nil { + return nil, errors.New("no icon for emoji") + } + imageURL, err := ExtractIconURL(i) + if err != nil { + return nil, errors.New("no url for emoji image") + } + emoji.ImageRemoteURL = imageURL.String() + + return emoji, nil +} + +// ExtractMentions extracts a slice of gtsmodel Mentions from a WithTag interface. +func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) { + mentions := []*gtsmodel.Mention{} + tagsProp := i.GetActivityStreamsTag() + if tagsProp == nil { + return mentions, nil + } + for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { + t := iter.GetType() + if t == nil { + continue + } + + if t.GetTypeName() != "Mention" { + continue + } + + mentionable, ok := t.(Mentionable) + if !ok { + continue + } + + mention, err := ExtractMention(mentionable) + if err != nil { + continue + } + + mentions = append(mentions, mention) + } + return mentions, nil +} + +// ExtractMention extracts a gts model mention from a Mentionable. +func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) { + mention := >smodel.Mention{} + + mentionString, err := ExtractName(i) + if err != nil { + return nil, err + } + + // just make sure the mention string is valid so we can handle it properly later on... + username, domain, err := util.ExtractMentionParts(mentionString) + if err != nil { + return nil, err + } + if username == "" || domain == "" { + return nil, errors.New("username or domain was empty") + } + mention.NameString = mentionString + + // the href prop should be the AP URI of a user we know, eg https://example.org/users/whatever_user + hrefProp := i.GetActivityStreamsHref() + if hrefProp == nil || !hrefProp.IsIRI() { + return nil, errors.New("no href prop") + } + mention.MentionedAccountURI = hrefProp.GetIRI().String() + return mention, nil +} + +// ExtractActor extracts the actor ID/IRI from an interface WithActor. +func ExtractActor(i WithActor) (*url.URL, error) { + actorProp := i.GetActivityStreamsActor() + if actorProp == nil { + return nil, errors.New("actor property was nil") + } + for iter := actorProp.Begin(); iter != actorProp.End(); iter = iter.Next() { + if iter.IsIRI() && iter.GetIRI() != nil { + return iter.GetIRI(), nil + } + } + return nil, errors.New("no iri found for actor prop") +} + +// ExtractObject extracts a URL object from a WithObject interface. +func ExtractObject(i WithObject) (*url.URL, error) { + objectProp := i.GetActivityStreamsObject() + if objectProp == nil { + return nil, errors.New("object property was nil") + } + for iter := objectProp.Begin(); iter != objectProp.End(); iter = iter.Next() { + if iter.IsIRI() && iter.GetIRI() != nil { + return iter.GetIRI(), nil + } + } + return nil, errors.New("no iri found for object prop") +} diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go new file mode 100644 index 000000000..43dd149d5 --- /dev/null +++ b/internal/ap/interfaces.go @@ -0,0 +1,321 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package ap + +import "github.com/go-fed/activity/streams/vocab" + +// Accountable represents the minimum activitypub interface for representing an 'account'. +// This interface is fulfilled by: Person, Application, Organization, Service, and Group +type Accountable interface { + WithJSONLDId + WithTypeName + + WithPreferredUsername + WithIcon + WithName + WithImage + WithSummary + WithDiscoverable + WithURL + WithPublicKey + WithInbox + WithOutbox + WithFollowing + WithFollowers + WithFeatured +} + +// Statusable represents the minimum activitypub interface for representing a 'status'. +// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile +type Statusable interface { + WithJSONLDId + WithTypeName + + WithSummary + WithInReplyTo + WithPublished + WithURL + WithAttributedTo + WithTo + WithCC + WithSensitive + WithConversation + WithContent + WithAttachment + WithTag + WithReplies +} + +// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. +// This interface is fulfilled by: Audio, Document, Image, Video +type Attachmentable interface { + WithTypeName + WithMediaType + WithURL + WithName +} + +// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. +type Hashtaggable interface { + WithTypeName + WithHref + WithName +} + +// Emojiable represents the minimum interface for an 'emoji' tag. +type Emojiable interface { + WithJSONLDId + WithTypeName + WithName + WithUpdated + WithIcon +} + +// Mentionable represents the minimum interface for a 'mention' tag. +type Mentionable interface { + WithName + WithHref +} + +// Followable represents the minimum interface for an activitystreams 'follow' activity. +type Followable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Likeable represents the minimum interface for an activitystreams 'like' activity. +type Likeable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Blockable represents the minimum interface for an activitystreams 'block' activity. +type Blockable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Announceable represents the minimum interface for an activitystreams 'announce' activity. +type Announceable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject + WithPublished + WithTo + WithCC +} + +// CollectionPageable represents the minimum interface for an activitystreams 'CollectionPage' object. +type CollectionPageable interface { + WithJSONLDId + WithTypeName + + WithNext + WithPartOf + WithItems +} + +// WithJSONLDId represents an activity with JSONLDIdProperty +type WithJSONLDId interface { + GetJSONLDId() vocab.JSONLDIdProperty +} + +// WithTypeName represents an activity with a type name +type WithTypeName interface { + GetTypeName() string +} + +// WithPreferredUsername represents an activity with ActivityStreamsPreferredUsernameProperty +type WithPreferredUsername interface { + GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty +} + +// WithIcon represents an activity with ActivityStreamsIconProperty +type WithIcon interface { + GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty +} + +// WithName represents an activity with ActivityStreamsNameProperty +type WithName interface { + GetActivityStreamsName() vocab.ActivityStreamsNameProperty +} + +// WithImage represents an activity with ActivityStreamsImageProperty +type WithImage interface { + GetActivityStreamsImage() vocab.ActivityStreamsImageProperty +} + +// WithSummary represents an activity with ActivityStreamsSummaryProperty +type WithSummary interface { + GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty +} + +// WithDiscoverable represents an activity with TootDiscoverableProperty +type WithDiscoverable interface { + GetTootDiscoverable() vocab.TootDiscoverableProperty +} + +// WithURL represents an activity with ActivityStreamsUrlProperty +type WithURL interface { + GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty +} + +// WithPublicKey represents an activity with W3IDSecurityV1PublicKeyProperty +type WithPublicKey interface { + GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty +} + +// WithInbox represents an activity with ActivityStreamsInboxProperty +type WithInbox interface { + GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty +} + +// WithOutbox represents an activity with ActivityStreamsOutboxProperty +type WithOutbox interface { + GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty +} + +// WithFollowing represents an activity with ActivityStreamsFollowingProperty +type WithFollowing interface { + GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty +} + +// WithFollowers represents an activity with ActivityStreamsFollowersProperty +type WithFollowers interface { + GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty +} + +// WithFeatured represents an activity with TootFeaturedProperty +type WithFeatured interface { + GetTootFeatured() vocab.TootFeaturedProperty +} + +// WithAttributedTo represents an activity with ActivityStreamsAttributedToProperty +type WithAttributedTo interface { + GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty +} + +// WithAttachment represents an activity with ActivityStreamsAttachmentProperty +type WithAttachment interface { + GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty +} + +// WithTo represents an activity with ActivityStreamsToProperty +type WithTo interface { + GetActivityStreamsTo() vocab.ActivityStreamsToProperty +} + +// WithInReplyTo represents an activity with ActivityStreamsInReplyToProperty +type WithInReplyTo interface { + GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty +} + +// WithCC represents an activity with ActivityStreamsCcProperty +type WithCC interface { + GetActivityStreamsCc() vocab.ActivityStreamsCcProperty +} + +// WithSensitive ... +type WithSensitive interface { + // TODO +} + +// WithConversation ... +type WithConversation interface { + // TODO +} + +// WithContent represents an activity with ActivityStreamsContentProperty +type WithContent interface { + GetActivityStreamsContent() vocab.ActivityStreamsContentProperty +} + +// WithPublished represents an activity with ActivityStreamsPublishedProperty +type WithPublished interface { + GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty +} + +// WithTag represents an activity with ActivityStreamsTagProperty +type WithTag interface { + GetActivityStreamsTag() vocab.ActivityStreamsTagProperty +} + +// WithReplies represents an activity with ActivityStreamsRepliesProperty +type WithReplies interface { + GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty +} + +// WithMediaType represents an activity with ActivityStreamsMediaTypeProperty +type WithMediaType interface { + GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty +} + +// type withBlurhash interface { +// GetTootBlurhashProperty() vocab.TootBlurhashProperty +// } + +// type withFocalPoint interface { +// // TODO +// } + +// WithHref represents an activity with ActivityStreamsHrefProperty +type WithHref interface { + GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty +} + +// WithUpdated represents an activity with ActivityStreamsUpdatedProperty +type WithUpdated interface { + GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty +} + +// WithActor represents an activity with ActivityStreamsActorProperty +type WithActor interface { + GetActivityStreamsActor() vocab.ActivityStreamsActorProperty +} + +// WithObject represents an activity with ActivityStreamsObjectProperty +type WithObject interface { + GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty +} + +// WithNext represents an activity with ActivityStreamsNextProperty +type WithNext interface { + GetActivityStreamsNext() vocab.ActivityStreamsNextProperty +} + +// WithPartOf represents an activity with ActivityStreamsPartOfProperty +type WithPartOf interface { + GetActivityStreamsPartOf() vocab.ActivityStreamsPartOfProperty +} + +// WithItems represents an activity with ActivityStreamsItemsProperty +type WithItems interface { + GetActivityStreamsItems() vocab.ActivityStreamsItemsProperty +} diff --git a/internal/api/client/account/accountupdate_test.go b/internal/api/client/account/accountupdate_test.go index 341b865ff..349429625 100644 --- a/internal/api/client/account/accountupdate_test.go +++ b/internal/api/client/account/accountupdate_test.go @@ -53,10 +53,10 @@ func (suite *AccountUpdateTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.accountModule = account.New(suite.config, suite.processor, suite.log).(*account.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -80,6 +80,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandler() ctx, _ := gin.CreateTestContext(recorder) ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) ctx.Set(oauth.SessionAuthorizedToken, oauth.TokenToOauthToken(suite.testTokens["local_account_1"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) ctx.Request = httptest.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:8080/%s", account.UpdateCredentialsPath), bytes.NewReader(requestBody.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx) diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index cb503facb..4eec3bae5 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -78,7 +78,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.log = testrig.NewTestLog() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) @@ -95,7 +95,7 @@ func (suite *ServeFileTestSuite) TearDownSuite() { } func (suite *ServeFileTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 89a77a729..a61a36324 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -84,7 +84,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) // setup module being tested @@ -98,7 +98,7 @@ func (suite *MediaCreateTestSuite) TearDownSuite() { } func (suite *MediaCreateTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/status/statusboost_test.go b/internal/api/client/status/statusboost_test.go index 9400aeddc..fbe267fac 100644 --- a/internal/api/client/status/statusboost_test.go +++ b/internal/api/client/status/statusboost_test.go @@ -52,10 +52,10 @@ func (suite *StatusBoostTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index dd4a4386b..603432724 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -58,10 +58,10 @@ func (suite *StatusCreateTestSuite) SetupTest() { suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfave_test.go b/internal/api/client/status/statusfave_test.go index b1cafc2fb..0f44b5e90 100644 --- a/internal/api/client/status/statusfave_test.go +++ b/internal/api/client/status/statusfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusFaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfavedby_test.go b/internal/api/client/status/statusfavedby_test.go index b6e1591e0..22a549b30 100644 --- a/internal/api/client/status/statusfavedby_test.go +++ b/internal/api/client/status/statusfavedby_test.go @@ -55,10 +55,10 @@ func (suite *StatusFavedByTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusget_test.go b/internal/api/client/status/statusget_test.go index 1bbf48a91..1c700aaa5 100644 --- a/internal/api/client/status/statusget_test.go +++ b/internal/api/client/status/statusget_test.go @@ -45,10 +45,10 @@ func (suite *StatusGetTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusunfave_test.go b/internal/api/client/status/statusunfave_test.go index 36144c5ce..a5f267f4c 100644 --- a/internal/api/client/status/statusunfave_test.go +++ b/internal/api/client/status/statusunfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusUnfaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/s2s/user/repliesget.go b/internal/api/s2s/user/repliesget.go new file mode 100644 index 000000000..951cc428c --- /dev/null +++ b/internal/api/s2s/user/repliesget.go @@ -0,0 +1,186 @@ +package user + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// StatusRepliesGETHandler swagger:operation GET /users/{username}/statuses/{status}/replies s2sRepliesGet +// +// Get the replies collection for a status. +// +// Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`. +// +// If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`. +// +// HTTP signature is required on the request. +// +// --- +// tags: +// - s2s/federation +// +// produces: +// - application/activity+json +// +// parameters: +// - name: username +// type: string +// description: Username of the account. +// in: path +// required: true +// - name: status +// type: string +// description: ID of the status. +// in: path +// required: true +// - name: page +// type: boolean +// description: Return response as a CollectionPage. +// in: query +// default: false +// - name: only_other_accounts +// type: boolean +// description: Return replies only from accounts other than the status owner. +// in: query +// default: false +// - name: min_id +// type: string +// description: Minimum ID of the next status, used for paging. +// in: query +// +// responses: +// '200': +// in: body +// schema: +// "$ref": "#/definitions/swaggerStatusRepliesCollection" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '404': +// description: not found +func (m *Module) StatusRepliesGETHandler(c *gin.Context) { + l := m.log.WithFields(logrus.Fields{ + "func": "StatusRepliesGETHandler", + "url": c.Request.RequestURI, + }) + + requestedUsername := c.Param(UsernameKey) + if requestedUsername == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"}) + return + } + + requestedStatusID := c.Param(StatusIDKey) + if requestedStatusID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no status id specified in request"}) + return + } + + page := false + pageString := c.Query(PageKey) + if pageString != "" { + i, err := strconv.ParseBool(pageString) + if err != nil { + l.Debugf("error parsing page string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse page query param"}) + return + } + page = i + } + + onlyOtherAccounts := false + onlyOtherAccountsString := c.Query(OnlyOtherAccountsKey) + if onlyOtherAccountsString != "" { + i, err := strconv.ParseBool(onlyOtherAccountsString) + if err != nil { + l.Debugf("error parsing only_other_accounts string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse only_other_accounts query param"}) + return + } + onlyOtherAccounts = i + } + + minID := "" + minIDString := c.Query(MinIDKey) + if minIDString != "" { + minID = minIDString + } + + // make sure this actually an AP request + format := c.NegotiateFormat(ActivityPubAcceptHeaders...) + if format == "" { + c.JSON(http.StatusNotAcceptable, gin.H{"error": "could not negotiate format with given Accept header(s)"}) + return + } + l.Tracef("negotiated format: %s", format) + + // transfer the signature verifier from the gin context to the request context + ctx := c.Request.Context() + verifier, signed := c.Get(string(util.APRequestingPublicKeyVerifier)) + if signed { + ctx = context.WithValue(ctx, util.APRequestingPublicKeyVerifier, verifier) + } + + replies, err := m.processor.GetFediStatusReplies(ctx, requestedUsername, requestedStatusID, page, onlyOtherAccounts, minID, c.Request.URL) + if err != nil { + l.Info(err.Error()) + c.JSON(err.Code(), gin.H{"error": err.Safe()}) + return + } + + b, mErr := json.Marshal(replies) + if mErr != nil { + err := fmt.Errorf("could not marshal json: %s", mErr) + l.Error(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.Data(http.StatusOK, format, b) +} + +// SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies. +// swagger:model swaggerStatusRepliesCollection +type SwaggerStatusRepliesCollection struct { + // ActivityStreams context. + // example: https://www.w3.org/ns/activitystreams + Context string `json:"@context"` + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + ID string `json:"id"` + // ActivityStreams type. + // example: Collection + Type string `json:"type"` + // ActivityStreams first property. + First SwaggerStatusRepliesCollectionPage `json:"first"` +} + +// SwaggerStatusRepliesCollectionPage represents one page of a collection. +// swagger:model swaggerStatusRepliesCollectionPage +type SwaggerStatusRepliesCollectionPage struct { + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true + ID string `json:"id"` + // ActivityStreams type. + // example: CollectionPage + Type string `json:"type"` + // Link to the next page. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true + Next string `json:"next"` + // Collection this page belongs to. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + PartOf string `json:"partOf"` + // Items on this page. + // example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"] + Items []string `json:"items"` +} diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go new file mode 100644 index 000000000..75edbc882 --- /dev/null +++ b/internal/api/s2s/user/repliesget_test.go @@ -0,0 +1,241 @@ +package user_test + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type RepliesGetTestSuite struct { + UserStandardTestSuite +} + +func (suite *RepliesGetTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *RepliesGetTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.tc = testrig.NewTestTypeConverter(suite.db) + suite.storage = testrig.NewTestStorage() + suite.log = testrig.NewTestLog() + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) + suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") +} + +func (suite *RepliesGetTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func (suite *RepliesGetTestSuite) TestGetReplies() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","first":{"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"Collection"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + _, ok := t.(vocab.ActivityStreamsCollection) + assert.True(suite.T(), ok) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesNext() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_next"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 1) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesLast() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_last"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + fmt.Println(string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 0) +} + +func TestRepliesGetTestSuite(t *testing.T) { + suite.Run(t, new(RepliesGetTestSuite)) +} diff --git a/internal/api/s2s/user/user.go b/internal/api/s2s/user/user.go index 0cb8e1e90..b5ff9a699 100644 --- a/internal/api/s2s/user/user.go +++ b/internal/api/s2s/user/user.go @@ -34,6 +34,13 @@ const ( UsernameKey = "username" // StatusIDKey is for status IDs StatusIDKey = "status" + // OnlyOtherAccountsKey is for filtering status responses. + OnlyOtherAccountsKey = "only_other_accounts" + // MinIDKey is for filtering status responses. + MinIDKey = "min_id" + // PageKey is for filtering status responses. + PageKey = "page" + // UsersBasePath is the base path for serving information about Users eg https://example.org/users UsersBasePath = "/" + util.UsersPath // UsersBasePathWithUsername is just the users base path with the Username key in it. @@ -50,6 +57,8 @@ const ( UsersFollowingPath = UsersBasePathWithUsername + "/" + util.FollowingPath // UsersStatusPath is for serving GET requests to a particular status by a user, with the given username key and status ID UsersStatusPath = UsersBasePathWithUsername + "/" + util.StatusesPath + "/:" + StatusIDKey + // UsersStatusRepliesPath is for serving the replies collection of a status. + UsersStatusRepliesPath = UsersStatusPath + "/replies" ) // ActivityPubAcceptHeaders represents the Accept headers mentioned here: @@ -83,5 +92,6 @@ func (m *Module) Route(s router.Router) error { s.AttachHandler(http.MethodGet, UsersFollowingPath, m.FollowingGETHandler) s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler) s.AttachHandler(http.MethodGet, UsersPublicKeyPath, m.PublicKeyGETHandler) + s.AttachHandler(http.MethodGet, UsersStatusRepliesPath, m.StatusRepliesGETHandler) return nil } diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 91d1ea32d..71d4395eb 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -4,6 +4,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/internal/blob" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -18,13 +19,14 @@ import ( type UserStandardTestSuite struct { // standard suite interfaces suite.Suite - config *config.Config - db db.DB - log *logrus.Logger - tc typeutils.TypeConverter - federator federation.Federator - processor processing.Processor - storage blob.Storage + config *config.Config + db db.DB + log *logrus.Logger + tc typeutils.TypeConverter + federator federation.Federator + processor processing.Processor + storage blob.Storage + securityModule *security.Module // standard suite models testTokens map[string]*oauth.Token diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go index d20148802..ab0015c57 100644 --- a/internal/api/s2s/user/userget_test.go +++ b/internal/api/s2s/user/userget_test.go @@ -1,16 +1,11 @@ package user_test import ( - "bytes" "context" - "crypto/x509" "encoding/json" - "encoding/pem" - "fmt" "io/ioutil" "net/http" "net/http/httptest" - "strings" "testing" "github.com/gin-gonic/gin" @@ -19,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -42,10 +38,11 @@ func (suite *UserGetTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) - testrig.StandardDBSetup(suite.db) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -56,48 +53,11 @@ func (suite *UserGetTestSuite) TearDownTest() { func (suite *UserGetTestSuite) TestGetUser() { // the dereference we're gonna use - signedRequest := testrig.NewTestDereferenceRequests(suite.testAccounts)["foss_satan_dereference_zork"] - - requestingAccount := suite.testAccounts["remote_account_1"] + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_zork"] targetAccount := suite.testAccounts["local_account_1"] - encodedPublicKey, err := x509.MarshalPKIXPublicKey(requestingAccount.PublicKey) - assert.NoError(suite.T(), err) - publicKeyBytes := pem.EncodeToMemory(&pem.Block{ - Type: "PUBLIC KEY", - Bytes: encodedPublicKey, - }) - publicKeyString := strings.ReplaceAll(string(publicKeyBytes), "\n", "\\n") - - // for this test we need the client to return the public key of the requester on the 'remote' instance - responseBodyString := fmt.Sprintf(` - { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1" - ], - - "id": "%s", - "type": "Person", - "preferredUsername": "%s", - "inbox": "%s", - - "publicKey": { - "id": "%s", - "owner": "%s", - "publicKeyPem": "%s" - } - }`, requestingAccount.URI, requestingAccount.Username, requestingAccount.InboxURI, requestingAccount.PublicKeyURI, requestingAccount.URI, publicKeyString) - - // create a transport controller whose client will just return the response body string we specified above - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - r := ioutil.NopCloser(bytes.NewReader([]byte(responseBodyString))) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - })) - // get this transport controller embedded right in the user module we're testing + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) federator := testrig.NewTestFederator(suite.db, tc, suite.storage) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) userModule := user.New(suite.config, processor, suite.log).(*user.Module) @@ -105,7 +65,12 @@ func (suite *UserGetTestSuite) TestGetUser() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:8080%s", strings.Replace(user.UsersBasePathWithUsername, ":username", targetAccount.Username, 1)), nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.URI, nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) // normally the router would populate these params from the path values, // but because we're calling the function directly, we need to set them manually. @@ -116,11 +81,6 @@ func (suite *UserGetTestSuite) TestGetUser() { }, } - // we need these headers for the request to be validated - ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) - ctx.Request.Header.Set("Date", signedRequest.DateHeader) - ctx.Request.Header.Set("Digest", signedRequest.DigestHeader) - // trigger the function being tested userModule.UsersGETHandler(ctx) diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 3c4f97dea..2314e2608 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -115,7 +115,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log // build backend handlers mediaHandler := media.New(c, dbService, storageBackend, log) oauthServer := oauth.New(dbService, log) - transportController := transport.NewController(c, &federation.Clock{}, http.DefaultClient, log) + transportController := transport.NewController(c, dbService, &federation.Clock{}, http.DefaultClient, log) federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler) processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log) if err := processor.Start(); err != nil { diff --git a/internal/cliactions/testrig/testrig.go b/internal/cliactions/testrig/testrig.go index e2b97fe61..a7032825c 100644 --- a/internal/cliactions/testrig/testrig.go +++ b/internal/cliactions/testrig/testrig.go @@ -46,7 +46,7 @@ import ( var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error { c := testrig.NewTestConfig() dbService := testrig.NewTestDB() - testrig.StandardDBSetup(dbService) + testrig.StandardDBSetup(dbService, nil) router := testrig.NewTestRouter(dbService) storageBackend := testrig.NewTestStorage() testrig.StandardStorageSetup(storageBackend, "./testrig/media") @@ -59,7 +59,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log StatusCode: 200, Body: r, }, nil - })) + }), dbService) federator := testrig.NewTestFederator(dbService, transportController, storageBackend) processor := testrig.NewTestProcessor(dbService, storageBackend, federator) diff --git a/internal/db/db.go b/internal/db/db.go index c764cc716..d0b23fbc6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -218,10 +218,14 @@ type DB interface { GetFaveCountForStatus(status *gtsmodel.Status) (int, error) // StatusParents get the parent statuses of a given status. - StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate parent will be returned. + StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) // StatusChildren gets the child statuses of a given status. - StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate children will be returned. + StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) // StatusFavedBy checks if a given status has been faved by a given account ID StatusFavedBy(status *gtsmodel.Status, accountID string) (bool, error) diff --git a/internal/db/pg/statuscontext.go b/internal/db/pg/statuscontext.go index 732485ab5..2ff1a20bb 100644 --- a/internal/db/pg/statuscontext.go +++ b/internal/db/pg/statuscontext.go @@ -25,14 +25,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (ps *postgresService) StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) { parents := []*gtsmodel.Status{} - ps.statusParent(status, &parents) + ps.statusParent(status, &parents, onlyDirect) return parents, nil } -func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status) { +func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status, onlyDirect bool) { if status.InReplyToID == "" { return } @@ -42,13 +42,16 @@ func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses * *foundStatuses = append(*foundStatuses, parentStatus) } - ps.statusParent(parentStatus, foundStatuses) + if onlyDirect { + return + } + ps.statusParent(parentStatus, foundStatuses, false) } -func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) { foundStatuses := &list.List{} foundStatuses.PushFront(status) - ps.statusChildren(status, foundStatuses) + ps.statusChildren(status, foundStatuses, onlyDirect, minID) children := []*gtsmodel.Status{} for e := foundStatuses.Front(); e != nil; e = e.Next() { @@ -66,11 +69,15 @@ func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel. return children, nil } -func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List) { +func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) { immediateChildren := []*gtsmodel.Status{} - err := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID).Select() - if err != nil { + q := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID) + if minID != "" { + q = q.Where("status.id > ?", minID) + } + + if err := q.Select(); err != nil { return } @@ -88,6 +95,10 @@ func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses } } - ps.statusChildren(child, foundStatuses) + // only do one loop if we only want direct children + if onlyDirect { + return + } + ps.statusChildren(child, foundStatuses, false, minID) } } diff --git a/internal/federation/authenticate.go b/internal/federation/authenticate.go index 0cb8db6dc..699691ca6 100644 --- a/internal/federation/authenticate.go +++ b/internal/federation/authenticate.go @@ -147,6 +147,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU if strings.EqualFold(requestingHost, f.config.Host) { // LOCAL ACCOUNT REQUEST // the request is coming from INSIDE THE HOUSE so skip the remote dereferencing + l.Tracef("proceeding without dereference for local public key %s", requestingPublicKeyID) if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil { return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err) } @@ -158,6 +159,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil { // REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY // this is a remote account and we already have the public key for it so use that + l.Tracef("proceeding without dereference for cached public key %s", requestingPublicKeyID) publicKey = requestingRemoteAccount.PublicKey pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI) if err != nil { @@ -167,7 +169,8 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU // REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY // the request is remote and we don't have the public key yet, // so we need to authenticate the request properly by dereferencing the remote key - transport, err := f.GetTransportForUser(requestedUsername) + l.Tracef("proceeding with dereference for uncached public key %s", requestingPublicKeyID) + transport, err := f.transportController.NewTransportForUsername(requestedUsername) if err != nil { return nil, false, fmt.Errorf("transport err: %s", err) } @@ -209,15 +212,28 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } pkOwnerURI = pkOwnerProp.GetIRI() } + + // after all that, public key should be defined if publicKey == nil { return nil, false, errors.New("returned public key was empty") } // do the actual authentication here! - algo := httpsig.RSA_SHA256 // TODO: make this more robust - if err := verifier.Verify(publicKey, algo); err != nil { - return nil, false, nil + algos := []httpsig.Algorithm{ + httpsig.RSA_SHA512, + httpsig.RSA_SHA256, + httpsig.ED25519, + } + + for _, algo := range algos { + l.Tracef("trying algo: %s", algo) + if err := verifier.Verify(publicKey, algo); err == nil { + l.Tracef("authentication for %s PASSED with algorithm %s", pkOwnerURI, algo) + return pkOwnerURI, true, nil + } + l.Tracef("authentication for %s NOT PASSED with algorithm %s: %s", pkOwnerURI, algo, err) } - return pkOwnerURI, true, nil + l.Infof("authentication not passed for %s", pkOwnerURI) + return nil, false, nil } diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go index b87462acd..8975d6c0c 100644 --- a/internal/federation/dereference.go +++ b/internal/federation/dereference.go @@ -1,526 +1,32 @@ package federation import ( - "context" - "encoding/json" - "errors" - "fmt" "net/url" - "github.com/go-fed/activity/streams" - "github.com/go-fed/activity/streams/vocab" - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) -func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) { - f.startHandshake(username, remoteAccountID) - defer f.stopHandshake(username, remoteAccountID) - - if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteAccountID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - switch t.GetTypeName() { - case string(gtsmodel.ActivityStreamsPerson): - p, ok := t.(vocab.ActivityStreamsPerson) - if !ok { - return nil, errors.New("error resolving type as activitystreams person") - } - return p, nil - case string(gtsmodel.ActivityStreamsApplication): - p, ok := t.(vocab.ActivityStreamsApplication) - if !ok { - return nil, errors.New("error resolving type as activitystreams application") - } - return p, nil - case string(gtsmodel.ActivityStreamsService): - p, ok := t.(vocab.ActivityStreamsService) - if !ok { - return nil, errors.New("error resolving type as activitystreams service") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + return f.dereferencer.GetRemoteAccount(username, remoteAccountID, refresh) } -func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) { - if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteStatusID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile - switch t.GetTypeName() { - case gtsmodel.ActivityStreamsArticle: - p, ok := t.(vocab.ActivityStreamsArticle) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsArticle") - } - return p, nil - case gtsmodel.ActivityStreamsDocument: - p, ok := t.(vocab.ActivityStreamsDocument) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsDocument") - } - return p, nil - case gtsmodel.ActivityStreamsImage: - p, ok := t.(vocab.ActivityStreamsImage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsImage") - } - return p, nil - case gtsmodel.ActivityStreamsVideo: - p, ok := t.(vocab.ActivityStreamsVideo) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsVideo") - } - return p, nil - case gtsmodel.ActivityStreamsNote: - p, ok := t.(vocab.ActivityStreamsNote) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsNote") - } - return p, nil - case gtsmodel.ActivityStreamsPage: - p, ok := t.(vocab.ActivityStreamsPage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPage") - } - return p, nil - case gtsmodel.ActivityStreamsEvent: - p, ok := t.(vocab.ActivityStreamsEvent) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsEvent") - } - return p, nil - case gtsmodel.ActivityStreamsPlace: - p, ok := t.(vocab.ActivityStreamsPlace) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPlace") - } - return p, nil - case gtsmodel.ActivityStreamsProfile: - p, ok := t.(vocab.ActivityStreamsProfile) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsProfile") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + return f.dereferencer.GetRemoteStatus(username, remoteStatusID, refresh) } -func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { - if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +func (f *federator) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + return f.dereferencer.EnrichRemoteStatus(username, status) } -// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming -// federated status, back in the federating db's Create function. -// -// When a status comes in from the federation API, there are certain fields that -// haven't been dereferenced yet, because we needed to provide a snappy synchronous -// response to the caller. By the time it reaches this function though, it's being -// processed asynchronously, so we have all the time in the world to fetch the various -// bits and bobs that are attached to the status, and properly flesh it out, before we -// send the status to any timelines and notify people. -// -// Things to dereference and fetch here: -// -// 1. Media attachments. -// 2. Hashtags. -// 3. Emojis. -// 4. Mentions. -// 5. Posting account. -// 6. Replied-to-status. -// -// SIDE EFFECTS: -// This function will deference all of the above, insert them in the database as necessary, -// and attach them to the status. The status itself will not be added to the database yet, -// that's up the caller to do. -func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceStatusFields", - "status": fmt.Sprintf("%+v", status), - }) - l.Debug("entering function") - - statusURI, err := url.Parse(status.URI) - if err != nil { - return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) - } - if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error creating transport: %s", err) - } - - // the status should have an ID by now, but just in case it doesn't let's generate one here - // because we'll need it further down - if status.ID == "" { - newID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return err - } - status.ID = newID - } - - // 1. Media attachments. - // - // At this point we should know: - // * the media type of the file we're looking for (a.File.ContentType) - // * the blurhash (a.Blurhash) - // * the file type (a.Type) - // * the remote URL (a.RemoteURL) - // This should be enough to pass along to the media processor. - attachmentIDs := []string{} - for _, a := range status.GTSMediaAttachments { - l.Debugf("dereferencing attachment: %+v", a) - - // it might have been processed elsewhere so check first if it's already in the database or not - maybeAttachment := >smodel.MediaAttachment{} - err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) - if err == nil { - // we already have it in the db, dereferenced, no need to do it again - l.Debugf("attachment already exists with id %s", maybeAttachment.ID) - attachmentIDs = append(attachmentIDs, maybeAttachment.ID) - continue - } - if _, ok := err.(db.ErrNoEntries); !ok { - // we have a real error - return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) - } - // it just doesn't exist yet so carry on - l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) - deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) - if err != nil { - l.Errorf("error dereferencing status attachment: %s", err) - continue - } - l.Debugf("dereferenced attachment: %+v", deferencedAttachment) - deferencedAttachment.StatusID = status.ID - deferencedAttachment.Description = a.Description - if err := f.db.Put(deferencedAttachment); err != nil { - return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) - } - attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) - } - status.Attachments = attachmentIDs - - // 2. Hashtags - - // 3. Emojis - - // 4. Mentions - // At this point, mentions should have the namestring and mentionedAccountURI set on them. - // - // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. - mentions := []string{} - for _, m := range status.GTSMentions { - if m.ID == "" { - mID, err := id.NewRandomULID() - if err != nil { - return err - } - m.ID = mID - } - - uri, err := url.Parse(m.MentionedAccountURI) - if err != nil { - l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) - continue - } - - m.StatusID = status.ID - m.OriginAccountID = status.GTSAuthorAccount.ID - m.OriginAccountURI = status.GTSAuthorAccount.URI - - targetAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil { - // proper error - if _, ok := err.(db.ErrNoEntries); !ok { - return fmt.Errorf("db error checking for account with uri %s", uri.String()) - } - - // we just don't have it yet, so we should go get it.... - accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri) - if err != nil { - // we can't dereference it so just skip it - l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err) - continue - } - - targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err) - continue - } - - targetAccountID, err := id.NewRandomULID() - if err != nil { - return err - } - targetAccount.ID = targetAccountID - - if err := f.db.Put(targetAccount); err != nil { - return fmt.Errorf("db error inserting account with uri %s", uri.String()) - } - } - - // by this point, we know the targetAccount exists in our database with an ID :) - m.TargetAccountID = targetAccount.ID - if err := f.db.Put(m); err != nil { - return fmt.Errorf("error creating mention: %s", err) - } - mentions = append(mentions, m.ID) - } - status.Mentions = mentions - - return nil +func (f *federator) DereferenceRemoteThread(username string, statusIRI *url.URL) error { + return f.dereferencer.DereferenceThread(username, statusIRI) } -func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceAccountFields", - "requestingUsername": requestingUsername, - }) - - accountURI, err := url.Parse(account.URI) - if err != nil { - return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error getting transport for user: %s", err) - } - - // fetch the header and avatar - if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { - // if this doesn't work, just skip it -- we can do it later - l.Debugf("error fetching header/avi for account: %s", err) - } - - if err := f.db.UpdateByID(account.ID, account); err != nil { - return fmt.Errorf("error updating account in database: %s", err) - } - - return nil +func (f *federator) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + return f.dereferencer.GetRemoteInstance(username, remoteInstanceURI) } func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { - if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { - // we can't do anything unfortunately - return errors.New("DereferenceAnnounce: no URI to dereference") - } - - boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) - if err != nil { - return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) - } - if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) - } - - // check if we already have the boosted status in the database - boostedStatus := >smodel.Status{} - err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus) - if err == nil { - // nice, we already have it so we don't actually need to dereference it from remote - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil - } - - // we don't have it so we need to dereference it - statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // make sure we have the author account in the db - attributedToProp := statusable.GetActivityStreamsAttributedTo() - for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() { - accountURI := iter.GetIRI() - if accountURI == nil { - continue - } - - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, >smodel.Account{}); err == nil { - // we already have it, fine - continue - } - - // we don't have the boosted status author account yet so dereference it - accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err) - } - account, err := f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err) - } - - accountID, err := id.NewRandomULID() - if err != nil { - return err - } - account.ID = accountID - - if err := f.db.Put(account); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err) - } - - if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err) - } - } - - // now convert the statusable into something we can understand - boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err) - } - - boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt) - if err != nil { - return nil - } - boostedStatus.ID = boostedStatusID - - if err := f.db.Put(boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err) - } - - // now dereference additional fields straight away (we're already async here so we have time) - if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // update with the newly dereferenced fields - if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err) - } - - // we have everything we need! - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil -} - -// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport -// on behalf of requestingUsername. -// -// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. -// -// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated -// to reflect the creation of these new attachments. -func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { - accountURI, err := url.Parse(targetAccount.URI) - if err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) - } - - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.AvatarRemoteURL, - Avatar: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing avatar for user: %s", err) - } - targetAccount.AvatarMediaAttachmentID = a.ID - } - - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.HeaderRemoteURL, - Header: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing header for user: %s", err) - } - targetAccount.HeaderMediaAttachmentID = a.ID - } - return nil + return f.dereferencer.DereferenceAnnounce(announce, requestingUsername) } diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go new file mode 100644 index 000000000..c403ec66f --- /dev/null +++ b/internal/federation/dereferencing/account.go @@ -0,0 +1,243 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/transport" +) + +// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by +// the federatingDB's Create function, or during the federated authorization flow. +func (d *deref) EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) { + if err := d.populateAccountFields(account, username, false); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(account.ID, account); err != nil { + return nil, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err) + } + + return account, nil +} + +// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, +// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new +// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before, +// it will be false. +// +// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling +// the remote instance again. +// +// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). +func (d *deref) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + new := true + + // check if we already have the account in our db + maybeAccount := >smodel.Account{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteAccountID.String()}}, maybeAccount); err == nil { + // we've seen this account before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeAccount as-is and avoid doing any external calls + if !refresh { + return maybeAccount, new, nil + } + } + + accountable, err := d.dereferenceAccountable(username, remoteAccountID) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err) + } + + gtsAccount, err := d.typeConverter.ASRepresentationToAccount(accountable, false) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err) + } + + if new { + // generate a new id since we haven't seen this account before, and do a put + ulid, err := id.NewRandomULID() + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err) + } + gtsAccount.ID = ulid + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.Put(gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err) + } + } else { + // take the id we already have and do an update + gtsAccount.ID = maybeAccount.ID + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.UpdateByID(gtsAccount.ID, gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error updating existing account: %s", err) + } + } + + return gtsAccount, new, nil +} + +// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever +// it finds as something that an account model can be constructed out of. +// +// Will work for Person, Application, or Service models. +func (d *deref) dereferenceAccountable(username string, remoteAccountID *url.URL) (ap.Accountable, error) { + d.startHandshake(username, remoteAccountID) + defer d.stopHandshake(username, remoteAccountID) + + if blocked, err := d.blockedDomain(remoteAccountID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteAccountID) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %s", remoteAccountID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %s", err) + } + + switch t.GetTypeName() { + case string(gtsmodel.ActivityStreamsPerson): + p, ok := t.(vocab.ActivityStreamsPerson) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person") + } + return p, nil + case string(gtsmodel.ActivityStreamsApplication): + p, ok := t.(vocab.ActivityStreamsApplication) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application") + } + return p, nil + case string(gtsmodel.ActivityStreamsService): + p, ok := t.(vocab.ActivityStreamsService) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName()) +} + +// populateAccountFields populates any fields on the given account that weren't populated by the initial +// dereferencing. This includes things like header and avatar etc. +func (d *deref) populateAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { + l := d.log.WithFields(logrus.Fields{ + "func": "PopulateAccountFields", + "requestingUsername": requestingUsername, + }) + + accountURI, err := url.Parse(account.URI) + if err != nil { + return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host) + } + + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err) + } + + // fetch the header and avatar + if err := d.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { + // if this doesn't work, just skip it -- we can do it later + l.Debugf("error fetching header/avi for account: %s", err) + } + + return nil +} + +// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport +// on behalf of requestingUsername. +// +// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. +// +// SIDE EFFECTS: remote header and avatar will be stored in local storage. +func (d *deref) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { + accountURI, err := url.Parse(targetAccount.URI) + if err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) + } + + if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.AvatarRemoteURL, + Avatar: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing avatar for user: %s", err) + } + targetAccount.AvatarMediaAttachmentID = a.ID + } + + if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.HeaderRemoteURL, + Header: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing header for user: %s", err) + } + targetAccount.HeaderMediaAttachmentID = a.ID + } + return nil +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go new file mode 100644 index 000000000..2522a4034 --- /dev/null +++ b/internal/federation/dereferencing/announce.go @@ -0,0 +1,65 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "errors" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { + if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { + // we can't do anything unfortunately + return errors.New("DereferenceAnnounce: no URI to dereference") + } + + boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) + } + if blocked, err := d.blockedDomain(boostedStatusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) + } + + // dereference statuses in the thread of the boosted status + if err := d.DereferenceThread(requestingUsername, boostedStatusURI); err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing thread of boosted status: %s", err) + } + + boostedStatus, _, _, err := d.GetRemoteStatus(requestingUsername, boostedStatusURI, false) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) + } + + announce.Content = boostedStatus.Content + announce.ContentWarning = boostedStatus.ContentWarning + announce.ActivityStreamsType = boostedStatus.ActivityStreamsType + announce.Sensitive = boostedStatus.Sensitive + announce.Language = boostedStatus.Language + announce.Text = boostedStatus.Text + announce.BoostOfID = boostedStatus.ID + announce.BoostOfAccountID = boostedStatus.AccountID + announce.Visibility = boostedStatus.Visibility + announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced + announce.GTSBoostedStatus = boostedStatus + return nil +} diff --git a/internal/federation/dereferencing/blocked.go b/internal/federation/dereferencing/blocked.go new file mode 100644 index 000000000..a66afbb60 --- /dev/null +++ b/internal/federation/dereferencing/blocked.go @@ -0,0 +1,41 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) blockedDomain(host string) (bool, error) { + b := >smodel.DomainBlock{} + err := d.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b) + if err == nil { + // block exists + return true, nil + } + + if _, ok := err.(db.ErrNoEntries); ok { + // there are no entries so there's no block + return false, nil + } + + // there's an actual error + return false, err +} diff --git a/internal/federation/dereferencing/collectionpage.go b/internal/federation/dereferencing/collectionpage.go new file mode 100644 index 000000000..5feadc1ad --- /dev/null +++ b/internal/federation/dereferencing/collectionpage.go @@ -0,0 +1,70 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong. +func (d *deref) DereferenceCollectionPage(username string, pageIRI *url.URL) (ap.CollectionPageable, error) { + if blocked, err := d.blockedDomain(pageIRI.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error creating transport: %s", err) + } + + b, err := transport.Dereference(context.Background(), pageIRI) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error deferencing %s: %s", pageIRI.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error resolving json into ap vocab type: %s", err) + } + + if t.GetTypeName() != gtsmodel.ActivityStreamsCollectionPage { + return nil, fmt.Errorf("DereferenceCollectionPage: type name %s not supported", t.GetTypeName()) + } + + p, ok := t.(vocab.ActivityStreamsCollectionPage) + if !ok { + return nil, errors.New("DereferenceCollectionPage: error resolving type as activitystreams collection page") + } + + return p, nil +} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go new file mode 100644 index 000000000..03b90569a --- /dev/null +++ b/internal/federation/dereferencing/dereferencer.go @@ -0,0 +1,73 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "net/url" + "sync" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. +type Dereferencer interface { + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + + DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error + DereferenceThread(username string, statusIRI *url.URL) error + + Handshaking(username string, remoteAccountID *url.URL) bool +} + +type deref struct { + log *logrus.Logger + db db.DB + typeConverter typeutils.TypeConverter + transportController transport.Controller + mediaHandler media.Handler + config *config.Config + handshakes map[string][]*url.URL + handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map +} + +// NewDereferencer returns a Dereferencer initialized with the given parameters. +func NewDereferencer(config *config.Config, db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler, log *logrus.Logger) Dereferencer { + return &deref{ + log: log, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaHandler: mediaHandler, + config: config, + handshakeSync: &sync.Mutex{}, + } +} diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go new file mode 100644 index 000000000..cda8eafd0 --- /dev/null +++ b/internal/federation/dereferencing/handshake.go @@ -0,0 +1,98 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import "net/url" + +func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + // handshakes isn't even initialized yet so we can't be handshaking with anyone + return false + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // user isn't handshaking with anyone, bail + return false + } + + for _, id := range remoteIDs { + if id.String() == remoteAccountID.String() { + // we are currently handshaking with the remote account, yep + return true + } + } + + // didn't find it which means we're not handshaking + return false +} + +func (d *deref) startHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + // lazily initialize handshakes + if d.handshakes == nil { + d.handshakes = make(map[string][]*url.URL) + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet, so just add this entry and return + d.handshakes[username] = []*url.URL{remoteAccountID} + return + } + + // add the remote ID to the slice + remoteIDs = append(remoteIDs, remoteAccountID) + d.handshakes[username] = remoteIDs +} + +func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + return + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet anyway so just bail + return + } + + newRemoteIDs := []*url.URL{} + for _, id := range remoteIDs { + if id.String() != remoteAccountID.String() { + newRemoteIDs = append(newRemoteIDs, id) + } + } + + if len(newRemoteIDs) == 0 { + // there are no handshakes so just remove this user entry from the map and save a few bytes + delete(d.handshakes, username) + } else { + // there are still other handshakes ongoing + d.handshakes[username] = newRemoteIDs + } +} diff --git a/internal/federation/dereferencing/instance.go b/internal/federation/dereferencing/instance.go new file mode 100644 index 000000000..80f626662 --- /dev/null +++ b/internal/federation/dereferencing/instance.go @@ -0,0 +1,40 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + if blocked, err := d.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { + return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("transport err: %s", err) + } + + return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go new file mode 100644 index 000000000..b05f6e72c --- /dev/null +++ b/internal/federation/dereferencing/status.go @@ -0,0 +1,369 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" +) + +// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by +// the federatingDB's Create function, but additional dereferencing is needed on it. +func (d *deref) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + if err := d.populateStatusFields(status, username); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(status.ID, status); err != nil { + return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err) + } + + return status, nil +} + +// GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status, +// puts it in the database, and returns it to a caller. The boolean indicates whether the status is new +// to us or not. If we haven't seen the status before, bool will be true. If we have seen the status before, +// it will be false. +// +// If refresh is true, then even if we have the status in our database already, it will be dereferenced from its +// remote representation, as will its owner. +// +// If a dereference was performed, then the function also returns the ap.Statusable representation for further processing. +// +// SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored. +func (d *deref) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + new := true + + // check if we already have the status in our db + maybeStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteStatusID.String()}}, maybeStatus); err == nil { + // we've seen this status before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeStatus as-is and avoid doing any external calls + if !refresh { + return maybeStatus, nil, new, nil + } + } + + statusable, err := d.dereferenceStatusable(username, remoteStatusID) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error dereferencing statusable: %s", err) + } + + accountURI, err := ap.ExtractAttributedTo(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err) + } + + // do this so we know we have the remote account of the status in the db + _, _, err = d.GetRemoteAccount(username, accountURI, false) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err) + } + + gtsStatus, err := d.typeConverter.ASStatusToStatus(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err) + } + + if new { + ulid, err := id.NewULIDFromTime(gtsStatus.CreatedAt) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err) + } + gtsStatus.ID = ulid + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.Put(gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error putting new status: %s", err) + } + } else { + gtsStatus.ID = maybeStatus.ID + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.UpdateByID(gtsStatus.ID, gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error updating status: %s", err) + } + } + + return gtsStatus, statusable, new, nil +} + +func (d *deref) dereferenceStatusable(username string, remoteStatusID *url.URL) (ap.Statusable, error) { + if blocked, err := d.blockedDomain(remoteStatusID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteStatusID) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error deferencing %s: %s", remoteStatusID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error resolving json into ap vocab type: %s", err) + } + + // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile + switch t.GetTypeName() { + case gtsmodel.ActivityStreamsArticle: + p, ok := t.(vocab.ActivityStreamsArticle) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsArticle") + } + return p, nil + case gtsmodel.ActivityStreamsDocument: + p, ok := t.(vocab.ActivityStreamsDocument) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsDocument") + } + return p, nil + case gtsmodel.ActivityStreamsImage: + p, ok := t.(vocab.ActivityStreamsImage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsImage") + } + return p, nil + case gtsmodel.ActivityStreamsVideo: + p, ok := t.(vocab.ActivityStreamsVideo) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsVideo") + } + return p, nil + case gtsmodel.ActivityStreamsNote: + p, ok := t.(vocab.ActivityStreamsNote) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsNote") + } + return p, nil + case gtsmodel.ActivityStreamsPage: + p, ok := t.(vocab.ActivityStreamsPage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPage") + } + return p, nil + case gtsmodel.ActivityStreamsEvent: + p, ok := t.(vocab.ActivityStreamsEvent) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsEvent") + } + return p, nil + case gtsmodel.ActivityStreamsPlace: + p, ok := t.(vocab.ActivityStreamsPlace) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPlace") + } + return p, nil + case gtsmodel.ActivityStreamsProfile: + p, ok := t.(vocab.ActivityStreamsProfile) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsProfile") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceStatusable: type name %s not supported", t.GetTypeName()) +} + +// populateStatusFields fetches all the information we temporarily pinned to an incoming +// federated status, back in the federating db's Create function. +// +// When a status comes in from the federation API, there are certain fields that +// haven't been dereferenced yet, because we needed to provide a snappy synchronous +// response to the caller. By the time it reaches this function though, it's being +// processed asynchronously, so we have all the time in the world to fetch the various +// bits and bobs that are attached to the status, and properly flesh it out, before we +// send the status to any timelines and notify people. +// +// Things to dereference and fetch here: +// +// 1. Media attachments. +// 2. Hashtags. +// 3. Emojis. +// 4. Mentions. +// 5. Posting account. +// 6. Replied-to-status. +// +// SIDE EFFECTS: +// This function will deference all of the above, insert them in the database as necessary, +// and attach them to the status. The status itself will not be added to the database yet, +// that's up the caller to do. +func (d *deref) populateStatusFields(status *gtsmodel.Status, requestingUsername string) error { + l := d.log.WithFields(logrus.Fields{ + "func": "dereferenceStatusFields", + "status": fmt.Sprintf("%+v", status), + }) + l.Debug("entering function") + + // make sure we have a status URI and that the domain in question isn't blocked + statusURI, err := url.Parse(status.URI) + if err != nil { + return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) + } + if blocked, err := d.blockedDomain(statusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) + } + + // we can continue -- create a new transport here because we'll probably need it + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("error creating transport: %s", err) + } + + // in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one + if status.ID == "" { + newID, err := id.NewULIDFromTime(status.CreatedAt) + if err != nil { + return err + } + status.ID = newID + } + + // 1. Media attachments. + // + // At this point we should know: + // * the media type of the file we're looking for (a.File.ContentType) + // * the blurhash (a.Blurhash) + // * the file type (a.Type) + // * the remote URL (a.RemoteURL) + // This should be enough to pass along to the media processor. + attachmentIDs := []string{} + for _, a := range status.GTSMediaAttachments { + l.Tracef("dereferencing attachment: %+v", a) + + // it might have been processed elsewhere so check first if it's already in the database or not + maybeAttachment := >smodel.MediaAttachment{} + err := d.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) + if err == nil { + // we already have it in the db, dereferenced, no need to do it again + l.Tracef("attachment already exists with id %s", maybeAttachment.ID) + attachmentIDs = append(attachmentIDs, maybeAttachment.ID) + continue + } + if _, ok := err.(db.ErrNoEntries); !ok { + // we have a real error + return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) + } + // it just doesn't exist yet so carry on + l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) + deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) + if err != nil { + l.Errorf("error dereferencing status attachment: %s", err) + continue + } + l.Debugf("dereferenced attachment: %+v", deferencedAttachment) + deferencedAttachment.StatusID = status.ID + deferencedAttachment.Description = a.Description + if err := d.db.Put(deferencedAttachment); err != nil { + return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) + } + attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) + } + status.Attachments = attachmentIDs + + // 2. Hashtags + + // 3. Emojis + + // 4. Mentions + // At this point, mentions should have the namestring and mentionedAccountURI set on them. + // + // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. + mentions := []string{} + for _, m := range status.GTSMentions { + + if m.ID != "" { + continue + // we've already populated this mention, since it has an ID + } + + mID, err := id.NewRandomULID() + if err != nil { + return err + } + m.ID = mID + + uri, err := url.Parse(m.MentionedAccountURI) + if err != nil { + l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) + continue + } + + m.StatusID = status.ID + m.OriginAccountID = status.GTSAuthorAccount.ID + m.OriginAccountURI = status.GTSAuthorAccount.URI + + targetAccount, _, err := d.GetRemoteAccount(requestingUsername, uri, false) + if err != nil { + continue + } + + // by this point, we know the targetAccount exists in our database with an ID :) + m.TargetAccountID = targetAccount.ID + if err := d.db.Put(m); err != nil { + return fmt.Errorf("error creating mention: %s", err) + } + mentions = append(mentions, m.ID) + } + status.Mentions = mentions + + // status has replyToURI but we don't have an ID yet for the status it replies to + if status.InReplyToURI != "" && status.InReplyToID == "" { + replyToStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: status.InReplyToURI}}, replyToStatus); err == nil { + // we have the status + status.InReplyToID = replyToStatus.ID + status.InReplyToAccountID = replyToStatus.AccountID + } + } + + return nil +} diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go new file mode 100644 index 000000000..2a407f923 --- /dev/null +++ b/internal/federation/dereferencing/thread.go @@ -0,0 +1,250 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo), +// and dereferences statusables in the conversation. +// +// This process involves working up and down the chain of replies, and parsing through the collections of IDs +// presented by remote instances as part of their replies collections, and will likely involve making several calls to +// multiple different hosts. +func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "DereferenceThread", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering DereferenceThread") + + // if it's our status we already have everything stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + // first make sure we have this status in our db + _, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true) + if err != nil { + return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err) + } + + // first iterate up through ancestors, dereferencing if necessary as we go + if err := d.iterateAncestors(username, *statusIRI); err != nil { + return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err) + } + + // now iterate down through descendants, again dereferencing as we go + if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil { + return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err) + } + + return nil +} + +// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way. +func (d *deref) iterateAncestors(username string, statusIRI url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateAncestors", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateAncestors") + + // if it's our status we don't need to dereference anything so we can immediately move up the chain + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, moving up to next ancestor") + + // since this is our status, we know we can extract the id from the status path + _, id, err := util.ParseStatusesPath(&statusIRI) + if err != nil { + return err + } + + status := >smodel.Status{} + if err := d.db.GetByID(id, status); err != nil { + return err + } + + if status.InReplyToURI == "" { + // status doesn't reply to anything + return nil + } + nextIRI, err := url.Parse(status.URI) + if err != nil { + return err + } + return d.iterateAncestors(username, *nextIRI) + } + + // If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus + // We call it with refresh to true because we want the statusable representation to parse inReplyTo from. + status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true) + if err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + inReplyTo := ap.ExtractInReplyToURI(statusable) + if inReplyTo == nil || inReplyTo.String() == "" { + // status doesn't reply to anything + return nil + } + + // get the ancestor status into our database if we don't have it yet + if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + // now enrich the current status, since we should have the ancestor in the db + if _, err := d.EnrichRemoteStatus(username, status); err != nil { + l.Debugf("error enriching remote status: %s", err) + return nil + } + + // now move up to the next ancestor + return d.iterateAncestors(username, *inReplyTo) +} + +func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateDescendants", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateDescendants") + + // if it's our status we already have descendants stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + replies := statusable.GetActivityStreamsReplies() + if replies == nil || !replies.IsActivityStreamsCollection() { + l.Debug("no replies, bailing") + return nil + } + + repliesCollection := replies.GetActivityStreamsCollection() + if repliesCollection == nil { + l.Debug("replies collection is nil, bailing") + return nil + } + + first := repliesCollection.GetActivityStreamsFirst() + if first == nil { + l.Debug("replies collection has no first, bailing") + return nil + } + + firstPage := first.GetActivityStreamsCollectionPage() + if firstPage == nil { + l.Debug("first has no collection page, bailing") + return nil + } + + firstPageNext := firstPage.GetActivityStreamsNext() + if firstPageNext == nil || !firstPageNext.IsIRI() { + l.Debug("next is not an iri, bailing") + return nil + } + + var foundReplies int + currentPageIRI := firstPageNext.GetIRI() + +pageLoop: + for { + l.Debugf("dereferencing page %s", currentPageIRI) + nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI) + if err != nil { + return nil + } + + // next items could be either a list of URLs or a list of statuses + + nextItems := nextPage.GetActivityStreamsItems() + if nextItems.Len() == 0 { + // no items on this page, which means we're done + break pageLoop + } + + // have a look through items and see what we can find + for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() { + // We're looking for a url to feed to GetRemoteStatus. + // Items can be either an IRI, or a Note. + // If a note, we grab the ID from it and call it, rather than parsing the note. + + var itemURI *url.URL + if iter.IsIRI() { + // iri, easy + itemURI = iter.GetIRI() + } else if iter.IsActivityStreamsNote() { + // note, get the id from it to use as iri + n := iter.GetActivityStreamsNote() + id := n.GetJSONLDId() + if id != nil && id.IsIRI() { + itemURI = id.GetIRI() + } + } else { + // if it's not an iri or a note, we don't know how to process it + continue + } + + if itemURI.Host == d.config.Host { + // skip if the reply is from us -- we already have it then + continue + } + + // we can confidently say now that we found something + foundReplies = foundReplies + 1 + + // get the remote statusable and put it in the db + _, statusable, new, err := d.GetRemoteStatus(username, itemURI, false) + if new && err == nil && statusable != nil { + // now iterate descendants of *that* status + if err := d.iterateDescendants(username, *itemURI, statusable); err != nil { + continue + } + } + } + + next := nextPage.GetActivityStreamsNext() + if next != nil && next.IsIRI() { + l.Debug("setting next page") + currentPageIRI = next.GetIRI() + } else { + l.Debug("no next page, bailing") + break pageLoop + } + } + + l.Debugf("foundReplies %d", foundReplies) + return nil +} diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index e4a4920c8..3f4e3e413 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -9,8 +9,8 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -78,7 +78,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { typeName == gtsmodel.ActivityStreamsPerson || typeName == gtsmodel.ActivityStreamsService { // it's an UPDATE to some kind of account - var accountable typeutils.Accountable + var accountable ap.Accountable switch asType.GetTypeName() { case gtsmodel.ActivityStreamsApplication: diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index 1acdb6cb1..9e21b43bf 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,7 +31,6 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -139,7 +138,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } // we don't have an entry for this instance yet so dereference it - i, err = f.DereferenceRemoteInstance(username, &url.URL{ + i, err = f.GetRemoteInstance(username, &url.URL{ Scheme: publicKeyOwnerURI.Scheme, Host: publicKeyOwnerURI.Host, }) @@ -153,51 +152,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil { - // there's been a proper error so return it - if _, ok := err.(db.ErrNoEntries); !ok { - return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - // we don't know this account (yet) so let's dereference it right now - person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI) - if err != nil { - return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - a, err := f.typeConverter.ASRepresentationToAccount(person, false) - if err != nil { - return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err) - } - - aID, err := id.NewRandomULID() - if err != nil { - return ctx, false, err - } - a.ID = aID - - if err := f.db.Put(a); err != nil { - l.Errorf("error inserting dereferenced remote account: %s", err) - } - - requestingAccount = a - - // send the newly dereferenced account into the processor channel for further async processing - fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey) - if fromFederatorChanI == nil { - l.Error("from federator channel wasn't set on context") - } - fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator) - if !ok { - l.Error("from federator channel was set on context but couldn't be parsed") - } - - fromFederatorChan <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } + requestingAccount, _, err := f.GetRemoteAccount(username, publicKeyOwnerURI, false) + if err != nil { + return nil, false, fmt.Errorf("couldn't get remote account: %s", err) } withRequester := context.WithValue(ctx, util.APRequestingAccount, requestingAccount) diff --git a/internal/federation/federator.go b/internal/federation/federator.go index a5ffb3de8..ea9e61831 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -21,12 +21,13 @@ package federation import ( "context" "net/url" - "sync" "github.com/go-fed/activity/pub" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -40,6 +41,7 @@ type Federator interface { FederatingActor() pub.FederatingActor // FederatingDB returns the underlying FederatingDB interface. FederatingDB() federatingdb.DB + // AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources. // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. // @@ -49,29 +51,21 @@ type Federator interface { // // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error) + // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that // account, or an error if it doesn't exist or can't be retrieved. FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) - // DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) - // DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) - // DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then - // does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo - DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - // DereferenceStatusFields does further dereferencing on a status. - DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error - // DereferenceAccountFields does further dereferencing on an account. - DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error - // DereferenceAnnounce does further dereferencing on an announce. + + DereferenceRemoteThread(username string, statusURI *url.URL) error DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error - // GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username. - // This can be used for making signed http requests. - // - // If username is an empty string, our instance user's credentials will be used instead. - GetTransportForUser(username string) (transport.Transport, error) + + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + // Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID. Handshaking(username string, remoteAccountID *url.URL) bool pub.CommonBehavior @@ -85,16 +79,17 @@ type federator struct { clock pub.Clock typeConverter typeutils.TypeConverter transportController transport.Controller + dereferencer dereferencing.Dereferencer mediaHandler media.Handler actor pub.FederatingActor log *logrus.Logger - handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewFederator returns a new federator func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator { + dereferencer := dereferencing.NewDereferencer(config, db, typeConverter, transportController, mediaHandler, log) + clock := &Clock{} f := &federator{ config: config, @@ -103,9 +98,9 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr clock: &Clock{}, typeConverter: typeConverter, transportController: transportController, + dereferencer: dereferencer, mediaHandler: mediaHandler, log: log, - handshakeSync: &sync.Mutex{}, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 4ba0796cd..d74070487 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -69,7 +69,7 @@ func (suite *ProtocolTestSuite) SetupSuite() { } func (suite *ProtocolTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, suite.accounts) } @@ -87,7 +87,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { // setup transport controller with a no-op client so we don't make external calls tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { return nil, nil - })) + }), suite.db) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) @@ -152,7 +152,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { StatusCode: 200, Body: r, }, nil - })) + }), suite.db) // now setup module being tested, with the mock transport controller federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) diff --git a/internal/federation/finger.go b/internal/federation/finger.go index 6c6e9f6dc..0ffc60e5a 100644 --- a/internal/federation/finger.go +++ b/internal/federation/finger.go @@ -34,7 +34,7 @@ func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsernam return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain) } - t, err := f.GetTransportForUser(requestingUsername) + t, err := f.transportController.NewTransportForUsername(requestingUsername) if err != nil { return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) } diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go index 511e3e174..47c8a6c84 100644 --- a/internal/federation/handshake.go +++ b/internal/federation/handshake.go @@ -3,78 +3,5 @@ package federation import "net/url" func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - // handshakes isn't even initialized yet so we can't be handshaking with anyone - return false - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // user isn't handshaking with anyone, bail - return false - } - - for _, id := range remoteIDs { - if id.String() == remoteAccountID.String() { - // we are currently handshaking with the remote account, yep - return true - } - } - - // didn't find it which means we're not handshaking - return false -} - -func (f *federator) startHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - // lazily initialize handshakes - if f.handshakes == nil { - f.handshakes = make(map[string][]*url.URL) - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet, so just add this entry and return - f.handshakes[username] = []*url.URL{remoteAccountID} - return - } - - // add the remote ID to the slice - remoteIDs = append(remoteIDs, remoteAccountID) - f.handshakes[username] = remoteIDs -} - -func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - return - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet anyway so just bail - return - } - - newRemoteIDs := []*url.URL{} - for _, id := range remoteIDs { - if id.String() != remoteAccountID.String() { - newRemoteIDs = append(newRemoteIDs, id) - } - } - - if len(newRemoteIDs) == 0 { - // there are no handshakes so just remove this user entry from the map and save a few bytes - delete(f.handshakes, username) - } else { - // there are still other handshakes ongoing - f.handshakes[username] = newRemoteIDs - } + return f.dereferencer.Handshaking(username, remoteAccountID) } diff --git a/internal/federation/transport.go b/internal/federation/transport.go index a92f66d25..ed28749a1 100644 --- a/internal/federation/transport.go +++ b/internal/federation/transport.go @@ -6,8 +6,6 @@ import ( "net/url" "github.com/go-fed/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -35,7 +33,6 @@ import ( // returned Transport so that any private credentials are able to be // garbage collected. func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) { - var username string var err error @@ -53,32 +50,5 @@ func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofe return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String()) } - account := >smodel.Account{} - if err := f.db.GetLocalAccountByUsername(username, account); err != nil { - return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err) - } - - return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey) -} - -func (f *federator) GetTransportForUser(username string) (transport.Transport, error) { - // We need an account to use to create a transport for dereferecing something. - // If a username has been given, we can fetch the account with that username and use it. - // Otherwise, we can take the instance account and use those credentials to make the request. - ourAccount := >smodel.Account{} - var u string - if username == "" { - u = f.config.Host - } else { - u = username - } - if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil { - return nil, fmt.Errorf("error getting account %s from db: %s", username, err) - } - - transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) - if err != nil { - return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) - } - return transport, nil + return f.transportController.NewTransportForUsername(username) } diff --git a/internal/gtsmodel/activitystreams.go b/internal/gtsmodel/activitystreams.go index 77c935c5f..5cd92015c 100644 --- a/internal/gtsmodel/activitystreams.go +++ b/internal/gtsmodel/activitystreams.go @@ -43,6 +43,10 @@ const ( ActivityStreamsTombstone = "Tombstone" // ActivityStreamsVideo https://www.w3.org/TR/activitystreams-vocabulary/#dfn-video ActivityStreamsVideo = "Video" + //ActivityStreamsCollection https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection + ActivityStreamsCollection = "Collection" + // ActivityStreamsCollectionPage https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage + ActivityStreamsCollectionPage = "CollectionPage" ) const ( diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index 524b9c3ef..106298bcd 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -18,7 +18,9 @@ package gtsmodel -import "time" +import ( + "time" +) // Status represents a user-created 'post' or 'status' in the database, either remote or local type Status struct { diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go index b937ace5b..d2994e246 100644 --- a/internal/processing/account/get.go +++ b/internal/processing/account/get.go @@ -36,15 +36,6 @@ func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID str return nil, fmt.Errorf("db error: %s", err) } - // lazily dereference things on the account if it hasn't been done yet - var requestingUsername string - if requestingAccount != nil { - requestingUsername = requestingAccount.Username - } - if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil { - p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err) - } - var blocked bool var err error if requestingAccount != nil { diff --git a/internal/processing/account/getfollowers.go b/internal/processing/account/getfollowers.go index bfc463d3f..0806a82c0 100644 --- a/internal/processing/account/getfollowers.go +++ b/internal/processing/account/getfollowers.go @@ -63,12 +63,6 @@ func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/account/getfollowing.go b/internal/processing/account/getfollowing.go index bb6a905f4..75e89dacb 100644 --- a/internal/processing/account/getfollowing.go +++ b/internal/processing/account/getfollowing.go @@ -63,12 +63,6 @@ func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/federation.go b/internal/processing/federation.go index 966dab08d..765fdf862 100644 --- a/internal/processing/federation.go +++ b/internal/processing/federation.go @@ -31,65 +31,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) -// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given -// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account -// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database, -// and passing it into the processor through a channel for further asynchronous processing. -func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) { - // OK now we can do the dereferencing part - // we might already have an entry for this account so check that first - requestingAccount := >smodel.Account{} - - err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount) - if err == nil { - // we do have it yay, return it - return requestingAccount, nil - } - - if _, ok := err.(db.ErrNoEntries); !ok { - // something has actually gone wrong so bail - return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // we just don't have an entry for this account yet - // what we do now should depend on our chosen federation method - // for now though, we'll just dereference it - // TODO: slow-fed - requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI) - if err != nil { - return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err) - } - - // convert it to our internal account representation - requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false) - if err != nil { - return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err) - } - - requestingAccountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - requestingAccount.ID = requestingAccountID - - if err := p.db.Put(requestingAccount); err != nil { - return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // put it in our channel to queue it for async processing - p.fromFederator <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } - - return requestingAccount, nil -} - func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} @@ -112,9 +56,9 @@ func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, r return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - // if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part + // if we're not already handshaking/dereferencing a remote account, dereference it now if !p.federator.Handshaking(requestedUsername, requestingAccountURI) { - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -158,7 +102,7 @@ func (p *processor) GetFediFollowers(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -203,7 +147,7 @@ func (p *processor) GetFediFollowing(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -248,7 +192,7 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -295,6 +239,139 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return data, nil } +func (p *processor) GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) { + // get the account the request is referring to + requestedAccount := >smodel.Account{} + if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err)) + } + + // authenticate the request + requestingAccountURI, authenticated, err := p.federator.AuthenticateFederatedRequest(ctx, requestedUsername) + if err != nil || !authenticated { + return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") + } + + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) + if err != nil { + return nil, gtserror.NewErrorNotAuthorized(err) + } + + // authorize the request: + // 1. check if a block exists between the requester and the requestee + blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + if blocked { + return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID)) + } + + // get the status out of the database here + s := >smodel.Status{} + if err := p.db.GetWhere([]db.Where{ + {Key: "id", Value: requestedStatusID}, + {Key: "account_id", Value: requestedAccount.ID}, + }, s); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err)) + } + + visible, err := p.filter.StatusVisible(s, requestingAccount) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + if !visible { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("status with id %s not visible to user with id %s", s.ID, requestingAccount.ID)) + } + + var data map[string]interface{} + + // now there are three scenarios: + // 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page. + // 2. we're asked for a page but only_other_accounts has not been set in the query -- so we should just return the first page of the collection, with no items. + // 3. we're asked for a page, and only_other_accounts has been set, and min_id has optionally been set -- so we need to return some actual items! + + if !page { + // scenario 1 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + data, err = streams.Serialize(collection) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else if page && requestURL.Query().Get("only_other_accounts") == "" { + // scenario 2 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + // but only return the first page + data, err = streams.Serialize(collection.GetActivityStreamsFirst().GetActivityStreamsCollectionPage()) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else { + // scenario 3 + // get immediate children + replies, err := p.db.StatusChildren(s, true, minID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + // filter children and extract URIs + replyURIs := map[string]*url.URL{} + for _, r := range replies { + // only show public or unlocked statuses as replies + if r.Visibility != gtsmodel.VisibilityPublic && r.Visibility != gtsmodel.VisibilityUnlocked { + continue + } + + // respect onlyOtherAccounts parameter + if onlyOtherAccounts && r.AccountID == requestedAccount.ID { + continue + } + + // only show replies that the status owner can see + visibleToStatusOwner, err := p.filter.StatusVisible(r, requestedAccount) + if err != nil || !visibleToStatusOwner { + continue + } + + // only show replies that the requester can see + visibleToRequester, err := p.filter.StatusVisible(r, requestingAccount) + if err != nil || !visibleToRequester { + continue + } + + rURI, err := url.Parse(r.URI) + if err != nil { + continue + } + + replyURIs[r.ID] = rURI + } + + repliesPage, err := p.tc.StatusURIsToASRepliesPage(s, onlyOtherAccounts, minID, replyURIs) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + data, err = streams.Serialize(repliesPage) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } + + return data, nil +} + func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 94a4e5af8..949a734c7 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -21,6 +21,7 @@ package processing import ( "errors" "fmt" + "net/url" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -47,36 +48,21 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("note was not parseable as *gtsmodel.Status") } - l.Trace("will now derefence incoming status") - if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil { - return fmt.Errorf("error dereferencing status from federator: %s", err) - } - if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil { - return fmt.Errorf("error updating dereferenced status in the db: %s", err) + status, err := p.federator.EnrichRemoteStatus(federatorMsg.ReceivingAccount.Username, incomingStatus) + if err != nil { + return err } - if err := p.timelineStatus(incomingStatus); err != nil { + if err := p.timelineStatus(status); err != nil { return err } - if err := p.notifyStatus(incomingStatus); err != nil { + if err := p.notifyStatus(status); err != nil { return err } - case gtsmodel.ActivityStreamsProfile: // CREATE AN ACCOUNT - incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account) - if !ok { - return errors.New("profile was not parseable as *gtsmodel.Account") - } - - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) - } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) - } + // nothing to do here case gtsmodel.ActivityStreamsLike: // CREATE A FAVE incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave) @@ -154,12 +140,13 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("profile was not parseable as *gtsmodel.Account") } - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) + incomingAccountURI, err := url.Parse(incomingAccount.URI) + if err != nil { + return err } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) + + if _, _, err := p.federator.GetRemoteAccount(federatorMsg.ReceivingAccount.Username, incomingAccountURI, true); err != nil { + return fmt.Errorf("error dereferencing account from federator: %s", err) } } case gtsmodel.ActivityStreamsDelete: diff --git a/internal/processing/processor.go b/internal/processing/processor.go index a09a370e9..16f9ac2a3 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -191,6 +191,10 @@ type Processor interface { // authentication before returning a JSON serializable interface to the caller. GetFediStatus(ctx context.Context, requestedUsername string, requestedStatusID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetFediStatus handles the getting of a fedi/activitypub representation of replies to a status, performing appropriate + // authentication before returning a JSON serializable interface to the caller. + GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups. GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) diff --git a/internal/processing/search.go b/internal/processing/search.go index 727ad13bd..737ad8f71 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -19,7 +19,6 @@ package processing import ( - "errors" "fmt" "net/url" "strings" @@ -29,7 +28,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -122,6 +120,11 @@ func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQu } func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) { + l := p.log.WithFields(logrus.Fields{ + "func": "searchStatusByURI", + "uri": uri.String(), + "resolve": resolve, + }) maybeStatus := >smodel.Status{} if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil { @@ -134,57 +137,12 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve // we don't have it locally so dereference it if we're allowed to if resolve { - statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri) + status, _, _, err := p.federator.GetRemoteStatus(authed.Account.Username, uri, true) if err == nil { - // it IS a status! - - // extract the status owner's IRI from the statusable - var statusOwnerURI *url.URL - statusAttributedTo := statusable.GetActivityStreamsAttributedTo() - for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() { - if i.IsIRI() { - statusOwnerURI = i.GetIRI() - break - } - } - if statusOwnerURI == nil { - return nil, errors.New("couldn't extract ownerAccountURI from statusable") - } - - // make sure the status owner exists in the db by searching for it - _, err := p.searchAccountByURI(authed, statusOwnerURI, resolve) - if err != nil { - return nil, err - } - - // we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly - - // first turn it into a gtsmodel.Status - status, err := p.tc.ASStatusToStatus(statusable) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - statusID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return nil, err + if err := p.federator.DereferenceRemoteThread(authed.Account.Username, uri); err != nil { + // try to deref the thread while we're here + l.Debugf("searchStatusByURI: error dereferencing remote thread: %s", err) } - status.ID = statusID - - if err := p.db.Put(status); err != nil { - return nil, fmt.Errorf("error putting status in the db: %s", err) - } - - // properly dereference everything in the status (media attachments etc) - if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil { - return nil, fmt.Errorf("error dereferencing status fields: %s", err) - } - - // update with the nicely dereferenced status - if err := p.db.UpdateByID(status.ID, status); err != nil { - return nil, fmt.Errorf("error updating status in the db: %s", err) - } - return status, nil } } @@ -202,31 +160,10 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve } if resolve { // we don't have it locally so try and dereference it - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri) - if err != nil { - return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) - } - - // it IS an account! - account, err := p.tc.ASRepresentationToAccount(accountable, false) + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, uri, true) if err != nil { return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) } - - accountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - account.ID = accountID - - if err := p.db.Put(account); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err) - } - - if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err) - } - return account, nil } return nil, nil @@ -275,35 +212,12 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err) } - // dereference the account based on the URI we retrieved from the webfinger lookup - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI) - if err != nil { - // something went wrong doing the dereferencing so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err) - } - - // convert the dereferenced account to the gts model of that account - foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false) - if err != nil { - // something went wrong doing the conversion to a gtsmodel.Account so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err) - } - - foundAccountID, err := id.NewULID() + // we don't have it locally so try and dereference it + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, acctURI, true) if err != nil { - return nil, err - } - foundAccount.ID = foundAccountID - - // put this new account in our database - if err := p.db.Put(foundAccount); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err) - } - - // properly dereference all the fields on the account immediately - if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err) + return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err) } + return account, nil } return nil, nil diff --git a/internal/processing/status/context.go b/internal/processing/status/context.go index 72b9b5623..32c528296 100644 --- a/internal/processing/status/context.go +++ b/internal/processing/status/context.go @@ -33,7 +33,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return nil, gtserror.NewErrorForbidden(fmt.Errorf("account with id %s does not have permission to view status %s", account.ID, targetStatusID)) } - parents, err := p.db.StatusParents(targetStatus) + parents, err := p.db.StatusParents(targetStatus, false) if err != nil { return nil, gtserror.NewErrorInternalError(err) } @@ -51,7 +51,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return context.Ancestors[i].ID < context.Ancestors[j].ID }) - children, err := p.db.StatusChildren(targetStatus) + children, err := p.db.StatusChildren(targetStatus, false, "") if err != nil { return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/text/link_test.go b/internal/text/link_test.go index 15e27f870..83c42f045 100644 --- a/internal/text/link_test.go +++ b/internal/text/link_test.go @@ -86,7 +86,7 @@ func (suite *LinkTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *LinkTestSuite) TearDownTest() { diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go index 1e0d1471a..183ccc478 100644 --- a/internal/text/plain_test.go +++ b/internal/text/plain_test.go @@ -57,7 +57,7 @@ func (suite *PlainTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *PlainTestSuite) TearDownTest() { diff --git a/internal/transport/controller.go b/internal/transport/controller.go index c01af0900..07d20cdcf 100644 --- a/internal/transport/controller.go +++ b/internal/transport/controller.go @@ -27,15 +27,19 @@ import ( "github.com/go-fed/httpsig" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) // Controller generates transports for use in making federation requests to other servers. type Controller interface { NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) + NewTransportForUsername(username string) (Transport, error) } type controller struct { config *config.Config + db db.DB clock pub.Clock client pub.HttpClient appAgent string @@ -43,9 +47,10 @@ type controller struct { } // NewController returns an implementation of the Controller interface for creating new transports -func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { +func NewController(config *config.Config, db db.DB, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { return &controller{ config: config, + db: db, clock: clock, client: client, appAgent: fmt.Sprintf("%s %s", config.ApplicationName, config.Host), @@ -55,10 +60,10 @@ func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient // NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key. func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) { - prefs := []httpsig.Algorithm{httpsig.RSA_SHA256, httpsig.RSA_SHA512} + prefs := []httpsig.Algorithm{httpsig.RSA_SHA512} digestAlgo := httpsig.DigestSha256 - getHeaders := []string{"(request-target)", "host", "date"} - postHeaders := []string{"(request-target)", "host", "date", "digest"} + getHeaders := []string{httpsig.RequestTarget, "host", "date"} + postHeaders := []string{httpsig.RequestTarget, "host", "date", "digest"} getSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, getHeaders, httpsig.Signature, 120) if err != nil { @@ -85,3 +90,25 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T log: c.log, }, nil } + +func (c *controller) NewTransportForUsername(username string) (Transport, error) { + // We need an account to use to create a transport for dereferecing something. + // If a username has been given, we can fetch the account with that username and use it. + // Otherwise, we can take the instance account and use those credentials to make the request. + ourAccount := >smodel.Account{} + var u string + if username == "" { + u = c.config.Host + } else { + u = username + } + if err := c.db.GetLocalAccountByUsername(u, ourAccount); err != nil { + return nil, fmt.Errorf("error getting account %s from db: %s", username, err) + } + + transport, err := c.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) + if err != nil { + return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) + } + return transport, nil +} diff --git a/internal/typeutils/asextractionutil.go b/internal/typeutils/asextractionutil.go deleted file mode 100644 index b3e6eb2c4..000000000 --- a/internal/typeutils/asextractionutil.go +++ /dev/null @@ -1,570 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package typeutils - -import ( - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "net/url" - "strings" - "time" - - "github.com/go-fed/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/util" -) - -func extractPreferredUsername(i withPreferredUsername) (string, error) { - u := i.GetActivityStreamsPreferredUsername() - if u == nil || !u.IsXMLSchemaString() { - return "", errors.New("preferredUsername was not a string") - } - if u.GetXMLSchemaString() == "" { - return "", errors.New("preferredUsername was empty") - } - return u.GetXMLSchemaString(), nil -} - -func extractName(i withName) (string, error) { - nameProp := i.GetActivityStreamsName() - if nameProp == nil { - return "", errors.New("activityStreamsName not found") - } - - // take the first name string we can find - for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() { - if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { - return iter.GetXMLSchemaString(), nil - } - } - - return "", errors.New("activityStreamsName not found") -} - -func extractInReplyToURI(i withInReplyTo) (*url.URL, error) { - inReplyToProp := i.GetActivityStreamsInReplyTo() - if inReplyToProp == nil { - return nil, errors.New("in reply to prop was nil") - } - for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() { - if iter.IsIRI() { - if iter.GetIRI() != nil { - return iter.GetIRI(), nil - } - } - } - return nil, errors.New("couldn't find iri for in reply to") -} - -func extractTos(i withTo) ([]*url.URL, error) { - to := []*url.URL{} - toProp := i.GetActivityStreamsTo() - if toProp == nil { - return nil, errors.New("toProp was nil") - } - for iter := toProp.Begin(); iter != toProp.End(); iter = iter.Next() { - if iter.IsIRI() { - if iter.GetIRI() != nil { - to = append(to, iter.GetIRI()) - } - } - } - return to, nil -} - -func extractCCs(i withCC) ([]*url.URL, error) { - cc := []*url.URL{} - ccProp := i.GetActivityStreamsCc() - if ccProp == nil { - return cc, nil - } - for iter := ccProp.Begin(); iter != ccProp.End(); iter = iter.Next() { - if iter.IsIRI() { - if iter.GetIRI() != nil { - cc = append(cc, iter.GetIRI()) - } - } - } - return cc, nil -} - -func extractAttributedTo(i withAttributedTo) (*url.URL, error) { - attributedToProp := i.GetActivityStreamsAttributedTo() - if attributedToProp == nil { - return nil, errors.New("attributedToProp was nil") - } - for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() { - if iter.IsIRI() { - if iter.GetIRI() != nil { - return iter.GetIRI(), nil - } - } - } - return nil, errors.New("couldn't find iri for attributed to") -} - -func extractPublished(i withPublished) (time.Time, error) { - publishedProp := i.GetActivityStreamsPublished() - if publishedProp == nil { - return time.Time{}, errors.New("published prop was nil") - } - - if !publishedProp.IsXMLSchemaDateTime() { - return time.Time{}, errors.New("published prop was not date time") - } - - t := publishedProp.Get() - if t.IsZero() { - return time.Time{}, errors.New("published time was zero") - } - return t, nil -} - -// extractIconURL extracts a URL to a supported image file from something like: -// "icon": { -// "mediaType": "image/jpeg", -// "type": "Image", -// "url": "http://example.org/path/to/some/file.jpeg" -// }, -func extractIconURL(i withIcon) (*url.URL, error) { - iconProp := i.GetActivityStreamsIcon() - if iconProp == nil { - return nil, errors.New("icon property was nil") - } - - // icon can potentially contain multiple entries, so we iterate through all of them - // here in order to find the first one that meets these criteria: - // 1. is an image - // 2. has a URL so we can grab it - for iter := iconProp.Begin(); iter != iconProp.End(); iter = iter.Next() { - // 1. is an image - if !iter.IsActivityStreamsImage() { - continue - } - imageValue := iter.GetActivityStreamsImage() - if imageValue == nil { - continue - } - - // 2. has a URL so we can grab it - url, err := extractURL(imageValue) - if err == nil && url != nil { - return url, nil - } - } - // if we get to this point we didn't find an icon meeting our criteria :'( - return nil, errors.New("could not extract valid image from icon") -} - -// extractImageURL extracts a URL to a supported image file from something like: -// "image": { -// "mediaType": "image/jpeg", -// "type": "Image", -// "url": "http://example.org/path/to/some/file.jpeg" -// }, -func extractImageURL(i withImage) (*url.URL, error) { - imageProp := i.GetActivityStreamsImage() - if imageProp == nil { - return nil, errors.New("icon property was nil") - } - - // icon can potentially contain multiple entries, so we iterate through all of them - // here in order to find the first one that meets these criteria: - // 1. is an image - // 2. has a URL so we can grab it - for iter := imageProp.Begin(); iter != imageProp.End(); iter = iter.Next() { - // 1. is an image - if !iter.IsActivityStreamsImage() { - continue - } - imageValue := iter.GetActivityStreamsImage() - if imageValue == nil { - continue - } - - // 2. has a URL so we can grab it - url, err := extractURL(imageValue) - if err == nil && url != nil { - return url, nil - } - } - // if we get to this point we didn't find an image meeting our criteria :'( - return nil, errors.New("could not extract valid image from image property") -} - -func extractSummary(i withSummary) (string, error) { - summaryProp := i.GetActivityStreamsSummary() - if summaryProp == nil { - return "", errors.New("summary property was nil") - } - - for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() { - if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { - return iter.GetXMLSchemaString(), nil - } - } - - return "", errors.New("could not extract summary") -} - -func extractDiscoverable(i withDiscoverable) (bool, error) { - if i.GetTootDiscoverable() == nil { - return false, errors.New("discoverable was nil") - } - return i.GetTootDiscoverable().Get(), nil -} - -func extractURL(i withURL) (*url.URL, error) { - urlProp := i.GetActivityStreamsUrl() - if urlProp == nil { - return nil, errors.New("url property was nil") - } - - for iter := urlProp.Begin(); iter != urlProp.End(); iter = iter.Next() { - if iter.IsIRI() && iter.GetIRI() != nil { - return iter.GetIRI(), nil - } - } - - return nil, errors.New("could not extract url") -} - -func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { - publicKeyProp := i.GetW3IDSecurityV1PublicKey() - if publicKeyProp == nil { - return nil, nil, errors.New("public key property was nil") - } - - for iter := publicKeyProp.Begin(); iter != publicKeyProp.End(); iter = iter.Next() { - pkey := iter.Get() - if pkey == nil { - continue - } - - pkeyID, err := pub.GetId(pkey) - if err != nil || pkeyID == nil { - continue - } - - if pkey.GetW3IDSecurityV1Owner() == nil || pkey.GetW3IDSecurityV1Owner().Get() == nil || pkey.GetW3IDSecurityV1Owner().Get().String() != forOwner.String() { - continue - } - - if pkey.GetW3IDSecurityV1PublicKeyPem() == nil { - continue - } - - pkeyPem := pkey.GetW3IDSecurityV1PublicKeyPem().Get() - if pkeyPem == "" { - continue - } - - block, _ := pem.Decode([]byte(pkeyPem)) - if block == nil || block.Type != "PUBLIC KEY" { - return nil, nil, errors.New("could not decode publicKeyPem to PUBLIC KEY pem block type") - } - - p, err := x509.ParsePKIXPublicKey(block.Bytes) - if err != nil { - return nil, nil, fmt.Errorf("could not parse public key from block bytes: %s", err) - } - if p == nil { - return nil, nil, errors.New("returned public key was empty") - } - - if publicKey, ok := p.(*rsa.PublicKey); ok { - return publicKey, pkeyID, nil - } - } - return nil, nil, errors.New("couldn't find public key") -} - -func extractContent(i withContent) (string, error) { - contentProperty := i.GetActivityStreamsContent() - if contentProperty == nil { - return "", errors.New("content property was nil") - } - for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() { - if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { - return iter.GetXMLSchemaString(), nil - } - } - return "", errors.New("no content found") -} - -func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { - attachments := []*gtsmodel.MediaAttachment{} - attachmentProp := i.GetActivityStreamsAttachment() - if attachmentProp == nil { - return attachments, nil - } - for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() { - t := iter.GetType() - if t == nil { - continue - } - attachmentable, ok := t.(Attachmentable) - if !ok { - continue - } - attachment, err := extractAttachment(attachmentable) - if err != nil { - continue - } - attachments = append(attachments, attachment) - } - return attachments, nil -} - -func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { - attachment := >smodel.MediaAttachment{ - File: gtsmodel.File{}, - } - - attachmentURL, err := extractURL(i) - if err != nil { - return nil, err - } - attachment.RemoteURL = attachmentURL.String() - - mediaType := i.GetActivityStreamsMediaType() - if mediaType == nil { - return nil, errors.New("no media type") - } - if mediaType.Get() == "" { - return nil, errors.New("no media type") - } - attachment.File.ContentType = mediaType.Get() - attachment.Type = gtsmodel.FileTypeImage - - name, err := extractName(i) - if err == nil { - attachment.Description = name - } - - attachment.Processing = gtsmodel.ProcessingStatusReceived - - return attachment, nil -} - -// func extractBlurhash(i withBlurhash) (string, error) { -// if i.GetTootBlurhashProperty() == nil { -// return "", errors.New("blurhash property was nil") -// } -// if i.GetTootBlurhashProperty().Get() == "" { -// return "", errors.New("empty blurhash string") -// } -// return i.GetTootBlurhashProperty().Get(), nil -// } - -func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { - tags := []*gtsmodel.Tag{} - tagsProp := i.GetActivityStreamsTag() - if tagsProp == nil { - return tags, nil - } - for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { - t := iter.GetType() - if t == nil { - continue - } - - if t.GetTypeName() != "Hashtag" { - continue - } - - hashtaggable, ok := t.(Hashtaggable) - if !ok { - continue - } - - tag, err := extractHashtag(hashtaggable) - if err != nil { - continue - } - - tags = append(tags, tag) - } - return tags, nil -} - -func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { - tag := >smodel.Tag{} - - hrefProp := i.GetActivityStreamsHref() - if hrefProp == nil || !hrefProp.IsIRI() { - return nil, errors.New("no href prop") - } - tag.URL = hrefProp.GetIRI().String() - - name, err := extractName(i) - if err != nil { - return nil, err - } - tag.Name = strings.TrimPrefix(name, "#") - - return tag, nil -} - -func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { - emojis := []*gtsmodel.Emoji{} - tagsProp := i.GetActivityStreamsTag() - if tagsProp == nil { - return emojis, nil - } - for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { - t := iter.GetType() - if t == nil { - continue - } - - if t.GetTypeName() != "Emoji" { - continue - } - - emojiable, ok := t.(Emojiable) - if !ok { - continue - } - - emoji, err := extractEmoji(emojiable) - if err != nil { - continue - } - - emojis = append(emojis, emoji) - } - return emojis, nil -} - -func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { - emoji := >smodel.Emoji{} - - idProp := i.GetJSONLDId() - if idProp == nil || !idProp.IsIRI() { - return nil, errors.New("no id for emoji") - } - uri := idProp.GetIRI() - emoji.URI = uri.String() - emoji.Domain = uri.Host - - name, err := extractName(i) - if err != nil { - return nil, err - } - emoji.Shortcode = strings.Trim(name, ":") - - if i.GetActivityStreamsIcon() == nil { - return nil, errors.New("no icon for emoji") - } - imageURL, err := extractIconURL(i) - if err != nil { - return nil, errors.New("no url for emoji image") - } - emoji.ImageRemoteURL = imageURL.String() - - return emoji, nil -} - -func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { - mentions := []*gtsmodel.Mention{} - tagsProp := i.GetActivityStreamsTag() - if tagsProp == nil { - return mentions, nil - } - for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() { - t := iter.GetType() - if t == nil { - continue - } - - if t.GetTypeName() != "Mention" { - continue - } - - mentionable, ok := t.(Mentionable) - if !ok { - continue - } - - mention, err := extractMention(mentionable) - if err != nil { - continue - } - - mentions = append(mentions, mention) - } - return mentions, nil -} - -func extractMention(i Mentionable) (*gtsmodel.Mention, error) { - mention := >smodel.Mention{} - - mentionString, err := extractName(i) - if err != nil { - return nil, err - } - - // just make sure the mention string is valid so we can handle it properly later on... - username, domain, err := util.ExtractMentionParts(mentionString) - if err != nil { - return nil, err - } - if username == "" || domain == "" { - return nil, errors.New("username or domain was empty") - } - mention.NameString = mentionString - - // the href prop should be the AP URI of a user we know, eg https://example.org/users/whatever_user - hrefProp := i.GetActivityStreamsHref() - if hrefProp == nil || !hrefProp.IsIRI() { - return nil, errors.New("no href prop") - } - mention.MentionedAccountURI = hrefProp.GetIRI().String() - return mention, nil -} - -func extractActor(i withActor) (*url.URL, error) { - actorProp := i.GetActivityStreamsActor() - if actorProp == nil { - return nil, errors.New("actor property was nil") - } - for iter := actorProp.Begin(); iter != actorProp.End(); iter = iter.Next() { - if iter.IsIRI() && iter.GetIRI() != nil { - return iter.GetIRI(), nil - } - } - return nil, errors.New("no iri found for actor prop") -} - -func extractObject(i withObject) (*url.URL, error) { - objectProp := i.GetActivityStreamsObject() - if objectProp == nil { - return nil, errors.New("object property was nil") - } - for iter := objectProp.Begin(); iter != objectProp.End(); iter = iter.Next() { - if iter.IsIRI() && iter.GetIRI() != nil { - return iter.GetIRI(), nil - } - } - return nil, errors.New("no iri found for object prop") -} diff --git a/internal/typeutils/asinterfaces.go b/internal/typeutils/asinterfaces.go deleted file mode 100644 index d0b1cf617..000000000 --- a/internal/typeutils/asinterfaces.go +++ /dev/null @@ -1,265 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package typeutils - -import "github.com/go-fed/activity/streams/vocab" - -// Accountable represents the minimum activitypub interface for representing an 'account'. -// This interface is fulfilled by: Person, Application, Organization, Service, and Group -type Accountable interface { - withJSONLDId - withTypeName - - withPreferredUsername - withIcon - withName - withImage - withSummary - withDiscoverable - withURL - withPublicKey - withInbox - withOutbox - withFollowing - withFollowers - withFeatured -} - -// Statusable represents the minimum activitypub interface for representing a 'status'. -// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile -type Statusable interface { - withJSONLDId - withTypeName - - withSummary - withInReplyTo - withPublished - withURL - withAttributedTo - withTo - withCC - withSensitive - withConversation - withContent - withAttachment - withTag - withReplies -} - -// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. -// This interface is fulfilled by: Audio, Document, Image, Video -type Attachmentable interface { - withTypeName - withMediaType - withURL - withName -} - -// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. -type Hashtaggable interface { - withTypeName - withHref - withName -} - -// Emojiable represents the minimum interface for an 'emoji' tag. -type Emojiable interface { - withJSONLDId - withTypeName - withName - withUpdated - withIcon -} - -// Mentionable represents the minimum interface for a 'mention' tag. -type Mentionable interface { - withName - withHref -} - -// Followable represents the minimum interface for an activitystreams 'follow' activity. -type Followable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Likeable represents the minimum interface for an activitystreams 'like' activity. -type Likeable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Blockable represents the minimum interface for an activitystreams 'block' activity. -type Blockable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Announceable represents the minimum interface for an activitystreams 'announce' activity. -type Announceable interface { - withJSONLDId - withTypeName - - withActor - withObject - withPublished - withTo - withCC -} - -type withJSONLDId interface { - GetJSONLDId() vocab.JSONLDIdProperty -} - -type withTypeName interface { - GetTypeName() string -} - -type withPreferredUsername interface { - GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty -} - -type withIcon interface { - GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty -} - -type withName interface { - GetActivityStreamsName() vocab.ActivityStreamsNameProperty -} - -type withImage interface { - GetActivityStreamsImage() vocab.ActivityStreamsImageProperty -} - -type withSummary interface { - GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty -} - -type withDiscoverable interface { - GetTootDiscoverable() vocab.TootDiscoverableProperty -} - -type withURL interface { - GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty -} - -type withPublicKey interface { - GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty -} - -type withInbox interface { - GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty -} - -type withOutbox interface { - GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty -} - -type withFollowing interface { - GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty -} - -type withFollowers interface { - GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty -} - -type withFeatured interface { - GetTootFeatured() vocab.TootFeaturedProperty -} - -type withAttributedTo interface { - GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty -} - -type withAttachment interface { - GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty -} - -type withTo interface { - GetActivityStreamsTo() vocab.ActivityStreamsToProperty -} - -type withInReplyTo interface { - GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty -} - -type withCC interface { - GetActivityStreamsCc() vocab.ActivityStreamsCcProperty -} - -type withSensitive interface { - // TODO -} - -type withConversation interface { - // TODO -} - -type withContent interface { - GetActivityStreamsContent() vocab.ActivityStreamsContentProperty -} - -type withPublished interface { - GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty -} - -type withTag interface { - GetActivityStreamsTag() vocab.ActivityStreamsTagProperty -} - -type withReplies interface { - GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty -} - -type withMediaType interface { - GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty -} - -// type withBlurhash interface { -// GetTootBlurhashProperty() vocab.TootBlurhashProperty -// } - -// type withFocalPoint interface { -// // TODO -// } - -type withHref interface { - GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty -} - -type withUpdated interface { - GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty -} - -type withActor interface { - GetActivityStreamsActor() vocab.ActivityStreamsActorProperty -} - -type withObject interface { - GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty -} diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 394de6e82..f754d282a 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -24,11 +24,12 @@ import ( "net/url" "strings" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (c *converter) ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) { +func (c *converter) ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) { // first check if we actually already know this account uriProp := accountable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { @@ -55,7 +56,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // Username aka preferredUsername // We need this one so bail if it's not set. - username, err := extractPreferredUsername(accountable) + username, err := ap.ExtractPreferredUsername(accountable) if err != nil { return nil, fmt.Errorf("couldn't extract username: %s", err) } @@ -66,27 +67,27 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // avatar aka icon // if this one isn't extractable in a format we recognise we'll just skip it - if avatarURL, err := extractIconURL(accountable); err == nil { + if avatarURL, err := ap.ExtractIconURL(accountable); err == nil { acct.AvatarRemoteURL = avatarURL.String() } // header aka image // if this one isn't extractable in a format we recognise we'll just skip it - if headerURL, err := extractImageURL(accountable); err == nil { + if headerURL, err := ap.ExtractImageURL(accountable); err == nil { acct.HeaderRemoteURL = headerURL.String() } // display name aka name // we default to the username, but take the more nuanced name property if it exists acct.DisplayName = username - if displayName, err := extractName(accountable); err == nil { + if displayName, err := ap.ExtractName(accountable); err == nil { acct.DisplayName = displayName } // TODO: fields aka attachment array // note aka summary - note, err := extractSummary(accountable) + note, err := ap.ExtractSummary(accountable) if err == nil && note != "" { acct.Note = note } @@ -110,13 +111,13 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // discoverable // default to false -- take custom value if it's set though acct.Discoverable = false - discoverable, err := extractDiscoverable(accountable) + discoverable, err := ap.ExtractDiscoverable(accountable) if err == nil { acct.Discoverable = discoverable } // url property - url, err := extractURL(accountable) + url, err := ap.ExtractURL(accountable) if err == nil { // take the URL if we can find it acct.URL = url.String() @@ -155,7 +156,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // TODO: alsoKnownAs // publicKey - pkey, pkeyURL, err := extractPublicKeyForOwner(accountable, uri) + pkey, pkeyURL, err := ap.ExtractPublicKeyForOwner(accountable, uri) if err != nil { return nil, fmt.Errorf("couldn't get public key for person %s: %s", uri.String(), err) } @@ -165,7 +166,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo return acct, nil } -func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) { +func (c *converter) ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) { status := >smodel.Status{} // uri at which this status is reachable @@ -176,49 +177,49 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.URI = uriProp.GetIRI().String() // web url for viewing this status - if statusURL, err := extractURL(statusable); err == nil { + if statusURL, err := ap.ExtractURL(statusable); err == nil { status.URL = statusURL.String() } // the html-formatted content of this status - if content, err := extractContent(statusable); err == nil { + if content, err := ap.ExtractContent(statusable); err == nil { status.Content = content } // attachments to dereference and fetch later on (we don't do that here) - if attachments, err := extractAttachments(statusable); err == nil { + if attachments, err := ap.ExtractAttachments(statusable); err == nil { status.GTSMediaAttachments = attachments } // hashtags to dereference later on - if hashtags, err := extractHashtags(statusable); err == nil { + if hashtags, err := ap.ExtractHashtags(statusable); err == nil { status.GTSTags = hashtags } // emojis to dereference and fetch later on - if emojis, err := extractEmojis(statusable); err == nil { + if emojis, err := ap.ExtractEmojis(statusable); err == nil { status.GTSEmojis = emojis } // mentions to dereference later on - if mentions, err := extractMentions(statusable); err == nil { + if mentions, err := ap.ExtractMentions(statusable); err == nil { status.GTSMentions = mentions } // cw string for this status - if cw, err := extractSummary(statusable); err == nil { + if cw, err := ap.ExtractSummary(statusable); err == nil { status.ContentWarning = cw } // when was this status created? - published, err := extractPublished(statusable) + published, err := ap.ExtractPublished(statusable) if err == nil { status.CreatedAt = published } // which account posted this status? // if we don't know the account yet we can dereference it later - attributedTo, err := extractAttributedTo(statusable) + attributedTo, err := ap.ExtractAttributedTo(statusable) if err != nil { return nil, errors.New("attributedTo was empty") } @@ -233,8 +234,8 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.GTSAuthorAccount = statusOwner // check if there's a post that this is a reply to - inReplyToURI, err := extractInReplyToURI(statusable) - if err == nil { + inReplyToURI := ap.ExtractInReplyToURI(statusable) + if inReplyToURI != nil { // something is set so we can at least set this field on the // status and dereference using this later if we need to status.InReplyToURI = inReplyToURI.String() @@ -259,12 +260,12 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e // visibility entry for this status var visibility gtsmodel.Visibility - to, err := extractTos(statusable) + to, err := ap.ExtractTos(statusable) if err != nil { return nil, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(statusable) + cc, err := ap.ExtractCCs(statusable) if err != nil { return nil, fmt.Errorf("error extracting CC values: %s", err) } @@ -315,7 +316,7 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e return status, nil } -func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) { +func (c *converter) ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { @@ -323,7 +324,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -332,7 +333,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -350,14 +351,14 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return followRequest, nil } -func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) { +func (c *converter) ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on follow, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -366,7 +367,7 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -384,14 +385,14 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return follow, nil } -func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) { +func (c *converter) ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) { idProp := likeable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on like, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(likeable) + origin, err := ap.ExtractActor(likeable) if err != nil { return nil, errors.New("error extracting actor property from like") } @@ -400,7 +401,7 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(likeable) + target, err := ap.ExtractObject(likeable) if err != nil { return nil, errors.New("error extracting object property from like") } @@ -426,14 +427,14 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error }, nil } -func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) { +func (c *converter) ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) { idProp := blockable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("ASBlockToBlock: no id property set on block, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(blockable) + origin, err := ap.ExtractActor(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting actor property from block") } @@ -442,7 +443,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) return nil, fmt.Errorf("ASBlockToBlock: error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(blockable) + target, err := ap.ExtractObject(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting object property from block") } @@ -461,7 +462,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) }, nil } -func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Status, bool, error) { +func (c *converter) ASAnnounceToStatus(announceable ap.Announceable) (*gtsmodel.Status, bool, error) { status := >smodel.Status{} isNew := true @@ -480,7 +481,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.URI = uri // get the URI of the announced/boosted status - boostedStatusURI, err := extractObject(announceable) + boostedStatusURI, err := ap.ExtractObject(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error getting object from announce: %s", err) } @@ -491,7 +492,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta } // get the published time for the announce - published, err := extractPublished(announceable) + published, err := ap.ExtractPublished(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting published time: %s", err) } @@ -499,7 +500,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.UpdatedAt = published // get the actor's IRI (ie., the person who boosted the status) - actor, err := extractActor(announceable) + actor, err := ap.ExtractActor(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting actor: %s", err) } @@ -522,12 +523,12 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta // parse the visibility from the To and CC entries var visibility gtsmodel.Visibility - to, err := extractTos(announceable) + to, err := ap.ExtractTos(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(announceable) + cc, err := ap.ExtractCCs(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting CC values: %s", err) } diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go index 9d6ce4e0a..2e33271c5 100644 --- a/internal/typeutils/astointernal_test.go +++ b/internal/typeutils/astointernal_test.go @@ -28,6 +28,7 @@ import ( "github.com/go-fed/activity/streams/vocab" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -342,7 +343,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() { } func (suite *ASToInternalTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *ASToInternalTestSuite) TestParsePerson() { @@ -364,7 +365,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() { t, err := streams.ToType(context.Background(), m) assert.NoError(suite.T(), err) - rep, ok := t.(typeutils.Accountable) + rep, ok := t.(ap.Accountable) assert.True(suite.T(), ok) acct, err := suite.typeconverter.ASRepresentationToAccount(rep, false) @@ -391,7 +392,7 @@ func (suite *ASToInternalTestSuite) TestParseStatus() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) @@ -418,7 +419,7 @@ func (suite *ASToInternalTestSuite) TestParseStatusWithMention() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 57c2a1f6d..10d9a0f18 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -19,7 +19,10 @@ package typeutils import ( + "net/url" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -99,17 +102,17 @@ type TypeConverter interface { // If update is false, and the account is already known in the database, then the existing account entry will be returned. // If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account // will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile. - ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) + ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) // ASStatus converts a remote activitystreams 'status' representation into a gts model status. - ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) + ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request. - ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) + ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow. - ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) + ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) // ASLikeToFave converts a remote activitystreams 'like' representation into a gts model status fave. - ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) + ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) // ASBlockToBlock converts a remote activity streams 'block' representation into a gts model block. - ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) + ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) // ASAnnounceToStatus converts an activitystreams 'announce' into a status. // // The returned bool indicates whether this status is new (true) or not new (false). @@ -122,7 +125,7 @@ type TypeConverter interface { // This is useful when multiple users on an instance might receive the same boost, and we only want to process the boost once. // // NOTE -- this is different from one status being boosted multiple times! In this case, new boosts should indeed be created. - ASAnnounceToStatus(announceable Announceable) (status *gtsmodel.Status, new bool, err error) + ASAnnounceToStatus(announceable ap.Announceable) (status *gtsmodel.Status, new bool, err error) /* INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL @@ -150,7 +153,10 @@ type TypeConverter interface { BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) // BlockToAS converts a gts model block into an activityStreams BLOCK, suitable for federation. BlockToAS(block *gtsmodel.Block) (vocab.ActivityStreamsBlock, error) - + // StatusToASRepliesCollection converts a gts model status into an activityStreams REPLIES collection. + StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) + // StatusURIsToASRepliesPage returns a collection page with appropriate next/part of pagination. + StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) /* INTERNAL (gts) MODEL TO INTERNAL MODEL */ diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index b2272f50c..c104ab06c 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -21,6 +21,7 @@ package typeutils_test import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -34,7 +35,7 @@ type ConverterStandardTestSuite struct { db db.DB log *logrus.Logger accounts map[string]*gtsmodel.Account - people map[string]typeutils.Accountable + people map[string]ap.Accountable typeconverter typeutils.TypeConverter } diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index b24b07e13..333f131d4 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -27,6 +27,7 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -505,7 +506,14 @@ func (c *converter) StatusToAS(s *gtsmodel.Status) (vocab.ActivityStreamsNote, e status.SetActivityStreamsAttachment(attachmentProp) // replies - // TODO + repliesCollection, err := c.StatusToASRepliesCollection(s, false) + if err != nil { + return nil, fmt.Errorf("error creating repliesCollection: %s", err) + } + + repliesProp := streams.NewActivityStreamsRepliesProperty() + repliesProp.SetActivityStreamsCollection(repliesCollection) + status.SetActivityStreamsReplies(repliesProp) return status, nil } @@ -850,3 +858,138 @@ func (c *converter) BlockToAS(b *gtsmodel.Block) (vocab.ActivityStreamsBlock, er return block, nil } + +/* + the goal is to end up with something like this: + + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "type": "Collection", + "first": { + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [] + } + } +*/ +func (c *converter) StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + + collection := streams.NewActivityStreamsCollection() + + // collection.id + collectionIDProp := streams.NewJSONLDIdProperty() + collectionIDProp.SetIRI(collectionIDURI) + collection.SetJSONLDId(collectionIDProp) + + // first + first := streams.NewActivityStreamsFirstProperty() + firstPage := streams.NewActivityStreamsCollectionPage() + + // first.id + firstPageIDProp := streams.NewJSONLDIdProperty() + firstPageID, err := url.Parse(fmt.Sprintf("%s?page=true", collectionID)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + firstPageIDProp.SetIRI(firstPageID) + firstPage.SetJSONLDId(firstPageIDProp) + + // first.next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropID, err := url.Parse(fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + firstPage.SetActivityStreamsNext(nextProp) + + // first.partOf + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + firstPage.SetActivityStreamsPartOf(partOfProp) + + first.SetActivityStreamsCollectionPage(firstPage) + + // collection.first + collection.SetActivityStreamsFirst(first) + + return collection, nil +} + +/* + the goal is to end up with something like this: + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?min_id=106720870266901180&only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [ + "https://example.com/users/someone/statuses/106720752853216226", + "https://somewhere.online/users/eeeeeeeeeep/statuses/106720870163727231" + ] + } +*/ +func (c *converter) StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + + page := streams.NewActivityStreamsCollectionPage() + + // .id + pageIDProp := streams.NewJSONLDIdProperty() + pageIDString := fmt.Sprintf("%s?page=true&only_other_accounts=%t", collectionID, onlyOtherAccounts) + if minID != "" { + pageIDString = fmt.Sprintf("%s&min_id=%s", pageIDString, minID) + } + + pageID, err := url.Parse(pageIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + pageIDProp.SetIRI(pageID) + page.SetJSONLDId(pageIDProp) + + // .partOf + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + page.SetActivityStreamsPartOf(partOfProp) + + // .items + items := streams.NewActivityStreamsItemsProperty() + var highestID string + for k, v := range replies { + items.AppendIRI(v) + if k > highestID { + highestID = k + } + } + page.SetActivityStreamsItems(items) + + // .next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropIDString := fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts) + if highestID != "" { + nextPropIDString = fmt.Sprintf("%s&min_id=%s", nextPropIDString, highestID) + } + + nextPropID, err := url.Parse(nextPropIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + page.SetActivityStreamsNext(nextProp) + + return page, nil +} diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 8eb827e35..caa56ce0d 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -47,7 +47,7 @@ func (suite *InternalToASTestSuite) SetupSuite() { } func (suite *InternalToASTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } // TearDownTest drops tables to make sure there's no data in the db -- cgit v1.2.3