diff options
Diffstat (limited to 'internal/federation')
-rw-r--r-- | internal/federation/dereference.go | 5 | ||||
-rw-r--r-- | internal/federation/dereferencing/account.go | 268 | ||||
-rw-r--r-- | internal/federation/dereferencing/account_test.go | 12 | ||||
-rw-r--r-- | internal/federation/dereferencing/dereferencer.go | 2 | ||||
-rw-r--r-- | internal/federation/dereferencing/dereferencer_test.go | 106 | ||||
-rw-r--r-- | internal/federation/dereferencing/finger.go | 80 | ||||
-rw-r--r-- | internal/federation/dereferencing/status.go | 10 | ||||
-rw-r--r-- | internal/federation/federatingactor_test.go | 43 | ||||
-rw-r--r-- | internal/federation/federatingdb/update.go | 2 | ||||
-rw-r--r-- | internal/federation/federatingprotocol.go | 6 | ||||
-rw-r--r-- | internal/federation/federatingprotocol_test.go | 36 | ||||
-rw-r--r-- | internal/federation/federator.go | 6 | ||||
-rw-r--r-- | internal/federation/finger.go | 72 |
13 files changed, 349 insertions, 299 deletions
diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go index 8efa0cc7e..705cdbd19 100644 --- a/internal/federation/dereference.go +++ b/internal/federation/dereference.go @@ -23,11 +23,12 @@ import ( "net/url" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { - return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh) +func (f *federator) GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) { + return f.dereferencer.GetRemoteAccount(ctx, params) } func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) { diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 7d5d80479..c479906d7 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -33,12 +33,15 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" ) +var webfingerInterval = -48 * time.Hour // 2 days in the past + func instanceAccount(account *gtsmodel.Account) bool { return strings.EqualFold(account.Username, account.Domain) || account.FollowersURI == "" || @@ -46,97 +49,238 @@ func instanceAccount(account *gtsmodel.Account) bool { (account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor")) } +// GetRemoteAccountParams wraps parameters for a remote account lookup. +type GetRemoteAccountParams struct { + // The username of the user doing the lookup request (optional). + // If not set, then the GtS instance account will be used to do the lookup. + RequestingUsername string + // The ActivityPub URI of the remote account (optional). + // If not set (nil), the ActivityPub URI of the remote account will be discovered + // via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost + // if this parameter is not set. + RemoteAccountID *url.URL + // The username of the remote account (optional). + // If RemoteAccountID is not set, then this value must be set. + RemoteAccountUsername string + // The host of the remote account (optional). + // If RemoteAccountID is not set, then this value must be set. + RemoteAccountHost string + // Whether to do a blocking call to the remote instance. If true, + // then the account's media and other fields will be fully dereferenced before it is returned. + // If false, then the account's media and other fields will be dereferenced in the background, + // so only a minimal account representation will be returned by GetRemoteAccount. + Blocking bool + // Whether to skip making calls to remote instances. This is useful when you want to + // quickly fetch a remote account from the database or fail, and don't want to cause + // http requests to go flying around. + SkipResolve bool +} + // GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, -// puts it in the database, and returns it to a caller. -// -// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling -// the remote instance again. Blocking indicates whether the function should block until processing of -// the fetched account is complete. -// -// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). -func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { - new := true - - // check if we already have the account in our db, and just return it unless we'd doing a refresh - remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String()) - if err == nil { - new = false - if !refresh { - // make sure the account fields are populated before returning: - // even if we're not doing a refresh, the caller might want to block - // until everything is loaded - changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking) +// puts or updates it in the database (if necessary), and returns it to a caller. +func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (remoteAccount *gtsmodel.Account, err error) { + + /* + In this function we want to retrieve a gtsmodel representation of a remote account, with its proper + accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth. + + There are a few different paths through this function, and the path taken depends on how much + initial information we are provided with via parameters, how much information we already have stored, + and what we're allowed to do according to the parameters we've been passed. + + Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the + account username + host, so we can check in our database and return if possible. + + Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host. + In this case, we can use the URI to resolve the remote account and find the username, + and then we can webfinger the account to discover the accountDomain if necessary. + + Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI. + In this case, we can webfinger the account to discover the URI, and then dereference + from that. + */ + + // first check if we can retrieve the account locally just with what we've been given + switch { + case params.RemoteAccountID != nil: + // try with uri + if a, dbErr := d.db.GetAccountByURI(ctx, params.RemoteAccountID.String()); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account %s: %s", params.RemoteAccountID, err) + } + case params.RemoteAccountUsername != "" && params.RemoteAccountHost != "": + // try with username/host + a := >smodel.Account{} + where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: params.RemoteAccountHost}} + if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err) + } + default: + err = errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account") + } + + if err != nil { + return + } + + if params.SkipResolve { + // if we can't resolve, return already since there's nothing more we can do + if remoteAccount == nil { + err = errors.New("GetRemoteAccount: error retrieving account with skipResolve set true") + } + return + } + + var accountable ap.Accountable + if params.RemoteAccountUsername == "" || params.RemoteAccountHost == "" { + // try to populate the missing params + // the first one is easy ... + params.RemoteAccountHost = params.RemoteAccountID.Host + // ... but we still need the username so we can do a finger for the accountDomain + + // check if we had the account stored already and got it earlier + if remoteAccount != nil { + params.RemoteAccountUsername = remoteAccount.Username + } else { + // if we didn't already have it, we have dereference it from remote and just... + accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) + err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + return } - if changed { - updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) - } - return updatedAccount, err + // ... take the username (for now) + params.RemoteAccountUsername, err = ap.ExtractPreferredUsername(accountable) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error extracting accountable username: %s", err) + return } - - return remoteAccount, nil } } - if new { - // we haven't seen this account before: dereference it from remote - accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) + // if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set + // params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it + + // we finger to fetch the account domain but just in case we're not fingering, make a best guess + // already about what the account domain might be; this var will be overwritten later if necessary + var accountDomain string + switch { + case remoteAccount != nil: + accountDomain = remoteAccount.Domain + case params.RemoteAccountID != nil: + accountDomain = params.RemoteAccountID.Host + default: + accountDomain = params.RemoteAccountHost + } + + // to save on remote calls: only webfinger if we don't have a remoteAccount yet, or if we haven't + // fingered the remote account for at least 2 days; don't finger instance accounts + var fingered time.Time + if remoteAccount == nil || (remoteAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(remoteAccount)) { + accountDomain, params.RemoteAccountID, err = d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + err = fmt.Errorf("GetRemoteAccount: error while fingering: %s", err) + return + } + fingered = time.Now() + } + + if !fingered.IsZero() && remoteAccount == nil { + // if we just fingered and now have a discovered account domain but still no account, + // we should do a final lookup in the database with the discovered username + accountDomain + // to make absolutely sure we don't already have this account + a := >smodel.Account{} + where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: accountDomain}} + if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err) + return + } + } + + // we may also have some extra information already, like the account we had in the db, or the + // accountable representation that we dereferenced from remote + if remoteAccount == nil { + // we still don't have the account, so deference it if we didn't earlier + if accountable == nil { + accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + return + } } - newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh) + // then convert + remoteAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err) + err = fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err) + return } - ulid, err := id.NewRandomULID() + // this is a new account so we need to generate a new ID for it + var ulid string + ulid, err = id.NewRandomULID() if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err) + err = fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err) + return } - newAccount.ID = ulid + remoteAccount.ID = ulid - if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) + _, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) + return } - if err := d.db.Put(ctx, newAccount); err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err) + remoteAccount.LastWebfingeredAt = fingered + remoteAccount.UpdatedAt = time.Now() + + err = d.db.Put(ctx, remoteAccount) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error putting new account: %s", err) + return } - return newAccount, nil + return // the new account } - // we have seen this account before, but we have to refresh it - refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err) + // we had the account already, but now we know the account domain, so update it if it's different + if !strings.EqualFold(remoteAccount.Domain, accountDomain) { + remoteAccount.Domain = accountDomain + remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error updating account: %s", err) + return + } } - refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh) + // make sure the account fields are populated before returning: + // the caller might want to block until everything is loaded + var fieldsChanged bool + fieldsChanged, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err) + return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) } - refreshedAccount.ID = remoteAccount.ID - changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err) + var fingeredChanged bool + if !fingered.IsZero() { + fingeredChanged = true + remoteAccount.LastWebfingeredAt = fingered } - if changed { - updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount) + if fieldsChanged || fingeredChanged { + remoteAccount.UpdatedAt = time.Now() + remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err) + return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) } - return updatedAccount, nil } - return refreshedAccount, nil + return // the account we already had + possibly updated } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -209,7 +353,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem // populateAccountFields populates any fields on the given account that weren't populated by the initial // dereferencing. This includes things like header and avatar etc. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) { +func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { // if we're dealing with an instance account, just bail, we don't need to do anything if instanceAccount(account) { return false, nil @@ -230,7 +374,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc } // fetch the header and avatar - changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking) + changed, err := d.fetchRemoteAccountMedia(ctx, account, t, blocking) if err != nil { return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err) } @@ -250,7 +394,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) { +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool) (bool, error) { changed := false accountURI, err := url.Parse(targetAccount.URI) @@ -262,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host) } - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { + if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") { var processingMedia *media.ProcessingMedia d.dereferencingAvatarsLock.Lock() // LOCK HERE @@ -320,7 +464,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm changed = true } - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { + if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") { var processingMedia *media.ProcessingMedia d.dereferencingHeadersLock.Lock() // LOCK HERE diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index 75c02af75..72092951b 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -35,7 +36,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() { fetchingAccount := suite.testAccounts["local_account_1"] groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group") - group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false) + group, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{ + RequestingUsername: fetchingAccount.Username, + RemoteAccountID: groupURL, + }) suite.NoError(err) suite.NotNil(group) suite.NotNil(group) @@ -55,7 +59,10 @@ func (suite *AccountTestSuite) TestDereferenceService() { fetchingAccount := suite.testAccounts["local_account_1"] serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh") - service, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, serviceURL, false, false) + service, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{ + RequestingUsername: fetchingAccount.Username, + RemoteAccountID: serviceURL, + }) suite.NoError(err) suite.NotNil(service) suite.NotNil(service) @@ -69,6 +76,7 @@ func (suite *AccountTestSuite) TestDereferenceService() { suite.NoError(err) suite.Equal(service.ID, dbService.ID) suite.Equal(ap.ActorService, dbService.ActorType) + suite.Equal("example.org", dbService.Domain) } func TestAccountTestSuite(t *testing.T) { diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index cae24d0fd..4f7559be3 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -33,7 +33,7 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index 96ec7869f..0f4732187 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -19,22 +19,14 @@ package dereferencing_test import ( - "bytes" - "encoding/json" - "io" - "net/http" - "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -66,106 +58,10 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage)) + suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } func (suite *DereferencerStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) } - -// mockTransportController returns basically a miniature muxer, which returns a different -// value based on the request URL. It can be used to return remote statuses, profiles, etc, -// as though they were actually being dereferenced. If the URL doesn't correspond to any person -// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body. -func (suite *DereferencerStandardTestSuite) mockTransportController() transport.Controller { - do := func(req *http.Request) (*http.Response, error) { - logrus.Debugf("received request for %s", req.URL) - - responseBytes := []byte{} - responseType := "" - responseLength := 0 - - if note, ok := suite.testRemoteStatuses[req.URL.String()]; ok { - // the request is for a note that we have stored - noteI, err := streams.Serialize(note) - if err != nil { - panic(err) - } - noteJson, err := json.Marshal(noteI) - if err != nil { - panic(err) - } - responseBytes = noteJson - responseType = "application/activity+json" - } - - if person, ok := suite.testRemotePeople[req.URL.String()]; ok { - // the request is for a person that we have stored - personI, err := streams.Serialize(person) - if err != nil { - panic(err) - } - personJson, err := json.Marshal(personI) - if err != nil { - panic(err) - } - responseBytes = personJson - responseType = "application/activity+json" - } - - if group, ok := suite.testRemoteGroups[req.URL.String()]; ok { - // the request is for a person that we have stored - groupI, err := streams.Serialize(group) - if err != nil { - panic(err) - } - groupJson, err := json.Marshal(groupI) - if err != nil { - panic(err) - } - responseBytes = groupJson - responseType = "application/activity+json" - } - - if service, ok := suite.testRemoteServices[req.URL.String()]; ok { - serviceI, err := streams.Serialize(service) - if err != nil { - panic(err) - } - serviceJson, err := json.Marshal(serviceI) - if err != nil { - panic(err) - } - responseBytes = serviceJson - responseType = "application/activity+json" - } - - if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok { - responseBytes = attachment.Data - responseType = attachment.ContentType - } - - if len(responseBytes) != 0 { - // we found something, so print what we're going to return - logrus.Debugf("returning response %s", string(responseBytes)) - } - responseLength = len(responseBytes) - - reader := bytes.NewReader(responseBytes) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(responseLength), - Header: http.Header{ - "content-type": {responseType}, - }, - } - - return response, nil - } - fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - mockClient := testrig.NewMockHTTPClient(do) - return testrig.NewTestTransportController(mockClient, suite.db, fedWorker) -} diff --git a/internal/federation/dereferencing/finger.go b/internal/federation/dereferencing/finger.go new file mode 100644 index 000000000..9613d2975 --- /dev/null +++ b/internal/federation/dereferencing/finger.go @@ -0,0 +1,80 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) { + t, err := d.transportController.NewTransportForUsername(ctx, username) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err) + return + } + + b, err := t.Finger(ctx, targetUsername, targetHost) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err) + return + } + + resp := &apimodel.WellKnownResponse{} + if err = json.Unmarshal(b, resp); err != nil { + err = fmt.Errorf("fingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse while dereferencing @%s@%s: %s", targetUsername, targetHost, err) + return + } + + if len(resp.Links) == 0 { + err = fmt.Errorf("fingerRemoteAccount: no links found in webfinger response %s", string(b)) + return + } + + if resp.Subject == "" { + err = fmt.Errorf("fingerRemoteAccount: no subject found in webfinger response %s", string(b)) + return + } + + _, accountDomain, err = util.ExtractWebfingerParts(resp.Subject) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error extracting webfinger subject parts: %s", err) + } + + // look through the links for the first one that matches what we need + for _, l := range resp.Links { + if l.Rel == "self" && (strings.EqualFold(l.Type, "application/activity+json") || strings.EqualFold(l.Type, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")) { + if uri, thiserr := url.Parse(l.Href); thiserr == nil && (uri.Scheme == "http" || uri.Scheme == "https") { + // found it! + accountURI = uri + return + } + } + } + + err = errors.New("fingerRemoteAccount: no match found in webfinger response") + return +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 7c4d588bb..8cb110f24 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -85,7 +85,10 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat return nil, nil, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err) } - _, err = d.GetRemoteAccount(ctx, username, accountURI, true, false) + _, err = d.GetRemoteAccount(ctx, GetRemoteAccountParams{ + RequestingUsername: username, + RemoteAccountID: accountURI, + }) if err != nil { return nil, nil, fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err) } @@ -316,7 +319,10 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta if targetAccount == nil { // we didn't find the account in our database already // check if we can get the account remotely (dereference it) - if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil { + if a, err := d.GetRemoteAccount(ctx, GetRemoteAccountParams{ + RequestingUsername: requestingUsername, + RemoteAccountID: targetAccountURI, + }); err != nil { errs = append(errs, err.Error()) } else { logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) diff --git a/internal/federation/federatingactor_test.go b/internal/federation/federatingactor_test.go index fdf907030..905b6a7b4 100644 --- a/internal/federation/federatingactor_test.go +++ b/internal/federation/federatingactor_test.go @@ -19,10 +19,7 @@ package federation_test import ( - "bytes" "context" - "io/ioutil" - "net/http" "net/url" "testing" "time" @@ -60,15 +57,9 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) // setup transport controller with a no-op client so we don't make external calls - sentMessages := []*url.URL{} - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - sentMessages = append(sentMessages, req.URL) - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -77,7 +68,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() { suite.NotNil(activity) // because zork has no remote followers, sent messages should be empty (no messages sent to own instance) - suite.Empty(sentMessages) + suite.Empty(httpClient.SentMessages) } func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { @@ -87,8 +78,8 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { err := suite.db.Put(ctx, >smodel.Follow{ ID: "01G1TRWV4AYCDBX5HRWT2EVBCV", - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + CreatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), + UpdatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), AccountID: testRemoteAccount.ID, TargetAccountID: testAccount.ID, ShowReblogs: true, @@ -100,7 +91,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { testNote := testrig.NewAPNote( testrig.URLMustParse("http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"), testrig.URLMustParse("http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"), - time.Now(), + testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), "boobies", "", testrig.URLMustParse(testAccount.URI), @@ -110,20 +101,12 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { nil, nil, ) - testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote) + testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - sentMessages := []*url.URL{} - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - sentMessages = append(sentMessages, req.URL) - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -132,8 +115,10 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { suite.NotNil(activity) // because we added 1 remote follower for zork, there should be a url in sentMessage - suite.Len(sentMessages, 1) - suite.Equal(testRemoteAccount.InboxURI, sentMessages[0].String()) + suite.Len(httpClient.SentMessages, 1) + msg, ok := httpClient.SentMessages[testRemoteAccount.InboxURI] + suite.True(ok) + suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","id":"http://localhost:8080/whatever_some_create","object":{"attributedTo":"http://localhost:8080/users/the_mighty_zork","content":"boobies","id":"http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5","published":"2022-06-02T12:22:21+02:00","tag":[],"to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Note","url":"http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"},"published":"2022-06-02T12:22:21+02:00","to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Create"}`, string(msg)) } func TestFederatingActorTestSuite(t *testing.T) { diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index 525932ea8..09d5c8fd8 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -119,7 +119,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { accountable = i } - updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, true) + updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true) if err != nil { return fmt.Errorf("UPDATE: error converting to account: %s", err) } diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index a41d1ae80..8944987c5 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,6 +31,7 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -197,7 +198,10 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false) + requestingAccount, err := f.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: username, + RemoteAccountID: publicKeyOwnerURI, + }) if err != nil { return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err) } diff --git a/internal/federation/federatingprotocol_test.go b/internal/federation/federatingprotocol_test.go index 992a55e6b..36832e009 100644 --- a/internal/federation/federatingprotocol_test.go +++ b/internal/federation/federatingprotocol_test.go @@ -45,10 +45,8 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook1() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -78,10 +76,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook2() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -112,10 +109,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook3() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -148,7 +144,9 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // now setup module being tested, with the mock transport controller federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -186,7 +184,8 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() { func (suite *FederatingProtocolTestSuite) TestBlocked1() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -208,7 +207,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked1() { func (suite *FederatingProtocolTestSuite) TestBlocked2() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -241,7 +241,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() { func (suite *FederatingProtocolTestSuite) TestBlocked3() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -277,7 +278,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() { func (suite *FederatingProtocolTestSuite) TestBlocked4() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 6412c9ee1..2f0606338 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -53,14 +53,10 @@ type Federator interface { // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode) - // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that - // account, or an error if it doesn't exist or can't be retrieved. - FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) - DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) diff --git a/internal/federation/finger.go b/internal/federation/finger.go deleted file mode 100644 index eba90a705..000000000 --- a/internal/federation/finger.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package federation - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/url" - "strings" - - apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" -) - -func (f *federator) FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) { - if blocked, err := f.db.IsDomainBlocked(ctx, targetDomain); blocked || err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain) - } - - t, err := f.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - b, err := t.Finger(ctx, targetUsername, targetDomain) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - resp := &apimodel.WellKnownResponse{} - if err := json.Unmarshal(b, resp); err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - if len(resp.Links) == 0 { - return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b)) - } - - // look through the links for the first one that matches "application/activity+json", this is what we need - for _, l := range resp.Links { - if strings.EqualFold(l.Type, "application/activity+json") { - if l.Href == "" || l.Rel != "self" { - continue - } - accountURI, err := url.Parse(l.Href) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err) - } - // found it! - return accountURI, nil - } - } - - return nil, errors.New("FingerRemoteAccount: no match found in webfinger response") -} |