diff options
Diffstat (limited to 'internal')
46 files changed, 574 insertions, 1105 deletions
diff --git a/internal/api/activitypub/users/inboxpost_test.go b/internal/api/activitypub/users/inboxpost_test.go index 26ee950ec..e43532e80 100644 --- a/internal/api/activitypub/users/inboxpost_test.go +++ b/internal/api/activitypub/users/inboxpost_test.go @@ -32,6 +32,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -237,10 +238,11 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { } func (suite *InboxPostTestSuite) TestPostUpdate() { + receivingAccount := suite.testAccounts["local_account_1"] updatedAccount := *suite.testAccounts["remote_account_1"] updatedAccount.DisplayName = "updated display name!" - // ad an emoji to the account; because we're serializing this remote + // add an emoji to the account; because we're serializing this remote // account from our own instance, we need to cheat a bit to get the emoji // to work properly, just for this test testEmoji := >smodel.Emoji{} @@ -251,8 +253,6 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { asAccount, err := suite.tc.AccountToAS(context.Background(), &updatedAccount) suite.NoError(err) - receivingAccount := suite.testAccounts["local_account_1"] - // create an update update := streams.NewActivityStreamsUpdate() @@ -294,7 +294,14 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) + // use a different version of the mock http client which serves the updated + // version of the remote account, as though it had been updated there too; + // this is needed so it can be dereferenced + updated properly + mockHTTPClient := testrig.NewMockHTTPClient(nil, "../../../../testrig/media") + mockHTTPClient.TestRemotePeople = map[string]vocab.ActivityStreamsPerson{ + updatedAccount.URI: asAccount, + } + tc := testrig.NewTestTransportController(mockHTTPClient, suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -346,8 +353,8 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { // emojis should be updated suite.Contains(dbUpdatedAccount.EmojiIDs, testEmoji.ID) - // account should be freshly webfingered - suite.WithinDuration(time.Now(), dbUpdatedAccount.LastWebfingeredAt, 10*time.Second) + // account should be freshly fetched + suite.WithinDuration(time.Now(), dbUpdatedAccount.FetchedAt, 10*time.Second) // everything else should be the same as it was before suite.EqualValues(updatedAccount.Username, dbUpdatedAccount.Username) diff --git a/internal/api/activitypub/users/userget_test.go b/internal/api/activitypub/users/userget_test.go index af58f62af..60fa0c4db 100644 --- a/internal/api/activitypub/users/userget_test.go +++ b/internal/api/activitypub/users/userget_test.go @@ -89,8 +89,7 @@ func (suite *UserGetTestSuite) TestGetUser() { suite.True(ok) // convert person to account - // since this account is already known, we should get a pretty full model of it from the conversion - a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false) + a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "") suite.NoError(err) suite.EqualValues(targetAccount.Username, a.Username) } @@ -167,7 +166,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() { suite.True(ok) // convert person to account - a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false) + a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "") suite.NoError(err) suite.EqualValues(targetAccount.Username, a.Username) } diff --git a/internal/db/bundb/account.go b/internal/db/bundb/account.go index 673d915ba..dccfee45b 100644 --- a/internal/db/bundb/account.go +++ b/internal/db/bundb/account.go @@ -21,7 +21,6 @@ package bundb import ( "context" "errors" - "fmt" "strings" "time" @@ -145,11 +144,27 @@ func (a *accountDB) getAccount(ctx context.Context, lookup string, dbQuery func( return nil, err } + if account.AvatarMediaAttachmentID != "" { + // Set the account's related avatar + account.AvatarMediaAttachment, err = a.state.DB.GetAttachmentByID(ctx, account.AvatarMediaAttachmentID) + if err != nil { + log.Errorf("error getting account %s avatar: %v", account.ID, err) + } + } + + if account.HeaderMediaAttachmentID != "" { + // Set the account's related header + account.HeaderMediaAttachment, err = a.state.DB.GetAttachmentByID(ctx, account.HeaderMediaAttachmentID) + if err != nil { + log.Errorf("error getting account %s header: %v", account.ID, err) + } + } + if len(account.EmojiIDs) > 0 { // Set the account's related emojis account.Emojis, err = a.state.DB.GetEmojisByIDs(ctx, account.EmojiIDs) if err != nil { - return nil, fmt.Errorf("error getting account emojis: %w", err) + log.Errorf("error getting account %s emojis: %v", account.ID, err) } } diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index 6a03d04d1..86a9995fc 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -92,7 +92,7 @@ func (suite *BasicTestSuite) TestPutAccountWithBunDefaultFields() { suite.Empty(a.StatusFormat) suite.Equal(testAccount.URI, a.URI) suite.Equal(testAccount.URL, a.URL) - suite.Zero(testAccount.LastWebfingeredAt) + suite.Zero(testAccount.FetchedAt) suite.Equal(testAccount.InboxURI, a.InboxURI) suite.Equal(testAccount.OutboxURI, a.OutboxURI) suite.Empty(a.FollowingURI) diff --git a/internal/db/bundb/migrations/20230202212700_rename_account_webfingered_to_fetched.go b/internal/db/bundb/migrations/20230202212700_rename_account_webfingered_to_fetched.go new file mode 100644 index 000000000..117347965 --- /dev/null +++ b/internal/db/bundb/migrations/20230202212700_rename_account_webfingered_to_fetched.go @@ -0,0 +1,44 @@ +/* + GoToSocial + Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package migrations + +import ( + "context" + + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := tx.ExecContext(ctx, "ALTER TABLE ? RENAME COLUMN ? TO ?", bun.Ident("accounts"), bun.Ident("last_webfingered_at"), bun.Ident("fetched_at")) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/notification_test.go b/internal/db/bundb/notification_test.go index 0f5cef6fc..5bb1c7d49 100644 --- a/internal/db/bundb/notification_test.go +++ b/internal/db/bundb/notification_test.go @@ -39,10 +39,7 @@ func (suite *NotificationTestSuite) spamNotifs() { zork := suite.testAccounts["local_account_1"] for i := 0; i < notifCount; i++ { - notifID, err := id.NewULID() - if err != nil { - panic(err) - } + notifID := id.NewULID() var targetAccountID string if i%2 == 0 { diff --git a/internal/db/bundb/session.go b/internal/db/bundb/session.go index 846b5fe76..fbc739ce6 100644 --- a/internal/db/bundb/session.go +++ b/internal/db/bundb/session.go @@ -63,13 +63,8 @@ func (s *sessionDB) createSession(ctx context.Context) (*gtsmodel.RouterSession, return nil, err } - id, err := id.NewULID() - if err != nil { - return nil, err - } - rs := >smodel.RouterSession{ - ID: id, + ID: id.NewULID(), Auth: auth, Crypt: crypt, } diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go deleted file mode 100644 index 5acc038e9..000000000 --- a/internal/federation/dereference.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package federation - -import ( - "context" - "net/url" - - "github.com/superseriousbusiness/gotosocial/internal/ap" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -func (f *federator) GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) { - return f.dereferencer.GetAccount(ctx, params) -} - -func (f *federator) GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) { - return f.dereferencer.GetStatus(ctx, username, remoteStatusID, refetch, includeParent) -} - -func (f *federator) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) { - return f.dereferencer.EnrichRemoteStatus(ctx, username, status, includeParent) -} - -func (f *federator) DereferenceRemoteThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) { - f.dereferencer.DereferenceThread(ctx, username, statusIRI, status, statusable) -} - -func (f *federator) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { - return f.dereferencer.GetRemoteInstance(ctx, username, remoteInstanceURI) -} - -func (f *federator) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error { - return f.dereferencer.DereferenceAnnounce(ctx, announce, requestingUsername) -} diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 4910273bd..be2ec9cee 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -25,10 +25,8 @@ import ( "fmt" "io" "net/url" - "strings" "time" - "github.com/miekg/dns" "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" @@ -38,375 +36,222 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/internal/transport" ) -var webfingerInterval = -48 * time.Hour // 2 days in the past +func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) { + var ( + account *gtsmodel.Account + uriStr = uri.String() + err error + ) -func instanceAccount(account *gtsmodel.Account) bool { - return strings.EqualFold(account.Username, account.Domain) || - account.FollowersURI == "" || - account.FollowingURI == "" || - (account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor")) -} + // Search the database for existing account with ID URI. + account, err = d.db.GetAccountByURI(ctx, uriStr) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err) + } -// GetAccountParams wraps parameters for an account lookup. -type GetAccountParams struct { - // The username of the user doing the lookup request (optional). - // If not set, then the GtS instance account will be used to do the lookup. - RequestingUsername string - // The ActivityPub URI of the account (optional). - // If not set (nil), the ActivityPub URI of the account will be discovered - // via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost - // if this parameter is not set. - RemoteAccountID *url.URL - // The username of the account (optional). - // If RemoteAccountID is not set, then this value must be set. - RemoteAccountUsername string - // The host of the account (optional). - // If RemoteAccountID is not set, then this value must be set. - RemoteAccountHost string - // Whether to do a blocking call to the remote instance. If true, - // then the account's media and other fields will be fully dereferenced before it is returned. - // If false, then the account's media and other fields will be dereferenced in the background, - // so only a minimal account representation will be returned by GetRemoteAccount. - Blocking bool - // Whether to skip making calls to remote instances. This is useful when you want to - // quickly fetch a remote account from the database or fail, and don't want to cause - // http requests to go flying around. - SkipResolve bool - // PartialAccount can be used if the GetRemoteAccount call results from a federated/ap - // account update. In this case, we will already have a partial representation of the account, - // derived from converting the AP representation to a gtsmodel representation. If this field - // is provided, then GetRemoteAccount will use this as a basis for building the full account. - PartialAccount *gtsmodel.Account -} + if account == nil { + // Else, search the database for existing by ID URL. + account, err = d.db.GetAccountByURL(ctx, uriStr) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err) + } + } -type lookupType int + if account == nil { + // Ensure that this is isn't a search for a local account. + if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() { + return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries + } -const ( - lookupPartialLocal lookupType = iota - lookupPartial - lookupURILocal - lookupURI - lookupMentionLocal - lookupMention - lookupBad -) + // Create and pass-through a new bare-bones model for dereferencing. + return d.enrichAccount(ctx, requestUser, uri, >smodel.Account{ + ID: id.NewULID(), + Domain: uri.Host, + URI: uriStr, + }, false, true) + } -func getLookupType(params GetAccountParams) lookupType { - switch { - case params.PartialAccount != nil: - if params.PartialAccount.Domain == "" || params.PartialAccount.Domain == config.GetHost() || params.PartialAccount.Domain == config.GetAccountDomain() { - return lookupPartialLocal - } - return lookupPartial - case params.RemoteAccountID != nil: - if host := params.RemoteAccountID.Host; host == config.GetHost() || host == config.GetAccountDomain() { - return lookupURILocal - } - return lookupURI - case params.RemoteAccountUsername != "": - if params.RemoteAccountHost == "" || params.RemoteAccountHost == config.GetHost() || params.RemoteAccountHost == config.GetAccountDomain() { - return lookupMentionLocal - } - return lookupMention - default: - return lookupBad + // Try to update existing account model + enriched, err := d.enrichAccount(ctx, requestUser, uri, account, false, block) + if err != nil { + log.Errorf("error enriching remote account: %v", err) + return account, nil // fall back to returning existing } + + return enriched, nil } -// GetAccount completely dereferences an account, converts it to a GtS model account, -// puts or updates it in the database (if necessary), and returns it to a caller. -// -// GetAccount will guard against trying to do http calls to fetch an account that belongs to this instance. -// Instead of making calls, it will just return the account early if it finds it, or return an error. -// -// Even if a fastfail context is used, and something goes wrong, an account might still be returned instead -// of an error, if we already had the account in our database (in other words, if we just needed to try -// fingering/refreshing the account again). The rationale for this is that it's more useful to be able -// to provide *something* to the caller, even if that something is not necessarily 100% up to date. -func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundAccount *gtsmodel.Account, err error) { - /* - In this function we want to retrieve a gtsmodel representation of a remote account, with its proper - accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth. - - There are a few different paths through this function, and the path taken depends on how much - initial information we are provided with via parameters, how much information we already have stored, - and what we're allowed to do according to the parameters we've been passed. - - Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the - account username + host, so we can check in our database and return if possible. - - Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host. - In this case, we can use the URI to resolve the remote account and find the username, - and then we can webfinger the account to discover the accountDomain if necessary. - - Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI. - In this case, we can webfinger the account to discover the URI, and then dereference - from that. - */ - - // this first step checks if we have the - // account in the database somewhere already, - // or if we've been provided it as a partial - switch getLookupType(params) { - case lookupPartialLocal: - params.SkipResolve = true - fallthrough - case lookupPartial: - foundAccount = params.PartialAccount - case lookupURILocal: - params.SkipResolve = true - fallthrough - case lookupURI: - // see if we have this in the db already with this uri/url - uri := params.RemoteAccountID.String() - - if a, dbErr := d.db.GetAccountByURI(ctx, uri); dbErr == nil { - // got it, break here to leave early - foundAccount = a - break - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with uri %s: %w", uri, dbErr)) - break - } +func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) { + if domain == config.GetHost() || domain == config.GetAccountDomain() { + // We do local lookups using an empty domain, + // else it will fail the db search below. + domain = "" + } - // dbErr was just db.ErrNoEntries so search by url instead - if a, dbErr := d.db.GetAccountByURL(ctx, uri); dbErr == nil { - // got it - foundAccount = a - break - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with url %s: %w", uri, dbErr)) - break - } - case lookupMentionLocal: - params.SkipResolve = true - params.RemoteAccountHost = "" - fallthrough - case lookupMention: - // see if we have this in the db already with this username/host - if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, params.RemoteAccountHost); dbErr == nil { - foundAccount = a - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr)) + // Search the database for existing account with USERNAME@DOMAIN + account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err) + } + + if account == nil { + // Check for failed local lookup. + if domain == "" { + return nil, NewErrNotRetrievable(err) // will be db.ErrNoEntries } - default: - err = newErrBadRequest(errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account")) + + // Create and pass-through a new bare-bones model for dereferencing. + return d.enrichAccount(ctx, requestUser, nil, >smodel.Account{ + ID: id.NewULID(), + Username: username, + Domain: domain, + }, false, true) } - // bail if we've set a real error, and not just no entries in the db + // Try to update existing account model + enriched, err := d.enrichAccount(ctx, requestUser, nil, account, false, block) if err != nil { - return + log.Errorf("GetAccountByUsernameDomain: error enriching account from remote: %v", err) + return account, nil // fall back to returning unchanged existing account model } - if params.SkipResolve { - // if we can't resolve, return already since there's nothing more we can do - if foundAccount == nil { - err = newErrNotRetrievable(errors.New("GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")) - } - return - } + return enriched, nil +} - // if we reach this point, we have some remote calls to make +func (d *deref) UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) { + return d.enrichAccount(ctx, requestUser, nil, account, force, false) +} - var accountable ap.Accountable - if params.RemoteAccountUsername == "" && params.RemoteAccountHost == "" { - // if we're still missing some params, try to populate them now - params.RemoteAccountHost = params.RemoteAccountID.Host - if foundAccount != nil { - // username is easy if we found something already - params.RemoteAccountUsername = foundAccount.Username - } else { - // if we didn't already have it, we have to dereference it from remote - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } +// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary. +func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, force, block bool) (*gtsmodel.Account, error) { + if account.IsLocal() { + // Can't update local accounts. + return account, nil + } - var apError error - params.RemoteAccountUsername, apError = ap.ExtractPreferredUsername(accountable) - if apError != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error extracting Accountable username: %w", apError)) - return - } - } + if !account.CreatedAt.IsZero() && account.IsInstance() { + // Existing instance account. No need for update. + return account, nil } - // if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set - // params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it - - // we finger to fetch the account domain but just in case we're not fingering, make a best guess - // already about what the account domain might be; this var will be overwritten later if necessary - var accountDomain string - switch { - case foundAccount != nil: - accountDomain = foundAccount.Domain - case params.RemoteAccountID != nil: - accountDomain = params.RemoteAccountID.Host - default: - accountDomain = params.RemoteAccountHost - } - - // to save on remote calls, only webfinger if: - // - we don't know the remote account ActivityPub ID yet OR - // - we haven't found the account yet in some other way OR - // - we were passed a partial account in params OR - // - we haven't webfingered the account for two days AND the account isn't an instance account - var fingered time.Time - var refreshFinger bool - if foundAccount != nil { - refreshFinger = foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount) - } - - if params.RemoteAccountID == nil || foundAccount == nil || params.PartialAccount != nil || refreshFinger { - if ad, accountURI, fingerError := d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost); fingerError != nil { - if !refreshFinger { - // only return with an error if this wasn't just a refresh finger; - // that is, if we actually *needed* to finger in order to get the account, - // otherwise we can just continue and we'll try again in 2 days - err = newErrNotRetrievable(fmt.Errorf("GetRemoteAccount: error while fingering: %w", fingerError)) - return - } - log.Infof("error doing non-vital webfinger refresh call to %s: %s", params.RemoteAccountHost, err) - } else { - accountDomain = ad - params.RemoteAccountID = accountURI - } - fingered = time.Now() - } - - if !fingered.IsZero() && foundAccount == nil { - // if we just fingered and now have a discovered account domain but still no account, - // we should do a final lookup in the database with the discovered username + accountDomain - // to make absolutely sure we don't already have this account - if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, accountDomain); dbErr == nil { - foundAccount = a - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr)) - return + if !force { + const interval = time.Hour * 48 + + // If this account was updated recently (last interval), we return as-is. + if next := account.FetchedAt.Add(interval); time.Now().Before(next) { + return account, nil } } - // we may have some extra information already, like the account we had in the db, or the - // accountable representation that we dereferenced from remote - if foundAccount == nil { - // if we still don't have a remoteAccountID here we're boned - if params.RemoteAccountID == nil { - err = newErrNotRetrievable(errors.New("GetRemoteAccount: could not populate find an account nor populate params.RemoteAccountID")) - return - } + if account.Username != "" { + // A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info. + accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain) - // deference accountable if we didn't earlier - if accountable == nil { - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } + if err != nil && account.URI == "" { + // this is a new account (to us) with username@domain but failed + // webfinger, there is nothing more we can do in this situation. + return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err) } - // then convert - foundAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false) - if err != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error converting Accountable to account: %w", err)) - return + if err == nil { + // Update account with latest info. + account.URI = accURI.String() + account.Domain = accDomain + uri = accURI } + } - // this is a new account so we need to generate a new ID for it - var ulid string - ulid, err = id.NewRandomULID() - if err != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error generating new id for account: %w", err)) - return - } - foundAccount.ID = ulid + if uri == nil { + var err error - if _, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking); populateErr != nil { - // it's not the end of the world if we can't populate account fields, but we do want to log it - log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr) + // No URI provided / found, must parse from account. + uri, err = url.Parse(account.URI) + if err != nil { + return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err) } + } - foundAccount.LastWebfingeredAt = fingered - foundAccount.UpdatedAt = time.Now() + // Check whether this account URI is a blocked domain / subdomain + if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil { + return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err)) + } else if blocked { + return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host) + } - if dbErr := d.db.PutAccount(ctx, foundAccount); dbErr != nil { - err = newErrDB(fmt.Errorf("GetRemoteAccount: error putting new account: %w", dbErr)) - return - } + // Mark deref+update handshake start + d.startHandshake(requestUser, uri) + defer d.stopHandshake(requestUser, uri) - return // the new account + // Dereference this account to get the latest available. + apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri) + if err != nil { + return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err) } - // we had the account already, but now we know the account domain, so update it if it's different - var accountDomainChanged bool - if !strings.EqualFold(foundAccount.Domain, accountDomain) { - accountDomainChanged = true - foundAccount.Domain = accountDomain + // Convert the dereferenced AP account object to our GTS model. + latestAcc, err := d.typeConverter.ASRepresentationToAccount( + ctx, apubAcc, account.Domain, + ) + if err != nil { + return nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err) } - // if SharedInboxURI is nil, that means we don't know yet if this account has - // a shared inbox available for it, so we need to check this here - var sharedInboxChanged bool - if foundAccount.SharedInboxURI == nil { - // we need the accountable for this, so get it if we don't have it yet - if accountable == nil { - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } - } + if account.Username == "" { + // No username was provided, so no webfinger was attempted earlier. + // + // Now we have a username we can attempt it now, this ensures up-to-date accountdomain info. + accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host) - // This can be: - // - an empty string (we know it doesn't have a shared inbox) OR - // - a string URL (we know it does a shared inbox). - // Set it either way! - var sharedInbox string - - if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil { - // only trust shared inbox if it has at least two domains, - // from the right, in common with the domain of the account - if dns.CompareDomainName(foundAccount.Domain, sharedInboxURI.Host) >= 2 { - sharedInbox = sharedInboxURI.String() - } + if err == nil { + // Update account with latest info. + latestAcc.Domain = accDomain } - - sharedInboxChanged = true - foundAccount.SharedInboxURI = &sharedInbox } - // make sure the account fields are populated before returning: - // the caller might want to block until everything is loaded - fieldsChanged, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking) - if populateErr != nil { - // it's not the end of the world if we can't populate account fields, but we do want to log it - log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr) + // Ensure ID is set and update fetch time. + latestAcc.ID = account.ID + latestAcc.FetchedAt = time.Now() + + // Fetch latest account media (TODO: check for changed URI to previous). + if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil { + log.Errorf("error fetching remote media for account %s: %v", uri, err) } - var fingeredChanged bool - if !fingered.IsZero() { - fingeredChanged = true - foundAccount.LastWebfingeredAt = fingered + // Fetch the latest remote account emoji IDs used in account display name/bio. + _, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser) + if err != nil { + log.Errorf("error fetching remote emojis for account %s: %v", uri, err) } - if accountDomainChanged || sharedInboxChanged || fieldsChanged || fingeredChanged { - if dbErr := d.db.UpdateAccount(ctx, foundAccount); dbErr != nil { - err = newErrDB(fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %w", dbErr)) - return + if account.CreatedAt.IsZero() { + // CreatedAt will be zero if no local copy was + // found in one of the GetAccountBy___() functions. + // + // Set time of creation from the last-fetched date. + latestAcc.CreatedAt = latestAcc.FetchedAt + latestAcc.UpdatedAt = latestAcc.FetchedAt + + // This is a new account, we need to place it in the database. + if err := d.db.PutAccount(ctx, latestAcc); err != nil { + return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err) + } + } else { + // Set time of update from the last-fetched date. + latestAcc.UpdatedAt = latestAcc.FetchedAt + + // Use existing account values. + latestAcc.CreatedAt = account.CreatedAt + latestAcc.Language = account.Language + + // This is an existing account, update the model in the database. + if err := d.db.UpdateAccount(ctx, latestAcc); err != nil { + return nil, fmt.Errorf("enrichAccount: error updating database: %w", err) } } - return // the account we already had + possibly updated + return latestAcc, nil } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -414,13 +259,6 @@ func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundA // // Will work for Person, Application, or Service models. func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) { - d.startHandshake(username, remoteAccountID) - defer d.stopHandshake(username, remoteAccountID) - - if blocked, err := d.db.IsDomainBlocked(ctx, remoteAccountID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host) - } - transport, err := d.transportController.NewTransportForUsername(ctx, username) if err != nil { return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err) @@ -441,83 +279,23 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err) } + //nolint shutup linter switch t.GetTypeName() { case ap.ActorApplication: - p, ok := t.(vocab.ActivityStreamsApplication) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application") - } - return p, nil + return t.(vocab.ActivityStreamsApplication), nil case ap.ActorGroup: - p, ok := t.(vocab.ActivityStreamsGroup) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams group") - } - return p, nil + return t.(vocab.ActivityStreamsGroup), nil case ap.ActorOrganization: - p, ok := t.(vocab.ActivityStreamsOrganization) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams organization") - } - return p, nil + return t.(vocab.ActivityStreamsOrganization), nil case ap.ActorPerson: - p, ok := t.(vocab.ActivityStreamsPerson) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person") - } - return p, nil + return t.(vocab.ActivityStreamsPerson), nil case ap.ActorService: - p, ok := t.(vocab.ActivityStreamsService) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service") - } - return p, nil + return t.(vocab.ActivityStreamsService), nil } return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName())) } -// populateAccountFields makes a best effort to populate fields on an account such as emojis, avatar, header. -// Will return true if one of these things changed on the passed-in account. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { - // if we're dealing with an instance account, just bail, we don't need to do anything - if instanceAccount(account) { - return false, nil - } - - accountURI, err := url.Parse(account.URI) - if err != nil { - return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %w", account.URI, err) - } - - blocked, dbErr := d.db.IsDomainBlocked(ctx, accountURI.Host) - if dbErr != nil { - return false, fmt.Errorf("populateAccountFields: eror checking for block of domain %s: %w", accountURI.Host, err) - } - - if blocked { - return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host) - } - - var changed bool - - // fetch the header and avatar - if mediaChanged, err := d.fetchRemoteAccountMedia(ctx, account, requestingUsername, blocking); err != nil { - return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %w", err) - } else if mediaChanged { - changed = mediaChanged - } - - // fetch any emojis used in note, fields, display name, etc - if emojisChanged, err := d.fetchRemoteAccountEmojis(ctx, account, requestingUsername); err != nil { - return false, fmt.Errorf("populateAccountFields: error fetching emojis for account: %w", err) - } else if emojisChanged { - changed = emojisChanged - } - - return changed, nil -} - // fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account, // using a transport on behalf of requestingUsername. // @@ -530,39 +308,30 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { - var ( - changed bool - t transport.Transport - ) +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error { + // Fetch a transport beforehand for either(or both) avatar / header dereferencing. + tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) + } - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") { + if targetAccount.AvatarRemoteURL != "" { var processingMedia *media.ProcessingMedia + // Parse the target account's avatar URL into URL object. + avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) + if err != nil { + return err + } + d.dereferencingAvatarsLock.Lock() // LOCK HERE // first check if we're already processing this media if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } else { - // we're not already processing it so start now - avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return changed, err - } - - if t == nil { - var err error - t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) - } - } - data := func(innerCtx context.Context) (io.ReadCloser, int64, error) { - return t.DereferenceMedia(innerCtx, avatarIRI) + return tsport.DereferenceMedia(innerCtx, avatarIRI) } avatar := true @@ -572,7 +341,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm }) if err != nil { d.dereferencingAvatarsLock.Unlock() - return changed, err + return err } // store it in our map to indicate it's in process @@ -595,7 +364,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := loadAndCleanup(ctx, load, cleanup); err != nil { - return changed, err + return err } } else { // ...otherwise do it async @@ -609,36 +378,25 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() - changed = true } - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") { + if targetAccount.HeaderRemoteURL != "" { var processingMedia *media.ProcessingMedia + // Parse the target account's header URL into URL object. + headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) + if err != nil { + return err + } + d.dereferencingHeadersLock.Lock() // LOCK HERE // first check if we're already processing this media if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } else { - // we're not already processing it so start now - headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return changed, err - } - - if t == nil { - var err error - t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) - } - } - data := func(innerCtx context.Context) (io.ReadCloser, int64, error) { - return t.DereferenceMedia(innerCtx, headerIRI) + return tsport.DereferenceMedia(innerCtx, headerIRI) } header := true @@ -648,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm }) if err != nil { d.dereferencingAvatarsLock.Unlock() - return changed, err + return err } // store it in our map to indicate it's in process @@ -671,7 +429,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := loadAndCleanup(ctx, load, cleanup); err != nil { - return changed, err + return err } } else { // ...otherwise do it async @@ -685,10 +443,9 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() - changed = true } - return changed, nil + return nil } func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) { diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index eb795d169..7898aec87 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -27,7 +27,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -39,17 +38,19 @@ func (suite *AccountTestSuite) TestDereferenceGroup() { fetchingAccount := suite.testAccounts["local_account_1"] groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group") - group, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: groupURL, - }) + group, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + groupURL, + false, + ) suite.NoError(err) suite.NotNil(group) // group values should be set suite.Equal("https://unknown-instance.com/groups/some_group", group.URI) suite.Equal("https://unknown-instance.com/@some_group", group.URL) - suite.WithinDuration(time.Now(), group.LastWebfingeredAt, 5*time.Second) + suite.WithinDuration(time.Now(), group.FetchedAt, 5*time.Second) // group should be in the database dbGroup, err := suite.db.GetAccountByURI(context.Background(), group.URI) @@ -62,17 +63,19 @@ func (suite *AccountTestSuite) TestDereferenceService() { fetchingAccount := suite.testAccounts["local_account_1"] serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh") - service, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: serviceURL, - }) + service, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + serviceURL, + false, + ) suite.NoError(err) suite.NotNil(service) // service values should be set suite.Equal("https://owncast.example.org/federation/user/rgh", service.URI) suite.Equal("https://owncast.example.org/federation/user/rgh", service.URL) - suite.WithinDuration(time.Now(), service.LastWebfingeredAt, 5*time.Second) + suite.WithinDuration(time.Now(), service.FetchedAt, 5*time.Second) // service should be in the database dbService, err := suite.db.GetAccountByURI(context.Background(), service.URI) @@ -93,10 +96,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -111,10 +116,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb suite.FailNow(err.Error()) } - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -124,10 +131,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: targetAccount.Username, - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -137,11 +146,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: targetAccount.Username, - RemoteAccountHost: config.GetHost(), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -151,12 +161,13 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - RemoteAccountUsername: targetAccount.Username, - RemoteAccountHost: config.GetHost(), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + targetAccount.Username, + config.GetHost(), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -165,248 +176,50 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: "thisaccountdoesnotexist", - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + "thisaccountdoesnotexist", + config.GetHost(), + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: "thisaccountdoesnotexist", - RemoteAccountHost: "localhost:8080", - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + "thisaccountdoesnotexist", + "localhost:8080", + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"), + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() { - fetchingAccount := suite.testAccounts["local_account_1"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // dereference an emoji we don't have stored yet - { - URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", - Shortcode: "kip_van_den_bos", - UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", - Disabled: testrig.FalseBool(), - VisibleInPicker: testrig.FalseBool(), - Domain: "fossbros-anonymous.io", - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) -} - -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() { - fetchingAccount := suite.testAccounts["local_account_1"] - - knownEmoji := suite.testEmojis["yell"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // an emoji we already have - { - URI: knownEmoji.URI, - Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.UpdatedAt, - ImageUpdatedAt: knownEmoji.ImageUpdatedAt, - ImageRemoteURL: knownEmoji.ImageRemoteURL, - Disabled: knownEmoji.Disabled, - VisibleInPicker: knownEmoji.VisibleInPicker, - Domain: knownEmoji.Domain, - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) -} - -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() { - fetchingAccount := suite.testAccounts["local_account_1"] - - knownEmoji := suite.testEmojis["yell"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // an emoji we already have - { - URI: knownEmoji.URI, - Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.UpdatedAt, - ImageUpdatedAt: knownEmoji.ImageUpdatedAt, - ImageRemoteURL: knownEmoji.ImageRemoteURL, - Disabled: knownEmoji.Disabled, - VisibleInPicker: knownEmoji.VisibleInPicker, - Domain: knownEmoji.Domain, - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) - suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI) - - remoteAccountPartial2 := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // dereference an emoji we don't have stored yet - { - URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", - Shortcode: "kip_van_den_bos", - UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", - Disabled: testrig.FalseBool(), - VisibleInPicker: testrig.FalseBool(), - Domain: "fossbros-anonymous.io", - }, - }, - } - - fetchedAccount2, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial2, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount2) - suite.NotNil(fetchedAccount2.EmojiIDs) - suite.NotNil(fetchedAccount2.Emojis) - suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI) -} - func TestAccountTestSuite(t *testing.T) { suite.Run(t, new(AccountTestSuite)) } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 23f91ecae..122cbb59c 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -33,7 +33,17 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetAccount(ctx context.Context, params GetAccountParams) (*gtsmodel.Account, error) + // GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the + // last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before. + GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) + + // GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either + // check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before. + GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) + + // UpdateAccount updates the given account if last_fetched is beyond fetch interval (or if force is set). An updated account model is returned, any media fetching is done async. + UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) + GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) @@ -44,7 +54,7 @@ type Dereferencer interface { GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) - Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool + Handshaking(username string, remoteAccountID *url.URL) bool } type deref struct { @@ -53,28 +63,25 @@ type deref struct { transportController transport.Controller mediaManager media.Manager dereferencingAvatars map[string]*media.ProcessingMedia - dereferencingAvatarsLock *sync.Mutex + dereferencingAvatarsLock sync.Mutex dereferencingHeaders map[string]*media.ProcessingMedia - dereferencingHeadersLock *sync.Mutex + dereferencingHeadersLock sync.Mutex dereferencingEmojis map[string]*media.ProcessingEmoji - dereferencingEmojisLock *sync.Mutex + dereferencingEmojisLock sync.Mutex handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map + handshakeSync sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewDereferencer returns a Dereferencer initialized with the given parameters. func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer { return &deref{ - db: db, - typeConverter: typeConverter, - transportController: transportController, - mediaManager: mediaManager, - dereferencingAvatars: make(map[string]*media.ProcessingMedia), - dereferencingAvatarsLock: &sync.Mutex{}, - dereferencingHeaders: make(map[string]*media.ProcessingMedia), - dereferencingHeadersLock: &sync.Mutex{}, - dereferencingEmojis: make(map[string]*media.ProcessingEmoji), - dereferencingEmojisLock: &sync.Mutex{}, - handshakeSync: &sync.Mutex{}, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaManager: mediaManager, + dereferencingAvatars: make(map[string]*media.ProcessingMedia), + dereferencingHeaders: make(map[string]*media.ProcessingMedia), + dereferencingEmojis: make(map[string]*media.ProcessingEmoji), + handshakes: make(map[string][]*url.URL), } } diff --git a/internal/federation/dereferencing/error.go b/internal/federation/dereferencing/error.go index 48804b50d..3ba357f7a 100644 --- a/internal/federation/dereferencing/error.go +++ b/internal/federation/dereferencing/error.go @@ -49,24 +49,10 @@ func (err *ErrNotRetrievable) Error() string { return fmt.Sprintf("item could not be retrieved: %v", err.wrapped) } -func newErrNotRetrievable(err error) error { +func NewErrNotRetrievable(err error) error { return &ErrNotRetrievable{wrapped: err} } -// ErrBadRequest denotes that insufficient or improperly formed parameters -// were passed into one of the dereference functions. -type ErrBadRequest struct { - wrapped error -} - -func (err *ErrBadRequest) Error() string { - return fmt.Sprintf("bad request: %v", err.wrapped) -} - -func newErrBadRequest(err error) error { - return &ErrBadRequest{wrapped: err} -} - // ErrTransportError indicates that something unforeseen went wrong creating // a transport, or while making an http call to a remote resource with a transport. type ErrTransportError struct { @@ -121,7 +107,7 @@ func wrapDerefError(derefErr error, fluff string) error { switch { case errors.Is(derefErr, transport.ErrGone): - err = newErrNotRetrievable(err) + err = NewErrNotRetrievable(err) case errors.As(derefErr, &errWrongType): err = newErrWrongType(err) default: diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go index 8533f3387..2b67e86bf 100644 --- a/internal/federation/dereferencing/handshake.go +++ b/internal/federation/dereferencing/handshake.go @@ -19,11 +19,10 @@ package dereferencing import ( - "context" "net/url" ) -func (d *deref) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool { +func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() @@ -53,11 +52,6 @@ func (d *deref) startHandshake(username string, remoteAccountID *url.URL) { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() - // lazily initialize handshakes - if d.handshakes == nil { - d.handshakes = make(map[string][]*url.URL) - } - remoteIDs, ok := d.handshakes[username] if !ok { // there was nothing in there yet, so just add this entry and return @@ -74,13 +68,8 @@ func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() - if d.handshakes == nil { - return - } - remoteIDs, ok := d.handshakes[username] if !ok { - // there was nothing in there yet anyway so just bail return } diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 8695c6f95..da4639c5d 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -100,7 +100,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U if status != nil { return status, nil, nil } - return nil, nil, newErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString)) + return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString)) } // if we got here, either we didn't have the status @@ -123,11 +123,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U } // we need to get the author of the status else we can't serialize it properly - if _, err = d.GetAccount(ctx, GetAccountParams{ - RequestingUsername: username, - RemoteAccountID: accountURI, - Blocking: true, - }); err != nil { + if _, err = d.GetAccountByURI(ctx, username, accountURI, true); err != nil { return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err)) } @@ -353,10 +349,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta if targetAccount == nil { // we didn't find the account in our database already // check if we can get the account remotely (dereference it) - if a, err := d.GetAccount(ctx, GetAccountParams{ - RequestingUsername: requestingUsername, - RemoteAccountID: targetAccountURI, - }); err != nil { + if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI, false); err != nil { errs = append(errs, err.Error()) } else { log.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) diff --git a/internal/federation/federatingdb/create.go b/internal/federation/federatingdb/create.go index fbe3d3ad6..62784e68d 100644 --- a/internal/federation/federatingdb/create.go +++ b/internal/federation/federatingdb/create.go @@ -100,11 +100,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec return fmt.Errorf("activityBlock: could not convert Block to gts model block") } - newID, err := id.NewULID() - if err != nil { - return err - } - block.ID = newID + block.ID = id.NewULID() if err := f.db.PutBlock(ctx, block); err != nil { return fmt.Errorf("activityBlock: database error inserting block: %s", err) @@ -263,11 +259,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re return fmt.Errorf("activityFollow: could not convert Follow to follow request: %s", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - followRequest.ID = newID + followRequest.ID = id.NewULID() if err := f.db.Put(ctx, followRequest); err != nil { return fmt.Errorf("activityFollow: database error inserting follow request: %s", err) @@ -298,11 +290,7 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece return fmt.Errorf("activityLike: could not convert Like to fave: %s", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - fave.ID = newID + fave.ID = id.NewULID() if err := f.db.Put(ctx, fave); err != nil { return fmt.Errorf("activityLike: database error inserting fave: %s", err) @@ -333,11 +321,7 @@ func (f *federatingDB) activityFlag(ctx context.Context, asType vocab.Type, rece return fmt.Errorf("activityFlag: could not convert Flag to report: %w", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - report.ID = newID + report.ID = id.NewULID() if err := f.db.PutReport(ctx, report); err != nil { return fmt.Errorf("activityFlag: database error inserting report: %w", err) diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index d84bce668..04ce39a77 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -116,7 +116,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { accountable = i } - updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true) + updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "") if err != nil { return fmt.Errorf("UPDATE: error converting to account: %s", err) } diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index a63aecc66..90a2c50b2 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,7 +31,6 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/transport" @@ -206,10 +205,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount, err := f.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: username, - RemoteAccountID: publicKeyOwnerURI, - }) + requestingAccount, err := f.GetAccountByURI( + transport.WithFastfail(ctx), username, publicKeyOwnerURI, false, + ) if err != nil { return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err) } diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 0ba370cc4..5aa44742d 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -23,12 +23,10 @@ import ( "net/url" "github.com/superseriousbusiness/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -53,20 +51,9 @@ type Federator interface { // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode) - /* - dereferencing functions - */ - DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) - DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error - GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) - GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) - EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) - GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - - // Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID. - Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool pub.CommonBehavior pub.FederatingProtocol + dereferencing.Dereferencer } type federator struct { @@ -75,9 +62,9 @@ type federator struct { clock pub.Clock typeConverter typeutils.TypeConverter transportController transport.Controller - dereferencer dereferencing.Dereferencer mediaManager media.Manager actor pub.FederatingActor + dereferencing.Dereferencer } // NewFederator returns a new federator @@ -91,8 +78,8 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr clock: &Clock{}, typeConverter: typeConverter, transportController: transportController, - dereferencer: dereferencer, mediaManager: mediaManager, + Dereferencer: dereferencer, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/gone.go b/internal/federation/gone.go index 3d9fe3b84..e6d784181 100644 --- a/internal/federation/gone.go +++ b/internal/federation/gone.go @@ -2,12 +2,10 @@ package federation import ( "context" - "fmt" "net/url" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" ) // CheckGone checks if a tombstone exists in the database for AP Actor or Object with the given uri. @@ -17,18 +15,10 @@ func (f *federator) CheckGone(ctx context.Context, uri *url.URL) (bool, error) { // HandleGone puts a tombstone in the database, which marks an AP Actor or Object with the given uri as gone. func (f *federator) HandleGone(ctx context.Context, uri *url.URL) error { - tombstoneID, err := id.NewULID() - if err != nil { - err = fmt.Errorf("HandleGone: error generating id for new tombstone %s: %s", uri, err) - log.Error(err) - return err - } - tombstone := >smodel.Tombstone{ - ID: tombstoneID, + ID: id.NewULID(), Domain: uri.Host, URI: uri.String(), } - return f.db.PutTombstone(ctx, tombstone) } diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go index 9669e9856..9dadb0e96 100644 --- a/internal/federation/handshake.go +++ b/internal/federation/handshake.go @@ -17,12 +17,3 @@ */ package federation - -import ( - "context" - "net/url" -) - -func (f *federator) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool { - return f.dereferencer.Handshaking(ctx, username, remoteAccountID) -} diff --git a/internal/gtsmodel/account.go b/internal/gtsmodel/account.go index c843c0787..076bd7a9a 100644 --- a/internal/gtsmodel/account.go +++ b/internal/gtsmodel/account.go @@ -24,14 +24,18 @@ package gtsmodel import ( "crypto/rsa" + "strings" "time" + + "github.com/superseriousbusiness/gotosocial/internal/config" ) // Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc). type Account struct { ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database - CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created. + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item was last updated. + FetchedAt time.Time `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"` // when was item (remote) last fetched. Username string `validate:"required" bun:",nullzero,notnull,unique:userdomain"` // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"` // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username. AvatarMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present @@ -60,7 +64,6 @@ type Account struct { CustomCSS string `validate:"-" bun:",nullzero"` // Custom CSS that should be displayed for this Account's profile and statuses. URI string `validate:"required,url" bun:",nullzero,notnull,unique"` // ActivityPub URI for this account. URL string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Web URL for this account's profile - LastWebfingeredAt time.Time `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"` // Last time this account was refreshed/located with webfinger. InboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's ActivityPub inbox, for sending activity to SharedInboxURI *string `validate:"-" bun:""` // Address of this account's ActivityPub sharedInbox. Gotcha warning: this is a string pointer because it has three possible states: 1. We don't know yet if the account has a shared inbox -- null. 2. We know it doesn't have a shared inbox -- empty string. 3. We know it does have a shared inbox -- url string. OutboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's activitypub outbox @@ -79,6 +82,24 @@ type Account struct { EnableRSS *bool `validate:"-" bun:",default:false"` // enable RSS feed subscription for this account's public posts at [URL]/feed } +// IsLocal returns whether account is a local user account. +func (a Account) IsLocal() bool { + return a.Domain == "" || a.Domain == config.GetHost() || a.Domain == config.GetAccountDomain() +} + +// IsRemote returns whether account is a remote user account. +func (a Account) IsRemote() bool { + return !a.IsLocal() +} + +// IsInstance returns whether account is an instance internal actor account. +func (a Account) IsInstance() bool { + return a.Username == a.Domain || + a.FollowersURI == "" || + a.FollowingURI == "" || + (a.Username == "internal.fetch" && strings.Contains(a.Note, "internal service actor")) +} + // AccountToEmoji is an intermediate struct to facilitate the many2many relationship between an account and one or more emojis. type AccountToEmoji struct { AccountID string `validate:"ulid,required" bun:"type:CHAR(26),unique:accountemoji,nullzero,notnull"` diff --git a/internal/id/ulid.go b/internal/id/ulid.go index bed7cf046..e02870b48 100644 --- a/internal/id/ulid.go +++ b/internal/id/ulid.go @@ -35,13 +35,15 @@ const ( // ULID represents a Universally Unique Lexicographically Sortable Identifier of 26 characters. See https://github.com/oklog/ulid type ULID string -// NewULID returns a new ULID string using the current time, or an error if something goes wrong. -func NewULID() (string, error) { - newUlid, err := ulid.New(ulid.Timestamp(time.Now()), rand.Reader) +// NewULID returns a new ULID string using the current time. +func NewULID() string { + ulid, err := ulid.New( + ulid.Timestamp(time.Now()), rand.Reader, + ) if err != nil { - return "", err + panic(err) } - return newUlid.String(), nil + return ulid.String() } // NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong. diff --git a/internal/processing/account/createblock.go b/internal/processing/account/createblock.go index d2a2bab94..68f28fafe 100644 --- a/internal/processing/account/createblock.go +++ b/internal/processing/account/createblock.go @@ -53,10 +53,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel // make the block block := >smodel.Block{} - newBlockID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } + newBlockID := id.NewULID() block.ID = newBlockID block.AccountID = requestingAccount.ID block.Account = requestingAccount diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go index 6c134547e..0592555da 100644 --- a/internal/processing/account/get.go +++ b/internal/processing/account/get.go @@ -26,7 +26,6 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/transport" @@ -94,13 +93,9 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err)) } - a, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestingAccount.Username, - RemoteAccountID: targetAccountURI, - RemoteAccountHost: targetAccount.Domain, - RemoteAccountUsername: targetAccount.Username, - Blocking: true, - }) + a, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestingAccount.Username, targetAccountURI, true, + ) if err == nil { targetAccount = a } diff --git a/internal/processing/admin/accountaction.go b/internal/processing/admin/accountaction.go index c71eb27e5..e4e6bbd97 100644 --- a/internal/processing/admin/accountaction.go +++ b/internal/processing/admin/accountaction.go @@ -18,13 +18,8 @@ func (p *processor) AccountAction(ctx context.Context, account *gtsmodel.Account return gtserror.NewErrorInternalError(err) } - adminActionID, err := id.NewULID() - if err != nil { - return gtserror.NewErrorInternalError(err) - } - adminAction := >smodel.AdminAccountAction{ - ID: adminActionID, + ID: id.NewULID(), AccountID: account.ID, TargetAccountID: targetAccount.ID, Text: form.Text, diff --git a/internal/processing/admin/createdomainblock.go b/internal/processing/admin/createdomainblock.go index 98ba9cf1b..1c80ee78f 100644 --- a/internal/processing/admin/createdomainblock.go +++ b/internal/processing/admin/createdomainblock.go @@ -50,14 +50,8 @@ func (p *processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc } // there's no block for this domain yet so create one - // note: we take a new ulid from timestamp here in case we need to sort blocks - blockID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new domain block %s: %s", domain, err)) - } - newBlock := >smodel.DomainBlock{ - ID: blockID, + ID: id.NewULID(), Domain: domain, CreatedByAccountID: account.ID, PrivateComment: text.SanitizePlaintext(privateComment), diff --git a/internal/processing/federation/getfollowers.go b/internal/processing/federation/getfollowers.go index ceed3f2d5..991954389 100644 --- a/internal/processing/federation/getfollowers.go +++ b/internal/processing/federation/getfollowers.go @@ -24,7 +24,6 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -42,10 +41,9 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string, return nil, errWithCode } - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getfollowing.go b/internal/processing/federation/getfollowing.go index 20ffae849..06acdad32 100644 --- a/internal/processing/federation/getfollowing.go +++ b/internal/processing/federation/getfollowing.go @@ -24,7 +24,6 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -42,10 +41,9 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string, return nil, errWithCode } - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getoutbox.go b/internal/processing/federation/getoutbox.go index 186967c2c..6d0f2f3fe 100644 --- a/internal/processing/federation/getoutbox.go +++ b/internal/processing/federation/getoutbox.go @@ -25,7 +25,6 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -43,10 +42,9 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag return nil, errWithCode } - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getstatus.go b/internal/processing/federation/getstatus.go index 08646ceda..b54e17b11 100644 --- a/internal/processing/federation/getstatus.go +++ b/internal/processing/federation/getstatus.go @@ -24,7 +24,6 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -42,10 +41,9 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req return nil, errWithCode } - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getstatusreplies.go b/internal/processing/federation/getstatusreplies.go index 0b6d22575..08c9b1119 100644 --- a/internal/processing/federation/getstatusreplies.go +++ b/internal/processing/federation/getstatusreplies.go @@ -25,7 +25,6 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/transport" @@ -44,10 +43,9 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri return nil, errWithCode } - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getuser.go b/internal/processing/federation/getuser.go index 97f452aa3..d3fb7bdf6 100644 --- a/internal/processing/federation/getuser.go +++ b/internal/processing/federation/getuser.go @@ -25,20 +25,20 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/uris" ) func (p *processor) GetUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) { - // get the account the request is referring to + // Get the instance-local account the request is referring to. requestedAccount, err := p.db.GetAccountByUsernameDomain(ctx, requestedUsername, "") if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err)) } var requestedPerson vocab.ActivityStreamsPerson + if uris.IsPublicKeyPath(requestURL) { // if it's a public key path, we don't need to authenticate but we'll only serve the bare minimum user profile needed for the public key requestedPerson, err = p.tc.AccountToASMinimal(ctx, requestedAccount) @@ -53,11 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque } // if we're not already handshaking/dereferencing a remote account, dereference it now - if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) { - requestingAccount, err := p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestedUsername, - RemoteAccountID: requestingAccountURI, - }) + if !p.federator.Handshaking(requestedUsername, requestingAccountURI) { + requestingAccount, err := p.federator.GetAccountByURI( + transport.WithFastfail(ctx), requestedUsername, requestingAccountURI, false, + ) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/fromcommon.go b/internal/processing/fromcommon.go index 9f0512941..1c6176565 100644 --- a/internal/processing/fromcommon.go +++ b/internal/processing/fromcommon.go @@ -76,13 +76,8 @@ func (p *processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e } // if we've reached this point we know the mention is for a local account, and the notification doesn't exist, so create it - notifID, err := id.NewULID() - if err != nil { - return err - } - notif := >smodel.Notification{ - ID: notifID, + ID: id.NewULID(), NotificationType: gtsmodel.NotificationMention, TargetAccountID: m.TargetAccountID, TargetAccount: m.TargetAccount, @@ -127,13 +122,8 @@ func (p *processor) notifyFollowRequest(ctx context.Context, followRequest *gtsm return nil } - notifID, err := id.NewULID() - if err != nil { - return err - } - notif := >smodel.Notification{ - ID: notifID, + ID: id.NewULID(), NotificationType: gtsmodel.NotificationFollowRequest, TargetAccountID: followRequest.TargetAccountID, OriginAccountID: followRequest.AccountID, @@ -172,13 +162,8 @@ func (p *processor) notifyFollow(ctx context.Context, follow *gtsmodel.Follow, t } // now create the new follow notification - notifID, err := id.NewULID() - if err != nil { - return err - } - notif := >smodel.Notification{ - ID: notifID, + ID: id.NewULID(), NotificationType: gtsmodel.NotificationFollow, TargetAccountID: follow.TargetAccountID, TargetAccount: follow.TargetAccount, @@ -222,13 +207,8 @@ func (p *processor) notifyFave(ctx context.Context, fave *gtsmodel.StatusFave) e return nil } - notifID, err := id.NewULID() - if err != nil { - return err - } - notif := >smodel.Notification{ - ID: notifID, + ID: id.NewULID(), NotificationType: gtsmodel.NotificationFave, TargetAccountID: fave.TargetAccountID, TargetAccount: fave.TargetAccount, @@ -301,13 +281,8 @@ func (p *processor) notifyAnnounce(ctx context.Context, status *gtsmodel.Status) } // now create the new reblog notification - notifID, err := id.NewULID() - if err != nil { - return err - } - notif := >smodel.Notification{ - ID: notifID, + ID: id.NewULID(), NotificationType: gtsmodel.NotificationReblog, TargetAccountID: status.BoostOfAccountID, TargetAccount: status.BoostOfAccount, diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index c30fc2692..e37488794 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -27,7 +27,6 @@ import ( "codeberg.org/gruf/go-kv" "codeberg.org/gruf/go-logger/v2/level" "github.com/superseriousbusiness/gotosocial/internal/ap" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -154,11 +153,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa return err } - a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{ - RequestingUsername: federatorMsg.ReceivingAccount.Username, - RemoteAccountID: remoteAccountID, - Blocking: true, - }) + a, err := p.federator.GetAccountByURI(ctx, + federatorMsg.ReceivingAccount.Username, + remoteAccountID, + true, + ) if err != nil { return err } @@ -200,11 +199,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato return err } - a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{ - RequestingUsername: federatorMsg.ReceivingAccount.Username, - RemoteAccountID: remoteAccountID, - Blocking: true, - }) + a, err := p.federator.GetAccountByURI(ctx, + federatorMsg.ReceivingAccount.Username, + remoteAccountID, + true, + ) if err != nil { return err } @@ -242,11 +241,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context, return err } - a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{ - RequestingUsername: federatorMsg.ReceivingAccount.Username, - RemoteAccountID: remoteAccountID, - Blocking: true, - }) + a, err := p.federator.GetAccountByURI(ctx, + federatorMsg.ReceivingAccount.Username, + remoteAccountID, + true, + ) if err != nil { return err } @@ -303,11 +302,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede return err } - a, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{ - RequestingUsername: federatorMsg.ReceivingAccount.Username, - RemoteAccountID: remoteAccountID, - Blocking: true, - }) + a, err := p.federator.GetAccountByURI(ctx, + federatorMsg.ReceivingAccount.Username, + remoteAccountID, + true, + ) if err != nil { return err } @@ -380,14 +379,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder } // further database updates occur inside getremoteaccount - if _, err := p.federator.GetAccount(ctx, dereferencing.GetAccountParams{ - RequestingUsername: federatorMsg.ReceivingAccount.Username, - RemoteAccountID: incomingAccountURL, - RemoteAccountHost: incomingAccount.Domain, - RemoteAccountUsername: incomingAccount.Username, - PartialAccount: incomingAccount, - Blocking: true, - }); err != nil { + if _, err := p.federator.GetAccountByURI(ctx, + federatorMsg.ReceivingAccount.Username, + incomingAccountURL, + true, + ); err != nil { return fmt.Errorf("error enriching updated account from federator: %s", err) } diff --git a/internal/processing/report/create.go b/internal/processing/report/create.go index cc2f2405c..a7e83b656 100644 --- a/internal/processing/report/create.go +++ b/internal/processing/report/create.go @@ -64,11 +64,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form } } - reportID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - + reportID := id.NewULID() report := >smodel.Report{ ID: reportID, URI: uris.GenerateURIForReport(reportID), diff --git a/internal/processing/search.go b/internal/processing/search.go index 6082d3539..6db938b37 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -27,6 +27,8 @@ import ( "codeberg.org/gruf/go-kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -85,7 +87,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *a if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil { l.Trace("search term is a mention, looking it up...") - foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve) + foundAccount, err := p.searchAccountByUsernameDomain(ctx, authed, username, domain, search.Resolve) if err != nil { var errNotRetrievable *dereferencing.ErrNotRetrievable if !errors.As(err, &errNotRetrievable) { @@ -210,27 +212,70 @@ func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, u if !*status.Local && statusable != nil { // Attempt to dereference the status thread while we are here - p.federator.DereferenceRemoteThread(transport.WithFastfail(ctx), authed.Account.Username, uri, status, statusable) + p.federator.DereferenceThread(transport.WithFastfail(ctx), authed.Account.Username, uri, status, statusable) } return status, nil } func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Account, error) { - return p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: authed.Account.Username, - RemoteAccountID: uri, - Blocking: true, - SkipResolve: !resolve, - }) + if !resolve { + var ( + account *gtsmodel.Account + err error + uriStr = uri.String() + ) + + // Search the database for existing account with ID URI. + account, err = p.db.GetAccountByURI(ctx, uriStr) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("searchAccountByURI: error checking database for account %s: %w", uriStr, err) + } + + if account == nil { + // Else, search the database for existing by ID URL. + account, err = p.db.GetAccountByURL(ctx, uriStr) + if err != nil { + if !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("searchAccountByURI: error checking database for account %s: %w", uriStr, err) + } + return nil, dereferencing.NewErrNotRetrievable(err) + } + } + + return account, nil + } + + return p.federator.GetAccountByURI( + transport.WithFastfail(ctx), + authed.Account.Username, + uri, false, + ) } -func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) { - return p.federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: authed.Account.Username, - RemoteAccountUsername: username, - RemoteAccountHost: domain, - Blocking: true, - SkipResolve: !resolve, - }) +func (p *processor) searchAccountByUsernameDomain(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) { + if !resolve { + if domain == config.GetHost() || domain == config.GetAccountDomain() { + // We do local lookups using an empty domain, + // else it will fail the db search below. + domain = "" + } + + // Search the database for existing account with USERNAME@DOMAIN + account, err := p.db.GetAccountByUsernameDomain(ctx, username, domain) + if err != nil { + if !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("searchAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err) + } + return nil, dereferencing.NewErrNotRetrievable(err) + } + + return account, nil + } + + return p.federator.GetAccountByUsernameDomain( + transport.WithFastfail(ctx), + authed.Account.Username, + username, domain, false, + ) } diff --git a/internal/processing/status/bookmark.go b/internal/processing/status/bookmark.go index 6f5c7f633..3cf64490a 100644 --- a/internal/processing/status/bookmark.go +++ b/internal/processing/status/bookmark.go @@ -55,14 +55,9 @@ func (p *processor) Bookmark(ctx context.Context, requestingAccount *gtsmodel.Ac } if newBookmark { - thisBookmarkID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - // we need to create a new bookmark in the database gtsBookmark := >smodel.StatusBookmark{ - ID: thisBookmarkID, + ID: id.NewULID(), AccountID: requestingAccount.ID, Account: requestingAccount, TargetAccountID: targetStatus.AccountID, diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index 9e9d24c84..5bc1629c4 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -35,11 +35,7 @@ import ( func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, application *gtsmodel.Application, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, gtserror.WithCode) { accountURIs := uris.GenerateURIsForAccount(account.Username) - thisStatusID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - + thisStatusID := id.NewULID() local := true sensitive := form.Sensitive diff --git a/internal/processing/status/fave.go b/internal/processing/status/fave.go index 3686eb96f..dd5d338b3 100644 --- a/internal/processing/status/fave.go +++ b/internal/processing/status/fave.go @@ -62,10 +62,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun } if newFave { - thisFaveID, err := id.NewULID() - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } + thisFaveID := id.NewULID() // we need to create a new fave in the database gtsFave := >smodel.StatusFave{ diff --git a/internal/processing/util.go b/internal/processing/util.go index 1e86c8f76..27cb9fabe 100644 --- a/internal/processing/util.go +++ b/internal/processing/util.go @@ -25,7 +25,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/transport" @@ -54,21 +53,24 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel. mentionedAccount = localAccount } else { var requestingUsername string + if originAccount.Domain == "" { requestingUsername = originAccount.Username } - remoteAccount, err := federator.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: requestingUsername, - RemoteAccountUsername: username, - RemoteAccountHost: domain, - }) + + remoteAccount, err := federator.GetAccountByUsernameDomain( + transport.WithFastfail(ctx), + requestingUsername, + username, + domain, + false, + ) if err != nil { - return nil, fmt.Errorf("error dereferencing account: %s", err) + return nil, fmt.Errorf("parseMentionFunc: error fetching account: %s", err) } // we were able to resolve it! mentionedAccount = remoteAccount - } mentionID, err := id.NewRandomULID() diff --git a/internal/trans/model/account.go b/internal/trans/model/account.go index 60c0af51c..e29293fdd 100644 --- a/internal/trans/model/account.go +++ b/internal/trans/model/account.go @@ -46,7 +46,6 @@ type Account struct { StatusFormat string `json:"statusFormat,omitempty" bun:",nullzero"` URI string `json:"uri" bun:",nullzero"` URL string `json:"url" bun:",nullzero"` - LastWebfingeredAt *time.Time `json:"lastWebfingeredAt,omitempty" bun:",nullzero"` InboxURI string `json:"inboxURI" bun:",nullzero"` OutboxURI string `json:"outboxURI" bun:",nullzero"` FollowingURI string `json:"followingUri" bun:",nullzero"` diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 7df2a138e..fff9c807a 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -33,7 +33,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/uris" ) -func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) { +func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string) (*gtsmodel.Account, error) { // first check if we actually already know this account uriProp := accountable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { @@ -41,18 +41,6 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a } uri := uriProp.GetIRI() - if !update { - acct, err := c.db.GetAccountByURI(ctx, uri.String()) - if err == nil { - // we already know this account so we can skip generating it - return acct, nil - } - if err != db.ErrNoEntries { - // we don't know the account and there's been a real error - return nil, fmt.Errorf("error getting account with uri %s from the database: %s", uri.String(), err) - } - } - // we don't know the account, or we're being told to update it, so we need to generate it from the person -- at least we already have the URI! acct := >smodel.Account{} acct.URI = uri.String() @@ -169,16 +157,12 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a acct.InboxURI = accountable.GetActivityStreamsInbox().GetIRI().String() } - // SharedInboxURI - if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil { - var sharedInbox string - - // only trust shared inbox if it has at least two domains, - // from the right, in common with the domain of the account - if dns.CompareDomainName(acct.Domain, sharedInboxURI.Host) >= 2 { - sharedInbox = sharedInboxURI.String() - } - + // SharedInboxURI: + // only trust shared inbox if it has at least two domains, + // from the right, in common with the domain of the account + if sharedInboxURI := ap.ExtractSharedInbox(accountable); // nocollapse + sharedInboxURI != nil && dns.CompareDomainName(acct.Domain, sharedInboxURI.Host) >= 2 { + sharedInbox := sharedInboxURI.String() acct.SharedInboxURI = &sharedInbox } diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go index 3613f38f4..ed7b97ebd 100644 --- a/internal/typeutils/astointernal_test.go +++ b/internal/typeutils/astointernal_test.go @@ -52,7 +52,7 @@ func (suite *ASToInternalTestSuite) jsonToType(in string) vocab.Type { func (suite *ASToInternalTestSuite) TestParsePerson() { testPerson := suite.testPeople["https://unknown-instance.com/users/brand_new_person"] - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "") suite.NoError(err) suite.Equal("https://unknown-instance.com/users/brand_new_person", acct.URI) @@ -74,7 +74,7 @@ func (suite *ASToInternalTestSuite) TestParsePerson() { func (suite *ASToInternalTestSuite) TestParsePersonWithSharedInbox() { testPerson := suite.testPeople["https://turnip.farm/users/turniplover6969"] - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "") suite.NoError(err) suite.Equal("https://turnip.farm/users/turniplover6969", acct.URI) @@ -131,7 +131,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() { suite.FailNow("type not coercible") } - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "") suite.NoError(err) suite.Equal("https://mastodon.social/inbox", *acct.SharedInboxURI) } @@ -183,7 +183,7 @@ func (suite *ASToInternalTestSuite) TestParseOwncastService() { suite.FailNow("type not coercible") } - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "") suite.NoError(err) suite.Equal("rgh", acct.Username) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 0f741ddb1..6a783e190 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -111,15 +111,10 @@ type TypeConverter interface { ACTIVITYSTREAMS MODEL TO INTERNAL (gts) MODEL */ - // ASPersonToAccount converts a remote account/person/application representation into a gts model account. + // ASRepresentationToAccount converts a remote account/person/application representation into a gts model account. // - // If update is false, and the account is already known in the database, then the existing account entry will be returned. - // If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account - // will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile. - // - // If accountDomain is set (not an empty string) then this value will be used as the account's Domain. If not set, - // then the Host of the accountable's AP ID will be used instead. - ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) + // If accountDomain is provided then this value will be used as the account's Domain, else the AP ID host. + ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string) (*gtsmodel.Account, error) // ASStatus converts a remote activitystreams 'status' representation into a gts model status. ASStatusToStatus(ctx context.Context, statusable ap.Statusable) (*gtsmodel.Status, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request. diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index f0c501fa9..62e43f8c7 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -27,11 +27,7 @@ func (c *converter) FollowRequestToFollow(ctx context.Context, f *gtsmodel.Follo func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boostingAccount *gtsmodel.Account) (*gtsmodel.Status, error) { // the wrapper won't use the same ID as the boosted status so we generate some new UUIDs accountURIs := uris.GenerateURIsForAccount(boostingAccount.Username) - boostWrapperStatusID, err := id.NewULID() - if err != nil { - return nil, err - } - + boostWrapperStatusID := id.NewULID() boostWrapperStatusURI := accountURIs.StatusesURI + "/" + boostWrapperStatusID boostWrapperStatusURL := accountURIs.StatusesURL + "/" + boostWrapperStatusID diff --git a/internal/validate/account_test.go b/internal/validate/account_test.go index 5303b4126..3c918e1fc 100644 --- a/internal/validate/account_test.go +++ b/internal/validate/account_test.go @@ -66,7 +66,7 @@ func happyAccount() *gtsmodel.Account { StatusFormat: "plain", URI: "http://localhost:8080/users/the_mighty_zork", URL: "http://localhost:8080/@the_mighty_zork", - LastWebfingeredAt: time.Time{}, + FetchedAt: time.Time{}, InboxURI: "http://localhost:8080/users/the_mighty_zork/inbox", OutboxURI: "http://localhost:8080/users/the_mighty_zork/outbox", FollowersURI: "http://localhost:8080/users/the_mighty_zork/followers", @@ -117,14 +117,14 @@ func (suite *AccountValidateTestSuite) TestValidateAccountNoCreatedAt() { suite.NoError(err) } -// LastWebfingeredAt must be defined if remote account +// FetchedAt must be defined if remote account func (suite *AccountValidateTestSuite) TestValidateAccountNoWebfingeredAt() { a := happyAccount() a.Domain = "example.org" - a.LastWebfingeredAt = time.Time{} + a.FetchedAt = time.Time{} err := validate.Struct(*a) - suite.EqualError(err, "Key: 'Account.LastWebfingeredAt' Error:Field validation for 'LastWebfingeredAt' failed on the 'required_with' tag") + suite.EqualError(err, "Key: 'Account.FetchedAt' Error:Field validation for 'FetchedAt' failed on the 'required_with' tag") } // Username must be set @@ -139,7 +139,7 @@ func (suite *AccountValidateTestSuite) TestValidateAccountUsername() { // Domain must be either empty (for local accounts) or proper fqdn (for remote accounts) func (suite *AccountValidateTestSuite) TestValidateAccountDomain() { a := happyAccount() - a.LastWebfingeredAt = time.Now() + a.FetchedAt = time.Now() a.Domain = "" err := validate.Struct(*a) @@ -204,7 +204,7 @@ func (suite *AccountValidateTestSuite) TestValidateAttachmentRemoteURLs() { // Default privacy must be set if account is local func (suite *AccountValidateTestSuite) TestValidatePrivacy() { a := happyAccount() - a.LastWebfingeredAt = time.Now() + a.FetchedAt = time.Now() a.Privacy = "" err := validate.Struct(*a) @@ -261,7 +261,7 @@ func (suite *AccountValidateTestSuite) TestValidateAccountURI() { // ActivityPub URIs must be set on account if it's local func (suite *AccountValidateTestSuite) TestValidateAccountURIs() { a := happyAccount() - a.LastWebfingeredAt = time.Now() + a.FetchedAt = time.Now() a.InboxURI = "invalid-uri" a.OutboxURI = "invalid-uri" @@ -319,7 +319,7 @@ func (suite *AccountValidateTestSuite) TestValidateActorType() { // Private key must be set on local accounts func (suite *AccountValidateTestSuite) TestValidatePrivateKey() { a := happyAccount() - a.LastWebfingeredAt = time.Now() + a.FetchedAt = time.Now() a.PrivateKey = nil err := validate.Struct(*a) |