diff options
Diffstat (limited to 'internal/federation')
-rw-r--r-- | internal/federation/dereference.go | 52 | ||||
-rw-r--r-- | internal/federation/dereferencing/account.go | 641 | ||||
-rw-r--r-- | internal/federation/dereferencing/account_test.go | 323 | ||||
-rw-r--r-- | internal/federation/dereferencing/dereferencer.go | 41 | ||||
-rw-r--r-- | internal/federation/dereferencing/error.go | 18 | ||||
-rw-r--r-- | internal/federation/dereferencing/handshake.go | 13 | ||||
-rw-r--r-- | internal/federation/dereferencing/status.go | 13 | ||||
-rw-r--r-- | internal/federation/federatingdb/create.go | 24 | ||||
-rw-r--r-- | internal/federation/federatingdb/update.go | 2 | ||||
-rw-r--r-- | internal/federation/federatingprotocol.go | 8 | ||||
-rw-r--r-- | internal/federation/federator.go | 19 | ||||
-rw-r--r-- | internal/federation/gone.go | 12 | ||||
-rw-r--r-- | internal/federation/handshake.go | 9 |
13 files changed, 309 insertions, 866 deletions
diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go deleted file mode 100644 index 5acc038e9..000000000 --- a/internal/federation/dereference.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package federation - -import ( - "context" - "net/url" - - "github.com/superseriousbusiness/gotosocial/internal/ap" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -func (f *federator) GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) { - return f.dereferencer.GetAccount(ctx, params) -} - -func (f *federator) GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) { - return f.dereferencer.GetStatus(ctx, username, remoteStatusID, refetch, includeParent) -} - -func (f *federator) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) { - return f.dereferencer.EnrichRemoteStatus(ctx, username, status, includeParent) -} - -func (f *federator) DereferenceRemoteThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) { - f.dereferencer.DereferenceThread(ctx, username, statusIRI, status, statusable) -} - -func (f *federator) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { - return f.dereferencer.GetRemoteInstance(ctx, username, remoteInstanceURI) -} - -func (f *federator) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error { - return f.dereferencer.DereferenceAnnounce(ctx, announce, requestingUsername) -} diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 4910273bd..be2ec9cee 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -25,10 +25,8 @@ import ( "fmt" "io" "net/url" - "strings" "time" - "github.com/miekg/dns" "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" @@ -38,375 +36,222 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/internal/transport" ) -var webfingerInterval = -48 * time.Hour // 2 days in the past +func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) { + var ( + account *gtsmodel.Account + uriStr = uri.String() + err error + ) -func instanceAccount(account *gtsmodel.Account) bool { - return strings.EqualFold(account.Username, account.Domain) || - account.FollowersURI == "" || - account.FollowingURI == "" || - (account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor")) -} + // Search the database for existing account with ID URI. + account, err = d.db.GetAccountByURI(ctx, uriStr) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err) + } -// GetAccountParams wraps parameters for an account lookup. -type GetAccountParams struct { - // The username of the user doing the lookup request (optional). - // If not set, then the GtS instance account will be used to do the lookup. - RequestingUsername string - // The ActivityPub URI of the account (optional). - // If not set (nil), the ActivityPub URI of the account will be discovered - // via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost - // if this parameter is not set. - RemoteAccountID *url.URL - // The username of the account (optional). - // If RemoteAccountID is not set, then this value must be set. - RemoteAccountUsername string - // The host of the account (optional). - // If RemoteAccountID is not set, then this value must be set. - RemoteAccountHost string - // Whether to do a blocking call to the remote instance. If true, - // then the account's media and other fields will be fully dereferenced before it is returned. - // If false, then the account's media and other fields will be dereferenced in the background, - // so only a minimal account representation will be returned by GetRemoteAccount. - Blocking bool - // Whether to skip making calls to remote instances. This is useful when you want to - // quickly fetch a remote account from the database or fail, and don't want to cause - // http requests to go flying around. - SkipResolve bool - // PartialAccount can be used if the GetRemoteAccount call results from a federated/ap - // account update. In this case, we will already have a partial representation of the account, - // derived from converting the AP representation to a gtsmodel representation. If this field - // is provided, then GetRemoteAccount will use this as a basis for building the full account. - PartialAccount *gtsmodel.Account -} + if account == nil { + // Else, search the database for existing by ID URL. + account, err = d.db.GetAccountByURL(ctx, uriStr) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err) + } + } -type lookupType int + if account == nil { + // Ensure that this is isn't a search for a local account. + if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() { + return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries + } -const ( - lookupPartialLocal lookupType = iota - lookupPartial - lookupURILocal - lookupURI - lookupMentionLocal - lookupMention - lookupBad -) + // Create and pass-through a new bare-bones model for dereferencing. + return d.enrichAccount(ctx, requestUser, uri, >smodel.Account{ + ID: id.NewULID(), + Domain: uri.Host, + URI: uriStr, + }, false, true) + } -func getLookupType(params GetAccountParams) lookupType { - switch { - case params.PartialAccount != nil: - if params.PartialAccount.Domain == "" || params.PartialAccount.Domain == config.GetHost() || params.PartialAccount.Domain == config.GetAccountDomain() { - return lookupPartialLocal - } - return lookupPartial - case params.RemoteAccountID != nil: - if host := params.RemoteAccountID.Host; host == config.GetHost() || host == config.GetAccountDomain() { - return lookupURILocal - } - return lookupURI - case params.RemoteAccountUsername != "": - if params.RemoteAccountHost == "" || params.RemoteAccountHost == config.GetHost() || params.RemoteAccountHost == config.GetAccountDomain() { - return lookupMentionLocal - } - return lookupMention - default: - return lookupBad + // Try to update existing account model + enriched, err := d.enrichAccount(ctx, requestUser, uri, account, false, block) + if err != nil { + log.Errorf("error enriching remote account: %v", err) + return account, nil // fall back to returning existing } + + return enriched, nil } -// GetAccount completely dereferences an account, converts it to a GtS model account, -// puts or updates it in the database (if necessary), and returns it to a caller. -// -// GetAccount will guard against trying to do http calls to fetch an account that belongs to this instance. -// Instead of making calls, it will just return the account early if it finds it, or return an error. -// -// Even if a fastfail context is used, and something goes wrong, an account might still be returned instead -// of an error, if we already had the account in our database (in other words, if we just needed to try -// fingering/refreshing the account again). The rationale for this is that it's more useful to be able -// to provide *something* to the caller, even if that something is not necessarily 100% up to date. -func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundAccount *gtsmodel.Account, err error) { - /* - In this function we want to retrieve a gtsmodel representation of a remote account, with its proper - accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth. - - There are a few different paths through this function, and the path taken depends on how much - initial information we are provided with via parameters, how much information we already have stored, - and what we're allowed to do according to the parameters we've been passed. - - Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the - account username + host, so we can check in our database and return if possible. - - Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host. - In this case, we can use the URI to resolve the remote account and find the username, - and then we can webfinger the account to discover the accountDomain if necessary. - - Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI. - In this case, we can webfinger the account to discover the URI, and then dereference - from that. - */ - - // this first step checks if we have the - // account in the database somewhere already, - // or if we've been provided it as a partial - switch getLookupType(params) { - case lookupPartialLocal: - params.SkipResolve = true - fallthrough - case lookupPartial: - foundAccount = params.PartialAccount - case lookupURILocal: - params.SkipResolve = true - fallthrough - case lookupURI: - // see if we have this in the db already with this uri/url - uri := params.RemoteAccountID.String() - - if a, dbErr := d.db.GetAccountByURI(ctx, uri); dbErr == nil { - // got it, break here to leave early - foundAccount = a - break - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with uri %s: %w", uri, dbErr)) - break - } +func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) { + if domain == config.GetHost() || domain == config.GetAccountDomain() { + // We do local lookups using an empty domain, + // else it will fail the db search below. + domain = "" + } - // dbErr was just db.ErrNoEntries so search by url instead - if a, dbErr := d.db.GetAccountByURL(ctx, uri); dbErr == nil { - // got it - foundAccount = a - break - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with url %s: %w", uri, dbErr)) - break - } - case lookupMentionLocal: - params.SkipResolve = true - params.RemoteAccountHost = "" - fallthrough - case lookupMention: - // see if we have this in the db already with this username/host - if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, params.RemoteAccountHost); dbErr == nil { - foundAccount = a - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr)) + // Search the database for existing account with USERNAME@DOMAIN + account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err) + } + + if account == nil { + // Check for failed local lookup. + if domain == "" { + return nil, NewErrNotRetrievable(err) // will be db.ErrNoEntries } - default: - err = newErrBadRequest(errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account")) + + // Create and pass-through a new bare-bones model for dereferencing. + return d.enrichAccount(ctx, requestUser, nil, >smodel.Account{ + ID: id.NewULID(), + Username: username, + Domain: domain, + }, false, true) } - // bail if we've set a real error, and not just no entries in the db + // Try to update existing account model + enriched, err := d.enrichAccount(ctx, requestUser, nil, account, false, block) if err != nil { - return + log.Errorf("GetAccountByUsernameDomain: error enriching account from remote: %v", err) + return account, nil // fall back to returning unchanged existing account model } - if params.SkipResolve { - // if we can't resolve, return already since there's nothing more we can do - if foundAccount == nil { - err = newErrNotRetrievable(errors.New("GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")) - } - return - } + return enriched, nil +} - // if we reach this point, we have some remote calls to make +func (d *deref) UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) { + return d.enrichAccount(ctx, requestUser, nil, account, force, false) +} - var accountable ap.Accountable - if params.RemoteAccountUsername == "" && params.RemoteAccountHost == "" { - // if we're still missing some params, try to populate them now - params.RemoteAccountHost = params.RemoteAccountID.Host - if foundAccount != nil { - // username is easy if we found something already - params.RemoteAccountUsername = foundAccount.Username - } else { - // if we didn't already have it, we have to dereference it from remote - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } +// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary. +func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, force, block bool) (*gtsmodel.Account, error) { + if account.IsLocal() { + // Can't update local accounts. + return account, nil + } - var apError error - params.RemoteAccountUsername, apError = ap.ExtractPreferredUsername(accountable) - if apError != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error extracting Accountable username: %w", apError)) - return - } - } + if !account.CreatedAt.IsZero() && account.IsInstance() { + // Existing instance account. No need for update. + return account, nil } - // if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set - // params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it - - // we finger to fetch the account domain but just in case we're not fingering, make a best guess - // already about what the account domain might be; this var will be overwritten later if necessary - var accountDomain string - switch { - case foundAccount != nil: - accountDomain = foundAccount.Domain - case params.RemoteAccountID != nil: - accountDomain = params.RemoteAccountID.Host - default: - accountDomain = params.RemoteAccountHost - } - - // to save on remote calls, only webfinger if: - // - we don't know the remote account ActivityPub ID yet OR - // - we haven't found the account yet in some other way OR - // - we were passed a partial account in params OR - // - we haven't webfingered the account for two days AND the account isn't an instance account - var fingered time.Time - var refreshFinger bool - if foundAccount != nil { - refreshFinger = foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount) - } - - if params.RemoteAccountID == nil || foundAccount == nil || params.PartialAccount != nil || refreshFinger { - if ad, accountURI, fingerError := d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost); fingerError != nil { - if !refreshFinger { - // only return with an error if this wasn't just a refresh finger; - // that is, if we actually *needed* to finger in order to get the account, - // otherwise we can just continue and we'll try again in 2 days - err = newErrNotRetrievable(fmt.Errorf("GetRemoteAccount: error while fingering: %w", fingerError)) - return - } - log.Infof("error doing non-vital webfinger refresh call to %s: %s", params.RemoteAccountHost, err) - } else { - accountDomain = ad - params.RemoteAccountID = accountURI - } - fingered = time.Now() - } - - if !fingered.IsZero() && foundAccount == nil { - // if we just fingered and now have a discovered account domain but still no account, - // we should do a final lookup in the database with the discovered username + accountDomain - // to make absolutely sure we don't already have this account - if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, accountDomain); dbErr == nil { - foundAccount = a - } else if !errors.Is(dbErr, db.ErrNoEntries) { - // a real error - err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr)) - return + if !force { + const interval = time.Hour * 48 + + // If this account was updated recently (last interval), we return as-is. + if next := account.FetchedAt.Add(interval); time.Now().Before(next) { + return account, nil } } - // we may have some extra information already, like the account we had in the db, or the - // accountable representation that we dereferenced from remote - if foundAccount == nil { - // if we still don't have a remoteAccountID here we're boned - if params.RemoteAccountID == nil { - err = newErrNotRetrievable(errors.New("GetRemoteAccount: could not populate find an account nor populate params.RemoteAccountID")) - return - } + if account.Username != "" { + // A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info. + accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain) - // deference accountable if we didn't earlier - if accountable == nil { - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } + if err != nil && account.URI == "" { + // this is a new account (to us) with username@domain but failed + // webfinger, there is nothing more we can do in this situation. + return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err) } - // then convert - foundAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false) - if err != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error converting Accountable to account: %w", err)) - return + if err == nil { + // Update account with latest info. + account.URI = accURI.String() + account.Domain = accDomain + uri = accURI } + } - // this is a new account so we need to generate a new ID for it - var ulid string - ulid, err = id.NewRandomULID() - if err != nil { - err = newErrOther(fmt.Errorf("GetRemoteAccount: error generating new id for account: %w", err)) - return - } - foundAccount.ID = ulid + if uri == nil { + var err error - if _, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking); populateErr != nil { - // it's not the end of the world if we can't populate account fields, but we do want to log it - log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr) + // No URI provided / found, must parse from account. + uri, err = url.Parse(account.URI) + if err != nil { + return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err) } + } - foundAccount.LastWebfingeredAt = fingered - foundAccount.UpdatedAt = time.Now() + // Check whether this account URI is a blocked domain / subdomain + if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil { + return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err)) + } else if blocked { + return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host) + } - if dbErr := d.db.PutAccount(ctx, foundAccount); dbErr != nil { - err = newErrDB(fmt.Errorf("GetRemoteAccount: error putting new account: %w", dbErr)) - return - } + // Mark deref+update handshake start + d.startHandshake(requestUser, uri) + defer d.stopHandshake(requestUser, uri) - return // the new account + // Dereference this account to get the latest available. + apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri) + if err != nil { + return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err) } - // we had the account already, but now we know the account domain, so update it if it's different - var accountDomainChanged bool - if !strings.EqualFold(foundAccount.Domain, accountDomain) { - accountDomainChanged = true - foundAccount.Domain = accountDomain + // Convert the dereferenced AP account object to our GTS model. + latestAcc, err := d.typeConverter.ASRepresentationToAccount( + ctx, apubAcc, account.Domain, + ) + if err != nil { + return nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err) } - // if SharedInboxURI is nil, that means we don't know yet if this account has - // a shared inbox available for it, so we need to check this here - var sharedInboxChanged bool - if foundAccount.SharedInboxURI == nil { - // we need the accountable for this, so get it if we don't have it yet - if accountable == nil { - var derefErr error - accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) - if derefErr != nil { - err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable") - return - } - } + if account.Username == "" { + // No username was provided, so no webfinger was attempted earlier. + // + // Now we have a username we can attempt it now, this ensures up-to-date accountdomain info. + accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host) - // This can be: - // - an empty string (we know it doesn't have a shared inbox) OR - // - a string URL (we know it does a shared inbox). - // Set it either way! - var sharedInbox string - - if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil { - // only trust shared inbox if it has at least two domains, - // from the right, in common with the domain of the account - if dns.CompareDomainName(foundAccount.Domain, sharedInboxURI.Host) >= 2 { - sharedInbox = sharedInboxURI.String() - } + if err == nil { + // Update account with latest info. + latestAcc.Domain = accDomain } - - sharedInboxChanged = true - foundAccount.SharedInboxURI = &sharedInbox } - // make sure the account fields are populated before returning: - // the caller might want to block until everything is loaded - fieldsChanged, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking) - if populateErr != nil { - // it's not the end of the world if we can't populate account fields, but we do want to log it - log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr) + // Ensure ID is set and update fetch time. + latestAcc.ID = account.ID + latestAcc.FetchedAt = time.Now() + + // Fetch latest account media (TODO: check for changed URI to previous). + if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil { + log.Errorf("error fetching remote media for account %s: %v", uri, err) } - var fingeredChanged bool - if !fingered.IsZero() { - fingeredChanged = true - foundAccount.LastWebfingeredAt = fingered + // Fetch the latest remote account emoji IDs used in account display name/bio. + _, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser) + if err != nil { + log.Errorf("error fetching remote emojis for account %s: %v", uri, err) } - if accountDomainChanged || sharedInboxChanged || fieldsChanged || fingeredChanged { - if dbErr := d.db.UpdateAccount(ctx, foundAccount); dbErr != nil { - err = newErrDB(fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %w", dbErr)) - return + if account.CreatedAt.IsZero() { + // CreatedAt will be zero if no local copy was + // found in one of the GetAccountBy___() functions. + // + // Set time of creation from the last-fetched date. + latestAcc.CreatedAt = latestAcc.FetchedAt + latestAcc.UpdatedAt = latestAcc.FetchedAt + + // This is a new account, we need to place it in the database. + if err := d.db.PutAccount(ctx, latestAcc); err != nil { + return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err) + } + } else { + // Set time of update from the last-fetched date. + latestAcc.UpdatedAt = latestAcc.FetchedAt + + // Use existing account values. + latestAcc.CreatedAt = account.CreatedAt + latestAcc.Language = account.Language + + // This is an existing account, update the model in the database. + if err := d.db.UpdateAccount(ctx, latestAcc); err != nil { + return nil, fmt.Errorf("enrichAccount: error updating database: %w", err) } } - return // the account we already had + possibly updated + return latestAcc, nil } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -414,13 +259,6 @@ func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundA // // Will work for Person, Application, or Service models. func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) { - d.startHandshake(username, remoteAccountID) - defer d.stopHandshake(username, remoteAccountID) - - if blocked, err := d.db.IsDomainBlocked(ctx, remoteAccountID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host) - } - transport, err := d.transportController.NewTransportForUsername(ctx, username) if err != nil { return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err) @@ -441,83 +279,23 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err) } + //nolint shutup linter switch t.GetTypeName() { case ap.ActorApplication: - p, ok := t.(vocab.ActivityStreamsApplication) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application") - } - return p, nil + return t.(vocab.ActivityStreamsApplication), nil case ap.ActorGroup: - p, ok := t.(vocab.ActivityStreamsGroup) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams group") - } - return p, nil + return t.(vocab.ActivityStreamsGroup), nil case ap.ActorOrganization: - p, ok := t.(vocab.ActivityStreamsOrganization) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams organization") - } - return p, nil + return t.(vocab.ActivityStreamsOrganization), nil case ap.ActorPerson: - p, ok := t.(vocab.ActivityStreamsPerson) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person") - } - return p, nil + return t.(vocab.ActivityStreamsPerson), nil case ap.ActorService: - p, ok := t.(vocab.ActivityStreamsService) - if !ok { - return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service") - } - return p, nil + return t.(vocab.ActivityStreamsService), nil } return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName())) } -// populateAccountFields makes a best effort to populate fields on an account such as emojis, avatar, header. -// Will return true if one of these things changed on the passed-in account. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { - // if we're dealing with an instance account, just bail, we don't need to do anything - if instanceAccount(account) { - return false, nil - } - - accountURI, err := url.Parse(account.URI) - if err != nil { - return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %w", account.URI, err) - } - - blocked, dbErr := d.db.IsDomainBlocked(ctx, accountURI.Host) - if dbErr != nil { - return false, fmt.Errorf("populateAccountFields: eror checking for block of domain %s: %w", accountURI.Host, err) - } - - if blocked { - return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host) - } - - var changed bool - - // fetch the header and avatar - if mediaChanged, err := d.fetchRemoteAccountMedia(ctx, account, requestingUsername, blocking); err != nil { - return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %w", err) - } else if mediaChanged { - changed = mediaChanged - } - - // fetch any emojis used in note, fields, display name, etc - if emojisChanged, err := d.fetchRemoteAccountEmojis(ctx, account, requestingUsername); err != nil { - return false, fmt.Errorf("populateAccountFields: error fetching emojis for account: %w", err) - } else if emojisChanged { - changed = emojisChanged - } - - return changed, nil -} - // fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account, // using a transport on behalf of requestingUsername. // @@ -530,39 +308,30 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { - var ( - changed bool - t transport.Transport - ) +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error { + // Fetch a transport beforehand for either(or both) avatar / header dereferencing. + tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) + } - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") { + if targetAccount.AvatarRemoteURL != "" { var processingMedia *media.ProcessingMedia + // Parse the target account's avatar URL into URL object. + avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) + if err != nil { + return err + } + d.dereferencingAvatarsLock.Lock() // LOCK HERE // first check if we're already processing this media if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } else { - // we're not already processing it so start now - avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return changed, err - } - - if t == nil { - var err error - t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) - } - } - data := func(innerCtx context.Context) (io.ReadCloser, int64, error) { - return t.DereferenceMedia(innerCtx, avatarIRI) + return tsport.DereferenceMedia(innerCtx, avatarIRI) } avatar := true @@ -572,7 +341,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm }) if err != nil { d.dereferencingAvatarsLock.Unlock() - return changed, err + return err } // store it in our map to indicate it's in process @@ -595,7 +364,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := loadAndCleanup(ctx, load, cleanup); err != nil { - return changed, err + return err } } else { // ...otherwise do it async @@ -609,36 +378,25 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() - changed = true } - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") { + if targetAccount.HeaderRemoteURL != "" { var processingMedia *media.ProcessingMedia + // Parse the target account's header URL into URL object. + headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) + if err != nil { + return err + } + d.dereferencingHeadersLock.Lock() // LOCK HERE // first check if we're already processing this media if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } else { - // we're not already processing it so start now - headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return changed, err - } - - if t == nil { - var err error - t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - d.dereferencingAvatarsLock.Unlock() - return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) - } - } - data := func(innerCtx context.Context) (io.ReadCloser, int64, error) { - return t.DereferenceMedia(innerCtx, headerIRI) + return tsport.DereferenceMedia(innerCtx, headerIRI) } header := true @@ -648,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm }) if err != nil { d.dereferencingAvatarsLock.Unlock() - return changed, err + return err } // store it in our map to indicate it's in process @@ -671,7 +429,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := loadAndCleanup(ctx, load, cleanup); err != nil { - return changed, err + return err } } else { // ...otherwise do it async @@ -685,10 +443,9 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() - changed = true } - return changed, nil + return nil } func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) { diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index eb795d169..7898aec87 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -27,7 +27,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -39,17 +38,19 @@ func (suite *AccountTestSuite) TestDereferenceGroup() { fetchingAccount := suite.testAccounts["local_account_1"] groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group") - group, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: groupURL, - }) + group, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + groupURL, + false, + ) suite.NoError(err) suite.NotNil(group) // group values should be set suite.Equal("https://unknown-instance.com/groups/some_group", group.URI) suite.Equal("https://unknown-instance.com/@some_group", group.URL) - suite.WithinDuration(time.Now(), group.LastWebfingeredAt, 5*time.Second) + suite.WithinDuration(time.Now(), group.FetchedAt, 5*time.Second) // group should be in the database dbGroup, err := suite.db.GetAccountByURI(context.Background(), group.URI) @@ -62,17 +63,19 @@ func (suite *AccountTestSuite) TestDereferenceService() { fetchingAccount := suite.testAccounts["local_account_1"] serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh") - service, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: serviceURL, - }) + service, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + serviceURL, + false, + ) suite.NoError(err) suite.NotNil(service) // service values should be set suite.Equal("https://owncast.example.org/federation/user/rgh", service.URI) suite.Equal("https://owncast.example.org/federation/user/rgh", service.URL) - suite.WithinDuration(time.Now(), service.LastWebfingeredAt, 5*time.Second) + suite.WithinDuration(time.Now(), service.FetchedAt, 5*time.Second) // service should be in the database dbService, err := suite.db.GetAccountByURI(context.Background(), service.URI) @@ -93,10 +96,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -111,10 +116,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb suite.FailNow(err.Error()) } - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -124,10 +131,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: targetAccount.Username, - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -137,11 +146,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() { fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: targetAccount.Username, - RemoteAccountHost: config.GetHost(), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse(targetAccount.URI), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -151,12 +161,13 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL fetchingAccount := suite.testAccounts["local_account_1"] targetAccount := suite.testAccounts["local_account_2"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(targetAccount.URI), - RemoteAccountUsername: targetAccount.Username, - RemoteAccountHost: config.GetHost(), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + targetAccount.Username, + config.GetHost(), + false, + ) suite.NoError(err) suite.NotNil(fetchedAccount) suite.Empty(fetchedAccount.Domain) @@ -165,248 +176,50 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: "thisaccountdoesnotexist", - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + "thisaccountdoesnotexist", + config.GetHost(), + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountUsername: "thisaccountdoesnotexist", - RemoteAccountHost: "localhost:8080", - }) + fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain( + context.Background(), + fetchingAccount.Username, + "thisaccountdoesnotexist", + "localhost:8080", + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() { fetchingAccount := suite.testAccounts["local_account_1"] - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"), - }) + fetchedAccount, err := suite.dereferencer.GetAccountByURI( + context.Background(), + fetchingAccount.Username, + testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"), + false, + ) var errNotRetrievable *dereferencing.ErrNotRetrievable suite.ErrorAs(err, &errNotRetrievable) - suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it") + suite.EqualError(err, "item could not be retrieved: no entries") suite.Nil(fetchedAccount) } -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() { - fetchingAccount := suite.testAccounts["local_account_1"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // dereference an emoji we don't have stored yet - { - URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", - Shortcode: "kip_van_den_bos", - UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", - Disabled: testrig.FalseBool(), - VisibleInPicker: testrig.FalseBool(), - Domain: "fossbros-anonymous.io", - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) -} - -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() { - fetchingAccount := suite.testAccounts["local_account_1"] - - knownEmoji := suite.testEmojis["yell"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // an emoji we already have - { - URI: knownEmoji.URI, - Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.UpdatedAt, - ImageUpdatedAt: knownEmoji.ImageUpdatedAt, - ImageRemoteURL: knownEmoji.ImageRemoteURL, - Disabled: knownEmoji.Disabled, - VisibleInPicker: knownEmoji.VisibleInPicker, - Domain: knownEmoji.Domain, - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) -} - -func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() { - fetchingAccount := suite.testAccounts["local_account_1"] - - knownEmoji := suite.testEmojis["yell"] - - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountPartial := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // an emoji we already have - { - URI: knownEmoji.URI, - Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.UpdatedAt, - ImageUpdatedAt: knownEmoji.ImageUpdatedAt, - ImageRemoteURL: knownEmoji.ImageRemoteURL, - Disabled: knownEmoji.Disabled, - VisibleInPicker: knownEmoji.VisibleInPicker, - Domain: knownEmoji.Domain, - }, - }, - } - - fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount) - suite.NotNil(fetchedAccount.EmojiIDs) - suite.NotNil(fetchedAccount.Emojis) - suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI) - - remoteAccountPartial2 := >smodel.Account{ - ID: remoteAccount.ID, - ActorType: remoteAccount.ActorType, - Language: remoteAccount.Language, - CreatedAt: remoteAccount.CreatedAt, - UpdatedAt: remoteAccount.UpdatedAt, - Username: remoteAccount.Username, - Domain: remoteAccount.Domain, - DisplayName: remoteAccount.DisplayName, - URI: remoteAccount.URI, - InboxURI: remoteAccount.URI, - SharedInboxURI: remoteAccount.SharedInboxURI, - PublicKeyURI: remoteAccount.PublicKeyURI, - URL: remoteAccount.URL, - FollowingURI: remoteAccount.FollowingURI, - FollowersURI: remoteAccount.FollowersURI, - OutboxURI: remoteAccount.OutboxURI, - FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI, - Emojis: []*gtsmodel.Emoji{ - // dereference an emoji we don't have stored yet - { - URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", - Shortcode: "kip_van_den_bos", - UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), - ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", - Disabled: testrig.FalseBool(), - VisibleInPicker: testrig.FalseBool(), - Domain: "fossbros-anonymous.io", - }, - }, - } - - fetchedAccount2, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{ - RequestingUsername: fetchingAccount.Username, - RemoteAccountID: testrig.URLMustParse(remoteAccount.URI), - RemoteAccountHost: remoteAccount.Domain, - RemoteAccountUsername: remoteAccount.Username, - PartialAccount: remoteAccountPartial2, - Blocking: true, - }) - suite.NoError(err) - suite.NotNil(fetchedAccount2) - suite.NotNil(fetchedAccount2.EmojiIDs) - suite.NotNil(fetchedAccount2.Emojis) - suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI) -} - func TestAccountTestSuite(t *testing.T) { suite.Run(t, new(AccountTestSuite)) } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 23f91ecae..122cbb59c 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -33,7 +33,17 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetAccount(ctx context.Context, params GetAccountParams) (*gtsmodel.Account, error) + // GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the + // last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before. + GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) + + // GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either + // check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before. + GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) + + // UpdateAccount updates the given account if last_fetched is beyond fetch interval (or if force is set). An updated account model is returned, any media fetching is done async. + UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) + GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) @@ -44,7 +54,7 @@ type Dereferencer interface { GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) - Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool + Handshaking(username string, remoteAccountID *url.URL) bool } type deref struct { @@ -53,28 +63,25 @@ type deref struct { transportController transport.Controller mediaManager media.Manager dereferencingAvatars map[string]*media.ProcessingMedia - dereferencingAvatarsLock *sync.Mutex + dereferencingAvatarsLock sync.Mutex dereferencingHeaders map[string]*media.ProcessingMedia - dereferencingHeadersLock *sync.Mutex + dereferencingHeadersLock sync.Mutex dereferencingEmojis map[string]*media.ProcessingEmoji - dereferencingEmojisLock *sync.Mutex + dereferencingEmojisLock sync.Mutex handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map + handshakeSync sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewDereferencer returns a Dereferencer initialized with the given parameters. func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer { return &deref{ - db: db, - typeConverter: typeConverter, - transportController: transportController, - mediaManager: mediaManager, - dereferencingAvatars: make(map[string]*media.ProcessingMedia), - dereferencingAvatarsLock: &sync.Mutex{}, - dereferencingHeaders: make(map[string]*media.ProcessingMedia), - dereferencingHeadersLock: &sync.Mutex{}, - dereferencingEmojis: make(map[string]*media.ProcessingEmoji), - dereferencingEmojisLock: &sync.Mutex{}, - handshakeSync: &sync.Mutex{}, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaManager: mediaManager, + dereferencingAvatars: make(map[string]*media.ProcessingMedia), + dereferencingHeaders: make(map[string]*media.ProcessingMedia), + dereferencingEmojis: make(map[string]*media.ProcessingEmoji), + handshakes: make(map[string][]*url.URL), } } diff --git a/internal/federation/dereferencing/error.go b/internal/federation/dereferencing/error.go index 48804b50d..3ba357f7a 100644 --- a/internal/federation/dereferencing/error.go +++ b/internal/federation/dereferencing/error.go @@ -49,24 +49,10 @@ func (err *ErrNotRetrievable) Error() string { return fmt.Sprintf("item could not be retrieved: %v", err.wrapped) } -func newErrNotRetrievable(err error) error { +func NewErrNotRetrievable(err error) error { return &ErrNotRetrievable{wrapped: err} } -// ErrBadRequest denotes that insufficient or improperly formed parameters -// were passed into one of the dereference functions. -type ErrBadRequest struct { - wrapped error -} - -func (err *ErrBadRequest) Error() string { - return fmt.Sprintf("bad request: %v", err.wrapped) -} - -func newErrBadRequest(err error) error { - return &ErrBadRequest{wrapped: err} -} - // ErrTransportError indicates that something unforeseen went wrong creating // a transport, or while making an http call to a remote resource with a transport. type ErrTransportError struct { @@ -121,7 +107,7 @@ func wrapDerefError(derefErr error, fluff string) error { switch { case errors.Is(derefErr, transport.ErrGone): - err = newErrNotRetrievable(err) + err = NewErrNotRetrievable(err) case errors.As(derefErr, &errWrongType): err = newErrWrongType(err) default: diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go index 8533f3387..2b67e86bf 100644 --- a/internal/federation/dereferencing/handshake.go +++ b/internal/federation/dereferencing/handshake.go @@ -19,11 +19,10 @@ package dereferencing import ( - "context" "net/url" ) -func (d *deref) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool { +func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() @@ -53,11 +52,6 @@ func (d *deref) startHandshake(username string, remoteAccountID *url.URL) { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() - // lazily initialize handshakes - if d.handshakes == nil { - d.handshakes = make(map[string][]*url.URL) - } - remoteIDs, ok := d.handshakes[username] if !ok { // there was nothing in there yet, so just add this entry and return @@ -74,13 +68,8 @@ func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) { d.handshakeSync.Lock() defer d.handshakeSync.Unlock() - if d.handshakes == nil { - return - } - remoteIDs, ok := d.handshakes[username] if !ok { - // there was nothing in there yet anyway so just bail return } diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 8695c6f95..da4639c5d 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -100,7 +100,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U if status != nil { return status, nil, nil } - return nil, nil, newErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString)) + return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString)) } // if we got here, either we didn't have the status @@ -123,11 +123,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U } // we need to get the author of the status else we can't serialize it properly - if _, err = d.GetAccount(ctx, GetAccountParams{ - RequestingUsername: username, - RemoteAccountID: accountURI, - Blocking: true, - }); err != nil { + if _, err = d.GetAccountByURI(ctx, username, accountURI, true); err != nil { return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err)) } @@ -353,10 +349,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta if targetAccount == nil { // we didn't find the account in our database already // check if we can get the account remotely (dereference it) - if a, err := d.GetAccount(ctx, GetAccountParams{ - RequestingUsername: requestingUsername, - RemoteAccountID: targetAccountURI, - }); err != nil { + if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI, false); err != nil { errs = append(errs, err.Error()) } else { log.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) diff --git a/internal/federation/federatingdb/create.go b/internal/federation/federatingdb/create.go index fbe3d3ad6..62784e68d 100644 --- a/internal/federation/federatingdb/create.go +++ b/internal/federation/federatingdb/create.go @@ -100,11 +100,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec return fmt.Errorf("activityBlock: could not convert Block to gts model block") } - newID, err := id.NewULID() - if err != nil { - return err - } - block.ID = newID + block.ID = id.NewULID() if err := f.db.PutBlock(ctx, block); err != nil { return fmt.Errorf("activityBlock: database error inserting block: %s", err) @@ -263,11 +259,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re return fmt.Errorf("activityFollow: could not convert Follow to follow request: %s", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - followRequest.ID = newID + followRequest.ID = id.NewULID() if err := f.db.Put(ctx, followRequest); err != nil { return fmt.Errorf("activityFollow: database error inserting follow request: %s", err) @@ -298,11 +290,7 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece return fmt.Errorf("activityLike: could not convert Like to fave: %s", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - fave.ID = newID + fave.ID = id.NewULID() if err := f.db.Put(ctx, fave); err != nil { return fmt.Errorf("activityLike: database error inserting fave: %s", err) @@ -333,11 +321,7 @@ func (f *federatingDB) activityFlag(ctx context.Context, asType vocab.Type, rece return fmt.Errorf("activityFlag: could not convert Flag to report: %w", err) } - newID, err := id.NewULID() - if err != nil { - return err - } - report.ID = newID + report.ID = id.NewULID() if err := f.db.PutReport(ctx, report); err != nil { return fmt.Errorf("activityFlag: database error inserting report: %w", err) diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index d84bce668..04ce39a77 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -116,7 +116,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { accountable = i } - updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true) + updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "") if err != nil { return fmt.Errorf("UPDATE: error converting to account: %s", err) } diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index a63aecc66..90a2c50b2 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,7 +31,6 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/transport" @@ -206,10 +205,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount, err := f.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{ - RequestingUsername: username, - RemoteAccountID: publicKeyOwnerURI, - }) + requestingAccount, err := f.GetAccountByURI( + transport.WithFastfail(ctx), username, publicKeyOwnerURI, false, + ) if err != nil { return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err) } diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 0ba370cc4..5aa44742d 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -23,12 +23,10 @@ import ( "net/url" "github.com/superseriousbusiness/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -53,20 +51,9 @@ type Federator interface { // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode) - /* - dereferencing functions - */ - DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) - DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error - GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) - GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) - EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) - GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - - // Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID. - Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool pub.CommonBehavior pub.FederatingProtocol + dereferencing.Dereferencer } type federator struct { @@ -75,9 +62,9 @@ type federator struct { clock pub.Clock typeConverter typeutils.TypeConverter transportController transport.Controller - dereferencer dereferencing.Dereferencer mediaManager media.Manager actor pub.FederatingActor + dereferencing.Dereferencer } // NewFederator returns a new federator @@ -91,8 +78,8 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr clock: &Clock{}, typeConverter: typeConverter, transportController: transportController, - dereferencer: dereferencer, mediaManager: mediaManager, + Dereferencer: dereferencer, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/gone.go b/internal/federation/gone.go index 3d9fe3b84..e6d784181 100644 --- a/internal/federation/gone.go +++ b/internal/federation/gone.go @@ -2,12 +2,10 @@ package federation import ( "context" - "fmt" "net/url" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" ) // CheckGone checks if a tombstone exists in the database for AP Actor or Object with the given uri. @@ -17,18 +15,10 @@ func (f *federator) CheckGone(ctx context.Context, uri *url.URL) (bool, error) { // HandleGone puts a tombstone in the database, which marks an AP Actor or Object with the given uri as gone. func (f *federator) HandleGone(ctx context.Context, uri *url.URL) error { - tombstoneID, err := id.NewULID() - if err != nil { - err = fmt.Errorf("HandleGone: error generating id for new tombstone %s: %s", uri, err) - log.Error(err) - return err - } - tombstone := >smodel.Tombstone{ - ID: tombstoneID, + ID: id.NewULID(), Domain: uri.Host, URI: uri.String(), } - return f.db.PutTombstone(ctx, tombstone) } diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go index 9669e9856..9dadb0e96 100644 --- a/internal/federation/handshake.go +++ b/internal/federation/handshake.go @@ -17,12 +17,3 @@ */ package federation - -import ( - "context" - "net/url" -) - -func (f *federator) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool { - return f.dereferencer.Handshaking(ctx, username, remoteAccountID) -} |