[chore] reformat GetAccount() functionality, support updating accounts based on last_fetch (#1411)

* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove now unused error type

* return nil when wrapping nil error

* update expected error messages

* return correct url for foss satan webfinger

* add AP model for Some_User

* normalize local domain

* return notretrievable where appropriate

* expose NewErrNotRetrievable

* ensure webfinger for new accounts searched by uri

* update local account short circuit

* allow enrich to fail for already-known accounts

* remove unused LastWebfingeredAt

* expose test maps on mock http client

* update Update test

* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove nil error checks (we shouldn't be passing nil errors to newError() initializers)

* remove mutex unlock on transport init fail (it hasn't yet been locked!)

* woops add back the error wrapping to use ErrNotRetrievable

* caches were never being started... :see_no_evil:

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

1 files changed, 68 insertions, 255 deletions

diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go
index eb795d169..7898aec87 100644
--- a/internal/federation/dereferencing/account_test.go
+++ b/internal/federation/dereferencing/account_test.go
@@ -27,7 +27,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -39,17 +38,19 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
-	group, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername: fetchingAccount.Username,
-		RemoteAccountID:    groupURL,
-	})
+	group, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		groupURL,
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(group)
 
 	// group values should be set
 	suite.Equal("https://unknown-instance.com/groups/some_group", group.URI)
 	suite.Equal("https://unknown-instance.com/@some_group", group.URL)
-	suite.WithinDuration(time.Now(), group.LastWebfingeredAt, 5*time.Second)
+	suite.WithinDuration(time.Now(), group.FetchedAt, 5*time.Second)
 
 	// group should be in the database
 	dbGroup, err := suite.db.GetAccountByURI(context.Background(), group.URI)
@@ -62,17 +63,19 @@ func (suite *AccountTestSuite) TestDereferenceService() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
-	service, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername: fetchingAccount.Username,
-		RemoteAccountID:    serviceURL,
-	})
+	service, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		serviceURL,
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(service)
 
 	// service values should be set
 	suite.Equal("https://owncast.example.org/federation/user/rgh", service.URI)
 	suite.Equal("https://owncast.example.org/federation/user/rgh", service.URL)
-	suite.WithinDuration(time.Now(), service.LastWebfingeredAt, 5*time.Second)
+	suite.WithinDuration(time.Now(), service.FetchedAt, 5*time.Second)
 
 	// service should be in the database
 	dbService, err := suite.db.GetAccountByURI(context.Background(), service.URI)
@@ -93,10 +96,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername: fetchingAccount.Username,
-		RemoteAccountID:    testrig.URLMustParse(targetAccount.URI),
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -111,10 +116,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb
 		suite.FailNow(err.Error())
 	}
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername: fetchingAccount.Username,
-		RemoteAccountID:    testrig.URLMustParse(targetAccount.URI),
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -124,10 +131,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountUsername: targetAccount.Username,
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -137,11 +146,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountUsername: targetAccount.Username,
-		RemoteAccountHost:     config.GetHost(),
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		testrig.URLMustParse(targetAccount.URI),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -151,12 +161,13 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
 	fetchingAccount := suite.testAccounts["local_account_1"]
 	targetAccount := suite.testAccounts["local_account_2"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(targetAccount.URI),
-		RemoteAccountUsername: targetAccount.Username,
-		RemoteAccountHost:     config.GetHost(),
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
+		context.Background(),
+		fetchingAccount.Username,
+		targetAccount.Username,
+		config.GetHost(),
+		false,
+	)
 	suite.NoError(err)
 	suite.NotNil(fetchedAccount)
 	suite.Empty(fetchedAccount.Domain)
@@ -165,248 +176,50 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountUsername: "thisaccountdoesnotexist",
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
+		context.Background(),
+		fetchingAccount.Username,
+		"thisaccountdoesnotexist",
+		config.GetHost(),
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountUsername: "thisaccountdoesnotexist",
-		RemoteAccountHost:     "localhost:8080",
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
+		context.Background(),
+		fetchingAccount.Username,
+		"thisaccountdoesnotexist",
+		"localhost:8080",
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
 func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername: fetchingAccount.Username,
-		RemoteAccountID:    testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
-	})
+	fetchedAccount, err := suite.dereferencer.GetAccountByURI(
+		context.Background(),
+		fetchingAccount.Username,
+		testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
+		false,
+	)
 	var errNotRetrievable *dereferencing.ErrNotRetrievable
 	suite.ErrorAs(err, &errNotRetrievable)
-	suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
+	suite.EqualError(err, "item could not be retrieved: no entries")
 	suite.Nil(fetchedAccount)
 }
 
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// dereference an emoji we don't have stored yet
-			{
-				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
-				Shortcode:       "kip_van_den_bos",
-				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
-				Disabled:        testrig.FalseBool(),
-				VisibleInPicker: testrig.FalseBool(),
-				Domain:          "fossbros-anonymous.io",
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-}
-
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	knownEmoji := suite.testEmojis["yell"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// an emoji we already have
-			{
-				URI:             knownEmoji.URI,
-				Shortcode:       knownEmoji.Shortcode,
-				UpdatedAt:       knownEmoji.UpdatedAt,
-				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,
-				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
-				Disabled:        knownEmoji.Disabled,
-				VisibleInPicker: knownEmoji.VisibleInPicker,
-				Domain:          knownEmoji.Domain,
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-}
-
-func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {
-	fetchingAccount := suite.testAccounts["local_account_1"]
-
-	knownEmoji := suite.testEmojis["yell"]
-
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountPartial := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// an emoji we already have
-			{
-				URI:             knownEmoji.URI,
-				Shortcode:       knownEmoji.Shortcode,
-				UpdatedAt:       knownEmoji.UpdatedAt,
-				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,
-				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
-				Disabled:        knownEmoji.Disabled,
-				VisibleInPicker: knownEmoji.VisibleInPicker,
-				Domain:          knownEmoji.Domain,
-			},
-		},
-	}
-
-	fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount)
-	suite.NotNil(fetchedAccount.EmojiIDs)
-	suite.NotNil(fetchedAccount.Emojis)
-	suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI)
-
-	remoteAccountPartial2 := &gtsmodel.Account{
-		ID:                    remoteAccount.ID,
-		ActorType:             remoteAccount.ActorType,
-		Language:              remoteAccount.Language,
-		CreatedAt:             remoteAccount.CreatedAt,
-		UpdatedAt:             remoteAccount.UpdatedAt,
-		Username:              remoteAccount.Username,
-		Domain:                remoteAccount.Domain,
-		DisplayName:           remoteAccount.DisplayName,
-		URI:                   remoteAccount.URI,
-		InboxURI:              remoteAccount.URI,
-		SharedInboxURI:        remoteAccount.SharedInboxURI,
-		PublicKeyURI:          remoteAccount.PublicKeyURI,
-		URL:                   remoteAccount.URL,
-		FollowingURI:          remoteAccount.FollowingURI,
-		FollowersURI:          remoteAccount.FollowersURI,
-		OutboxURI:             remoteAccount.OutboxURI,
-		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
-		Emojis: []*gtsmodel.Emoji{
-			// dereference an emoji we don't have stored yet
-			{
-				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
-				Shortcode:       "kip_van_den_bos",
-				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
-				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
-				Disabled:        testrig.FalseBool(),
-				VisibleInPicker: testrig.FalseBool(),
-				Domain:          "fossbros-anonymous.io",
-			},
-		},
-	}
-
-	fetchedAccount2, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
-		RequestingUsername:    fetchingAccount.Username,
-		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
-		RemoteAccountHost:     remoteAccount.Domain,
-		RemoteAccountUsername: remoteAccount.Username,
-		PartialAccount:        remoteAccountPartial2,
-		Blocking:              true,
-	})
-	suite.NoError(err)
-	suite.NotNil(fetchedAccount2)
-	suite.NotNil(fetchedAccount2.EmojiIDs)
-	suite.NotNil(fetchedAccount2.Emojis)
-	suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI)
-}
-
 func TestAccountTestSuite(t *testing.T) {
 	suite.Run(t, new(AccountTestSuite))
 }