[performance] refactoring + add fave / follow / request / visibility caching (#1607)

* refactor visibility checking, add caching for visibility

* invalidate visibility cache items on account / status deletes

* fix requester ID passed to visibility cache nil ptr

* de-interface caches, fix home / public timeline caching + visibility

* finish adding code comments for visibility filter

* fix angry goconst linter warnings

* actually finish adding filter visibility code comments for timeline functions

* move home timeline status author check to after visibility

* remove now-unused code

* add more code comments

* add TODO code comment, update printed cache start names

* update printed cache names on stop

* start adding separate follow(request) delete db functions, add specific visibility cache tests

* add relationship type caching

* fix getting local account follows / followed-bys, other small codebase improvements

* simplify invalidation using cache hooks, add more GetAccountBy___() functions

* fix boosting to return 404 if not boostable but no error (to not leak status ID)

* remove dead code

* improved placement of cache invalidation

* update license headers

* add example follow, follow-request config entries

* add example visibility cache configuration to config file

* use specific PutFollowRequest() instead of just Put()

* add tests for all GetAccountBy()

* add GetBlockBy() tests

* update block to check primitive fields

* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests

* fix copy-pasted code

* update envparsing test

* whitespace

* fix bun struct tag

* add license header to gtscontext

* fix old license header

* improved error creation to not use fmt.Errorf() when not needed

* fix various rebase conflicts, fix account test

* remove commented-out code, fix-up mention caching

* fix mention select bun statement

* ensure mention target account populated, pass in context to customrenderer logging

* remove more uncommented code, fix typeutil test

* add statusfave database model caching

* add status fave cache configuration

* add status fave cache example config

* woops, catch missed error. nice catch linter!

* add back testrig panic on nil db

* update example configuration to match defaults, slight tweak to cache configuration defaults

* update envparsing test with new defaults

* fetch followingget to use the follow target account

* use accounnt.IsLocal() instead of empty domain check

* use constants for the cache visibility type check

* use bun.In() for notification type restriction in db query

* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())

* use bun query building for nested select statements to ensure working with postgres

* update public timeline future status checks to match visibility filter

* same as previous, for home timeline

* update public timeline tests to dynamically check for appropriate statuses

* migrate accounts to allow unique constraint on public_key

* provide minimal account with publicKey

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

1 files changed, 29 insertions, 7 deletions

diff --git a/internal/gtsmodel/account.go b/internal/gtsmodel/account.go
index 54d8c3d71..42e4f5ea6 100644
--- a/internal/gtsmodel/account.go
+++ b/internal/gtsmodel/account.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
 // Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc).
@@ -35,8 +36,8 @@ type Account struct {
 	CreatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item created.
 	UpdatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item was last updated.
 	FetchedAt               time.Time        `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"`                                                       // when was item (remote) last fetched.
-	Username                string           `validate:"required" bun:",nullzero,notnull,unique:userdomain"`                                                         // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
-	Domain                  string           `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"`                                                           // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
+	Username                string           `validate:"required" bun:",nullzero,notnull,unique:usernamedomain"`                                                     // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
+	Domain                  string           `validate:"omitempty,fqdn" bun:",nullzero,unique:usernamedomain"`                                                       // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
 	AvatarMediaAttachmentID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // Database ID of the media attachment, if present
 	AvatarMediaAttachment   *MediaAttachment `validate:"-" bun:"rel:belongs-to"`                                                                                     // MediaAttachment corresponding to avatarMediaAttachmentID
 	AvatarRemoteURL         string           `validate:"omitempty,url" bun:",nullzero"`                                                                              // For a non-local account, where can the header be fetched?
@@ -70,8 +71,8 @@ type Account struct {
 	FollowersURI            string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // URI for getting the followers list of this account
 	FeaturedCollectionURI   string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // URL for getting the featured collection list of this account
 	ActorType               string           `validate:"oneof=Application Group Organization Person Service" bun:",nullzero,notnull"`                                // What type of activitypub actor is this account?
-	PrivateKey              *rsa.PrivateKey  `validate:"required_without=Domain"`                                                                                    // Privatekey for validating activitypub requests, will only be defined for local accounts
-	PublicKey               *rsa.PublicKey   `validate:"required"`                                                                                                   // Publickey for encoding activitypub requests, will be defined for both local and remote accounts
+	PrivateKey              *rsa.PrivateKey  `validate:"required_without=Domain" bun:""`                                                                             // Privatekey for validating activitypub requests, will only be defined for local accounts
+	PublicKey               *rsa.PublicKey   `validate:"required" bun:",notnull,unique"`                                                                             // Publickey for encoding activitypub requests, will be defined for both local and remote accounts
 	PublicKeyURI            string           `validate:"required,url" bun:",nullzero,notnull,unique"`                                                                // Web-reachable location of this account's public key
 	SensitizedAt            time.Time        `validate:"-" bun:"type:timestamptz,nullzero"`                                                                          // When was this account set to have all its media shown as sensitive?
 	SilencedAt              time.Time        `validate:"-" bun:"type:timestamptz,nullzero"`                                                                          // When was this account silenced (eg., statuses only visible to followers, not public)?
@@ -82,23 +83,44 @@ type Account struct {
 }
 
 // IsLocal returns whether account is a local user account.
-func (a Account) IsLocal() bool {
+func (a *Account) IsLocal() bool {
 	return a.Domain == "" || a.Domain == config.GetHost() || a.Domain == config.GetAccountDomain()
 }
 
 // IsRemote returns whether account is a remote user account.
-func (a Account) IsRemote() bool {
+func (a *Account) IsRemote() bool {
 	return !a.IsLocal()
 }
 
 // IsInstance returns whether account is an instance internal actor account.
-func (a Account) IsInstance() bool {
+func (a *Account) IsInstance() bool {
 	return a.Username == a.Domain ||
 		a.FollowersURI == "" ||
 		a.FollowingURI == "" ||
 		(a.Username == "internal.fetch" && strings.Contains(a.Note, "internal service actor"))
 }
 
+// EmojisPopulated returns whether emojis are populated according to current EmojiIDs.
+func (a *Account) EmojisPopulated() bool {
+	if len(a.EmojiIDs) != len(a.Emojis) {
+		// this is the quickest indicator.
+		return false
+	}
+
+	// Emojis must be in same order.
+	for i, id := range a.EmojiIDs {
+		if a.Emojis[i] == nil {
+			log.Warnf(nil, "nil emoji in slice for account %s", a.URI)
+			continue
+		}
+		if a.Emojis[i].ID != id {
+			return false
+		}
+	}
+
+	return true
+}
+
 // AccountToEmoji is an intermediate struct to facilitate the many2many relationship between an account and one or more emojis.
 type AccountToEmoji struct {
 	AccountID string   `validate:"ulid,required" bun:"type:CHAR(26),unique:accountemoji,nullzero,notnull"`