[performance] refactoring + add fave / follow / request / visibility caching (#1607)

* refactor visibility checking, add caching for visibility

* invalidate visibility cache items on account / status deletes

* fix requester ID passed to visibility cache nil ptr

* de-interface caches, fix home / public timeline caching + visibility

* finish adding code comments for visibility filter

* fix angry goconst linter warnings

* actually finish adding filter visibility code comments for timeline functions

* move home timeline status author check to after visibility

* remove now-unused code

* add more code comments

* add TODO code comment, update printed cache start names

* update printed cache names on stop

* start adding separate follow(request) delete db functions, add specific visibility cache tests

* add relationship type caching

* fix getting local account follows / followed-bys, other small codebase improvements

* simplify invalidation using cache hooks, add more GetAccountBy___() functions

* fix boosting to return 404 if not boostable but no error (to not leak status ID)

* remove dead code

* improved placement of cache invalidation

* update license headers

* add example follow, follow-request config entries

* add example visibility cache configuration to config file

* use specific PutFollowRequest() instead of just Put()

* add tests for all GetAccountBy()

* add GetBlockBy() tests

* update block to check primitive fields

* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests

* fix copy-pasted code

* update envparsing test

* whitespace

* fix bun struct tag

* add license header to gtscontext

* fix old license header

* improved error creation to not use fmt.Errorf() when not needed

* fix various rebase conflicts, fix account test

* remove commented-out code, fix-up mention caching

* fix mention select bun statement

* ensure mention target account populated, pass in context to customrenderer logging

* remove more uncommented code, fix typeutil test

* add statusfave database model caching

* add status fave cache configuration

* add status fave cache example config

* woops, catch missed error. nice catch linter!

* add back testrig panic on nil db

* update example configuration to match defaults, slight tweak to cache configuration defaults

* update envparsing test with new defaults

* fetch followingget to use the follow target account

* use accounnt.IsLocal() instead of empty domain check

* use constants for the cache visibility type check

* use bun.In() for notification type restriction in db query

* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())

* use bun query building for nested select statements to ensure working with postgres

* update public timeline future status checks to match visibility filter

* same as previous, for home timeline

* update public timeline tests to dynamically check for appropriate statuses

* migrate accounts to allow unique constraint on public_key

* provide minimal account with publicKey

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>

1 files changed, 34 insertions, 39 deletions

diff --git a/internal/processing/fromcommon.go b/internal/processing/fromcommon.go
index 49a05da5d..93d61c533 100644
--- a/internal/processing/fromcommon.go
+++ b/internal/processing/fromcommon.go
@@ -39,11 +39,12 @@ func (p *Processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e
 
 	if status.Mentions == nil {
 		// there are mentions but they're not fully populated on the status yet so do this
-		menchies, err := p.state.DB.GetMentions(ctx, status.MentionIDs)
+		mentions, err := p.state.DB.GetMentions(ctx, status.MentionIDs)
 		if err != nil {
 			return fmt.Errorf("notifyStatus: error getting mentions for status %s from the db: %s", status.ID, err)
 		}
-		status.Mentions = menchies
+
+		status.Mentions = mentions
 	}
 
 	// now we have mentions as full gtsmodel.Mention structs on the status we can continue
@@ -88,7 +89,7 @@ func (p *Processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e
 			Status:           status,
 		}
 
-		if err := p.state.DB.Put(ctx, notif); err != nil {
+		if err := p.state.DB.PutNotification(ctx, notif); err != nil {
 			return fmt.Errorf("notifyStatus: error putting notification in database: %s", err)
 		}
 
@@ -130,7 +131,7 @@ func (p *Processor) notifyFollowRequest(ctx context.Context, followRequest *gtsm
 		OriginAccountID:  followRequest.AccountID,
 	}
 
-	if err := p.state.DB.Put(ctx, notif); err != nil {
+	if err := p.state.DB.PutNotification(ctx, notif); err != nil {
 		return fmt.Errorf("notifyFollowRequest: error putting notification in database: %s", err)
 	}
 
@@ -171,7 +172,7 @@ func (p *Processor) notifyFollow(ctx context.Context, follow *gtsmodel.Follow, t
 		OriginAccountID:  follow.AccountID,
 		OriginAccount:    follow.Account,
 	}
-	if err := p.state.DB.Put(ctx, notif); err != nil {
+	if err := p.state.DB.PutNotification(ctx, notif); err != nil {
 		return fmt.Errorf("notifyFollow: error putting notification in database: %s", err)
 	}
 
@@ -219,7 +220,7 @@ func (p *Processor) notifyFave(ctx context.Context, fave *gtsmodel.StatusFave) e
 		Status:           fave.Status,
 	}
 
-	if err := p.state.DB.Put(ctx, notif); err != nil {
+	if err := p.state.DB.PutNotification(ctx, notif); err != nil {
 		return fmt.Errorf("notifyFave: error putting notification in database: %s", err)
 	}
 
@@ -293,7 +294,7 @@ func (p *Processor) notifyAnnounce(ctx context.Context, status *gtsmodel.Status)
 		Status:           status,
 	}
 
-	if err := p.state.DB.Put(ctx, notif); err != nil {
+	if err := p.state.DB.PutNotification(ctx, notif); err != nil {
 		return fmt.Errorf("notifyAnnounce: error putting notification in database: %s", err)
 	}
 
@@ -403,39 +404,39 @@ func (p *Processor) notifyReportClosed(ctx context.Context, report *gtsmodel.Rep
 // timelineStatus processes the given new status and inserts it into
 // the HOME timelines of accounts that follow the status author.
 func (p *Processor) timelineStatus(ctx context.Context, status *gtsmodel.Status) error {
-	// make sure the author account is pinned onto the status
 	if status.Account == nil {
-		a, err := p.state.DB.GetAccountByID(ctx, status.AccountID)
-		if err != nil {
-			return fmt.Errorf("timelineStatus: error getting author account with id %s: %s", status.AccountID, err)
+		// ensure status fully populated (including account)
+		if err := p.state.DB.PopulateStatus(ctx, status); err != nil {
+			return fmt.Errorf("timelineStatus: error populating status with id %s: %w", status.ID, err)
 		}
-		status.Account = a
 	}
 
-	// Get LOCAL followers of the account that posted the status;
-	// we know that remote accounts don't have timelines on this
-	// instance, so there's no point selecting them too.
-	accountIDs, err := p.state.DB.GetLocalFollowersIDs(ctx, status.AccountID)
+	// get local followers of the account that posted the status
+	follows, err := p.state.DB.GetAccountLocalFollowers(ctx, status.AccountID)
 	if err != nil {
-		return fmt.Errorf("timelineStatus: error getting followers for account id %s: %s", status.AccountID, err)
+		return fmt.Errorf("timelineStatus: error getting followers for account id %s: %w", status.AccountID, err)
 	}
 
 	// If the poster is also local, add a fake entry for them
 	// so they can see their own status in their timeline.
 	if status.Account.IsLocal() {
-		accountIDs = append(accountIDs, status.AccountID)
+		follows = append(follows, &gtsmodel.Follow{
+			AccountID: status.AccountID,
+			Account:   status.Account,
+		})
 	}
 
-	// Timeline the status for each local following account.
-	errors := gtserror.MultiError{}
-	for _, accountID := range accountIDs {
-		if err := p.timelineStatusForAccount(ctx, status, accountID); err != nil {
-			errors.Append(err)
+	var errs gtserror.MultiError
+
+	for _, follow := range follows {
+		// Timeline the status for each local following account.
+		if err := p.timelineStatusForAccount(ctx, follow.Account, status); err != nil {
+			errs.Append(err)
 		}
 	}
 
-	if len(errors) != 0 {
-		return fmt.Errorf("timelineStatus: one or more errors timelining statuses: %w", errors.Combine())
+	if len(errs) != 0 {
+		return fmt.Errorf("timelineStatus: one or more errors timelining statuses: %w", errs.Combine())
 	}
 
 	return nil
@@ -446,34 +447,28 @@ func (p *Processor) timelineStatus(ctx context.Context, status *gtsmodel.Status)
 //
 // If the status was inserted into the home timeline of the given account,
 // it will also be streamed via websockets to the user.
-func (p *Processor) timelineStatusForAccount(ctx context.Context, status *gtsmodel.Status, accountID string) error {
-	// get the timeline owner account
-	timelineAccount, err := p.state.DB.GetAccountByID(ctx, accountID)
-	if err != nil {
-		return fmt.Errorf("timelineStatusForAccount: error getting account for timeline with id %s: %w", accountID, err)
-	}
-
+func (p *Processor) timelineStatusForAccount(ctx context.Context, account *gtsmodel.Account, status *gtsmodel.Status) error {
 	// make sure the status is timelineable
-	if timelineable, err := p.filter.StatusHometimelineable(ctx, status, timelineAccount); err != nil {
-		return fmt.Errorf("timelineStatusForAccount: error getting timelineability for status for timeline with id %s: %w", accountID, err)
+	if timelineable, err := p.filter.StatusHomeTimelineable(ctx, account, status); err != nil {
+		return fmt.Errorf("timelineStatusForAccount: error getting timelineability for status for timeline with id %s: %w", account.ID, err)
 	} else if !timelineable {
 		return nil
 	}
 
 	// stick the status in the timeline for the account and then immediately prepare it so they can see it right away
-	if inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, timelineAccount.ID); err != nil {
+	if inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, account.ID); err != nil {
 		return fmt.Errorf("timelineStatusForAccount: error ingesting status %s: %w", status.ID, err)
 	} else if !inserted {
 		return nil
 	}
 
 	// the status was inserted so stream it to the user
-	apiStatus, err := p.tc.StatusToAPIStatus(ctx, status, timelineAccount)
+	apiStatus, err := p.tc.StatusToAPIStatus(ctx, status, account)
 	if err != nil {
 		return fmt.Errorf("timelineStatusForAccount: error converting status %s to frontend representation: %w", status.ID, err)
 	}
 
-	if err := p.stream.Update(apiStatus, timelineAccount, stream.TimelineHome); err != nil {
+	if err := p.stream.Update(apiStatus, account, stream.TimelineHome); err != nil {
 		return fmt.Errorf("timelineStatusForAccount: error streaming update for status %s: %w", status.ID, err)
 	}
 
@@ -513,8 +508,8 @@ func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Sta
 	}
 
 	// delete all mention entries generated by this status
-	for _, m := range statusToDelete.MentionIDs {
-		if err := p.state.DB.DeleteByID(ctx, m, &gtsmodel.Mention{}); err != nil {
+	for _, id := range statusToDelete.MentionIDs {
+		if err := p.state.DB.DeleteMentionByID(ctx, id); err != nil {
 			return err
 		}
 	}