From de6e3e5f2a8ea639d76e310a11cb9bc093fef3a9 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Tue, 28 Mar 2023 14:03:14 +0100 Subject: [performance] refactoring + add fave / follow / request / visibility caching (#1607) * refactor visibility checking, add caching for visibility * invalidate visibility cache items on account / status deletes * fix requester ID passed to visibility cache nil ptr * de-interface caches, fix home / public timeline caching + visibility * finish adding code comments for visibility filter * fix angry goconst linter warnings * actually finish adding filter visibility code comments for timeline functions * move home timeline status author check to after visibility * remove now-unused code * add more code comments * add TODO code comment, update printed cache start names * update printed cache names on stop * start adding separate follow(request) delete db functions, add specific visibility cache tests * add relationship type caching * fix getting local account follows / followed-bys, other small codebase improvements * simplify invalidation using cache hooks, add more GetAccountBy___() functions * fix boosting to return 404 if not boostable but no error (to not leak status ID) * remove dead code * improved placement of cache invalidation * update license headers * add example follow, follow-request config entries * add example visibility cache configuration to config file * use specific PutFollowRequest() instead of just Put() * add tests for all GetAccountBy() * add GetBlockBy() tests * update block to check primitive fields * update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests * fix copy-pasted code * update envparsing test * whitespace * fix bun struct tag * add license header to gtscontext * fix old license header * improved error creation to not use fmt.Errorf() when not needed * fix various rebase conflicts, fix account test * remove commented-out code, fix-up mention caching * fix mention select bun statement * ensure mention target account populated, pass in context to customrenderer logging * remove more uncommented code, fix typeutil test * add statusfave database model caching * add status fave cache configuration * add status fave cache example config * woops, catch missed error. nice catch linter! * add back testrig panic on nil db * update example configuration to match defaults, slight tweak to cache configuration defaults * update envparsing test with new defaults * fetch followingget to use the follow target account * use accounnt.IsLocal() instead of empty domain check * use constants for the cache visibility type check * use bun.In() for notification type restriction in db query * include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable()) * use bun query building for nested select statements to ensure working with postgres * update public timeline future status checks to match visibility filter * same as previous, for home timeline * update public timeline tests to dynamically check for appropriate statuses * migrate accounts to allow unique constraint on public_key * provide minimal account with publicKey --------- Signed-off-by: kim Co-authored-by: tsmethurst --- internal/processing/fromcommon.go | 73 ++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 39 deletions(-) (limited to 'internal/processing/fromcommon.go') diff --git a/internal/processing/fromcommon.go b/internal/processing/fromcommon.go index 49a05da5d..93d61c533 100644 --- a/internal/processing/fromcommon.go +++ b/internal/processing/fromcommon.go @@ -39,11 +39,12 @@ func (p *Processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e if status.Mentions == nil { // there are mentions but they're not fully populated on the status yet so do this - menchies, err := p.state.DB.GetMentions(ctx, status.MentionIDs) + mentions, err := p.state.DB.GetMentions(ctx, status.MentionIDs) if err != nil { return fmt.Errorf("notifyStatus: error getting mentions for status %s from the db: %s", status.ID, err) } - status.Mentions = menchies + + status.Mentions = mentions } // now we have mentions as full gtsmodel.Mention structs on the status we can continue @@ -88,7 +89,7 @@ func (p *Processor) notifyStatus(ctx context.Context, status *gtsmodel.Status) e Status: status, } - if err := p.state.DB.Put(ctx, notif); err != nil { + if err := p.state.DB.PutNotification(ctx, notif); err != nil { return fmt.Errorf("notifyStatus: error putting notification in database: %s", err) } @@ -130,7 +131,7 @@ func (p *Processor) notifyFollowRequest(ctx context.Context, followRequest *gtsm OriginAccountID: followRequest.AccountID, } - if err := p.state.DB.Put(ctx, notif); err != nil { + if err := p.state.DB.PutNotification(ctx, notif); err != nil { return fmt.Errorf("notifyFollowRequest: error putting notification in database: %s", err) } @@ -171,7 +172,7 @@ func (p *Processor) notifyFollow(ctx context.Context, follow *gtsmodel.Follow, t OriginAccountID: follow.AccountID, OriginAccount: follow.Account, } - if err := p.state.DB.Put(ctx, notif); err != nil { + if err := p.state.DB.PutNotification(ctx, notif); err != nil { return fmt.Errorf("notifyFollow: error putting notification in database: %s", err) } @@ -219,7 +220,7 @@ func (p *Processor) notifyFave(ctx context.Context, fave *gtsmodel.StatusFave) e Status: fave.Status, } - if err := p.state.DB.Put(ctx, notif); err != nil { + if err := p.state.DB.PutNotification(ctx, notif); err != nil { return fmt.Errorf("notifyFave: error putting notification in database: %s", err) } @@ -293,7 +294,7 @@ func (p *Processor) notifyAnnounce(ctx context.Context, status *gtsmodel.Status) Status: status, } - if err := p.state.DB.Put(ctx, notif); err != nil { + if err := p.state.DB.PutNotification(ctx, notif); err != nil { return fmt.Errorf("notifyAnnounce: error putting notification in database: %s", err) } @@ -403,39 +404,39 @@ func (p *Processor) notifyReportClosed(ctx context.Context, report *gtsmodel.Rep // timelineStatus processes the given new status and inserts it into // the HOME timelines of accounts that follow the status author. func (p *Processor) timelineStatus(ctx context.Context, status *gtsmodel.Status) error { - // make sure the author account is pinned onto the status if status.Account == nil { - a, err := p.state.DB.GetAccountByID(ctx, status.AccountID) - if err != nil { - return fmt.Errorf("timelineStatus: error getting author account with id %s: %s", status.AccountID, err) + // ensure status fully populated (including account) + if err := p.state.DB.PopulateStatus(ctx, status); err != nil { + return fmt.Errorf("timelineStatus: error populating status with id %s: %w", status.ID, err) } - status.Account = a } - // Get LOCAL followers of the account that posted the status; - // we know that remote accounts don't have timelines on this - // instance, so there's no point selecting them too. - accountIDs, err := p.state.DB.GetLocalFollowersIDs(ctx, status.AccountID) + // get local followers of the account that posted the status + follows, err := p.state.DB.GetAccountLocalFollowers(ctx, status.AccountID) if err != nil { - return fmt.Errorf("timelineStatus: error getting followers for account id %s: %s", status.AccountID, err) + return fmt.Errorf("timelineStatus: error getting followers for account id %s: %w", status.AccountID, err) } // If the poster is also local, add a fake entry for them // so they can see their own status in their timeline. if status.Account.IsLocal() { - accountIDs = append(accountIDs, status.AccountID) + follows = append(follows, >smodel.Follow{ + AccountID: status.AccountID, + Account: status.Account, + }) } - // Timeline the status for each local following account. - errors := gtserror.MultiError{} - for _, accountID := range accountIDs { - if err := p.timelineStatusForAccount(ctx, status, accountID); err != nil { - errors.Append(err) + var errs gtserror.MultiError + + for _, follow := range follows { + // Timeline the status for each local following account. + if err := p.timelineStatusForAccount(ctx, follow.Account, status); err != nil { + errs.Append(err) } } - if len(errors) != 0 { - return fmt.Errorf("timelineStatus: one or more errors timelining statuses: %w", errors.Combine()) + if len(errs) != 0 { + return fmt.Errorf("timelineStatus: one or more errors timelining statuses: %w", errs.Combine()) } return nil @@ -446,34 +447,28 @@ func (p *Processor) timelineStatus(ctx context.Context, status *gtsmodel.Status) // // If the status was inserted into the home timeline of the given account, // it will also be streamed via websockets to the user. -func (p *Processor) timelineStatusForAccount(ctx context.Context, status *gtsmodel.Status, accountID string) error { - // get the timeline owner account - timelineAccount, err := p.state.DB.GetAccountByID(ctx, accountID) - if err != nil { - return fmt.Errorf("timelineStatusForAccount: error getting account for timeline with id %s: %w", accountID, err) - } - +func (p *Processor) timelineStatusForAccount(ctx context.Context, account *gtsmodel.Account, status *gtsmodel.Status) error { // make sure the status is timelineable - if timelineable, err := p.filter.StatusHometimelineable(ctx, status, timelineAccount); err != nil { - return fmt.Errorf("timelineStatusForAccount: error getting timelineability for status for timeline with id %s: %w", accountID, err) + if timelineable, err := p.filter.StatusHomeTimelineable(ctx, account, status); err != nil { + return fmt.Errorf("timelineStatusForAccount: error getting timelineability for status for timeline with id %s: %w", account.ID, err) } else if !timelineable { return nil } // stick the status in the timeline for the account and then immediately prepare it so they can see it right away - if inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, timelineAccount.ID); err != nil { + if inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, account.ID); err != nil { return fmt.Errorf("timelineStatusForAccount: error ingesting status %s: %w", status.ID, err) } else if !inserted { return nil } // the status was inserted so stream it to the user - apiStatus, err := p.tc.StatusToAPIStatus(ctx, status, timelineAccount) + apiStatus, err := p.tc.StatusToAPIStatus(ctx, status, account) if err != nil { return fmt.Errorf("timelineStatusForAccount: error converting status %s to frontend representation: %w", status.ID, err) } - if err := p.stream.Update(apiStatus, timelineAccount, stream.TimelineHome); err != nil { + if err := p.stream.Update(apiStatus, account, stream.TimelineHome); err != nil { return fmt.Errorf("timelineStatusForAccount: error streaming update for status %s: %w", status.ID, err) } @@ -513,8 +508,8 @@ func (p *Processor) wipeStatus(ctx context.Context, statusToDelete *gtsmodel.Sta } // delete all mention entries generated by this status - for _, m := range statusToDelete.MentionIDs { - if err := p.state.DB.DeleteByID(ctx, m, >smodel.Mention{}); err != nil { + for _, id := range statusToDelete.MentionIDs { + if err := p.state.DB.DeleteMentionByID(ctx, id); err != nil { return err } } -- cgit v1.2.3