[feature/performance] Store account stats in separate table (#2831)

* [feature/performance] Store account stats in separate table, get stats from remote

* test account stats

* add some missing increment / decrement calls

* change stats function signatures

* rejig logging a bit

* use lock when updating stats

1 files changed, 113 insertions, 1 deletions

diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
index 305b3f05c..e8d32f58a 100644
--- a/internal/federation/dereferencing/account.go
+++ b/internal/federation/dereferencing/account.go
@@ -695,7 +695,7 @@ func (d *Dereferencer) enrichAccount(
 		representation of the target account, derived from
 		a combination of webfinger lookups and dereferencing.
 		Further fetching beyond this point is for peripheral
-		things like account avatar, header, emojis.
+		things like account avatar, header, emojis, stats.
 	*/
 
 	// Ensure internal db ID is
@@ -718,6 +718,11 @@ func (d *Dereferencer) enrichAccount(
 		log.Errorf(ctx, "error fetching remote emojis for account %s: %v", uri, err)
 	}
 
+	// Fetch followers/following count for this account.
+	if err := d.fetchRemoteAccountStats(ctx, latestAcc, requestUser); err != nil {
+		log.Errorf(ctx, "error fetching remote stats for account %s: %v", uri, err)
+	}
+
 	if account.IsNew() {
 		// Prefer published/created time from
 		// apubAcc, fall back to FetchedAt value.
@@ -1036,6 +1041,113 @@ func (d *Dereferencer) fetchRemoteAccountEmojis(ctx context.Context, targetAccou
 	return changed, nil
 }
 
+func (d *Dereferencer) fetchRemoteAccountStats(ctx context.Context, account *gtsmodel.Account, requestUser string) error {
+	// Ensure we have a stats model for this account.
+	if account.Stats == nil {
+		if err := d.state.DB.PopulateAccountStats(ctx, account); err != nil {
+			return gtserror.Newf("db error getting account stats: %w", err)
+		}
+	}
+
+	// We want to update stats by getting remote
+	// followers/following/statuses counts for
+	// this account.
+	//
+	// If we fail getting any particular stat,
+	// it will just fall back to counting local.
+
+	// Followers first.
+	if count, err := d.countCollection(
+		ctx,
+		account.FollowersURI,
+		requestUser,
+	); err != nil {
+		// Log this but don't bail.
+		log.Warnf(ctx,
+			"couldn't count followers for @%s@%s: %v",
+			account.Username, account.Domain, err,
+		)
+	} else if count > 0 {
+		// Positive integer is useful!
+		account.Stats.FollowersCount = &count
+	}
+
+	// Now following.
+	if count, err := d.countCollection(
+		ctx,
+		account.FollowingURI,
+		requestUser,
+	); err != nil {
+		// Log this but don't bail.
+		log.Warnf(ctx,
+			"couldn't count following for @%s@%s: %v",
+			account.Username, account.Domain, err,
+		)
+	} else if count > 0 {
+		// Positive integer is useful!
+		account.Stats.FollowingCount = &count
+	}
+
+	// Now statuses count.
+	if count, err := d.countCollection(
+		ctx,
+		account.OutboxURI,
+		requestUser,
+	); err != nil {
+		// Log this but don't bail.
+		log.Warnf(ctx,
+			"couldn't count statuses for @%s@%s: %v",
+			account.Username, account.Domain, err,
+		)
+	} else if count > 0 {
+		// Positive integer is useful!
+		account.Stats.StatusesCount = &count
+	}
+
+	// Update stats now.
+	if err := d.state.DB.UpdateAccountStats(
+		ctx,
+		account.Stats,
+		"followers_count",
+		"following_count",
+		"statuses_count",
+	); err != nil {
+		return gtserror.Newf("db error updating account stats: %w", err)
+	}
+
+	return nil
+}
+
+// countCollection parses the given uriStr,
+// dereferences the result as a collection
+// type, and returns total items as 0, or
+// a positive integer, or -1 if total items
+// cannot be counted.
+//
+// Error will be returned for invalid non-empty
+// URIs or dereferencing isses.
+func (d *Dereferencer) countCollection(
+	ctx context.Context,
+	uriStr string,
+	requestUser string,
+) (int, error) {
+	if uriStr == "" {
+		return -1, nil
+	}
+
+	uri, err := url.Parse(uriStr)
+	if err != nil {
+		return -1, err
+	}
+
+	collect, err := d.dereferenceCollection(ctx, requestUser, uri)
+	if err != nil {
+		return -1, err
+	}
+
+	return collect.TotalItems(), nil
+}
+
 // dereferenceAccountFeatured dereferences an account's featuredCollectionURI (if not empty). For each discovered status, this status will
 // be dereferenced (if necessary) and marked as pinned (if necessary). Then, old pins will be removed if they're not included in new pins.
 func (d *Dereferencer) dereferenceAccountFeatured(ctx context.Context, requestUser string, account *gtsmodel.Account) error {