From acc95923da555b2bf17a5638e62e533218c5840a Mon Sep 17 00:00:00 2001
From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
Date: Mon, 13 Feb 2023 18:40:48 +0000
Subject: [performance] processing media and scheduled jobs improvements
 (#1482)

* replace media workers with just runners.WorkerPool, move to state structure, use go-sched for global task scheduling

* improved code comment

* fix worker tryUntil function, update go-runners/go-sched

* make preprocess functions package public, use these where possible to stop doubled up processing

* remove separate emoji worker pool

* limit calls to time.Now() during media preprocessing

* use Processor{} to manage singular runtime of processing media

* ensure workers get started when media manager is used

* improved error setting in processing media, fix media test

* port changes from processingmedia to processing emoji

* finish code commenting

* finish code commenting and comment-out client API + federator worker pools until concurrency worker pools replaced

* linterrrrrrrrrrrrrrrr

---------

Signed-off-by: kim <grufwub@gmail.com>
---
 internal/federation/dereferencing/account.go | 4 ++--
 internal/federation/dereferencing/emoji.go   | 4 +---
 internal/federation/federatingdb/db.go       | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

(limited to 'internal/federation')

diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
index 82b69c7a1..143cafc9b 100644
--- a/internal/federation/dereferencing/account.go
+++ b/internal/federation/dereferencing/account.go
@@ -356,7 +356,7 @@ func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.T
 	}
 
 	// Create new media processing request from the media manager instance.
-	processing, err := d.mediaManager.ProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{
+	processing, err := d.mediaManager.PreProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{
 		Avatar:    func() *bool { v := false; return &v }(),
 		RemoteURL: &avatarURL,
 	})
@@ -407,7 +407,7 @@ func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.T
 	}
 
 	// Create new media processing request from the media manager instance.
-	processing, err := d.mediaManager.ProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{
+	processing, err := d.mediaManager.PreProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{
 		Header:    func() *bool { v := true; return &v }(),
 		RemoteURL: &headerURL,
 	})
diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go
index 3e9452f14..7d8526c73 100644
--- a/internal/federation/dereferencing/emoji.go
+++ b/internal/federation/dereferencing/emoji.go
@@ -61,7 +61,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r
 			return t.DereferenceMedia(innerCtx, derefURI)
 		}
 
-		newProcessing, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh)
+		newProcessing, err := d.mediaManager.PreProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh)
 		if err != nil {
 			return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji %s: %s", shortcodeDomain, err)
 		}
@@ -146,7 +146,6 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,
 					Disabled:             gotEmoji.Disabled,
 					VisibleInPicker:      gotEmoji.VisibleInPicker,
 				}, refresh)
-
 				if err != nil {
 					log.Errorf("populateEmojis: couldn't refresh remote emoji %s: %s", shortcodeDomain, err)
 					continue
@@ -172,7 +171,6 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,
 				Disabled:             e.Disabled,
 				VisibleInPicker:      e.VisibleInPicker,
 			}, refresh)
-
 			if err != nil {
 				log.Errorf("populateEmojis: couldn't get remote emoji %s: %s", shortcodeDomain, err)
 				continue
diff --git a/internal/federation/federatingdb/db.go b/internal/federation/federatingdb/db.go
index 0787cc7cf..24455a553 100644
--- a/internal/federation/federatingdb/db.go
+++ b/internal/federation/federatingdb/db.go
@@ -49,12 +49,12 @@ type federatingDB struct {
 }
 
 // New returns a DB interface using the given database and config
-func New(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) DB {
+func New(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator], tc typeutils.TypeConverter) DB {
 	fdb := federatingDB{
 		locks:         mutexes.NewMap(-1, -1), // use defaults
 		db:            db,
 		fedWorker:     fedWorker,
-		typeConverter: typeutils.NewConverter(db),
+		typeConverter: tc,
 	}
 	return &fdb
 }
-- 
cgit v1.2.3