[performance] processing media and scheduled jobs improvements (#1482)

* replace media workers with just runners.WorkerPool, move to state structure, use go-sched for global task scheduling

* improved code comment

* fix worker tryUntil function, update go-runners/go-sched

* make preprocess functions package public, use these where possible to stop doubled up processing

* remove separate emoji worker pool

* limit calls to time.Now() during media preprocessing

* use Processor{} to manage singular runtime of processing media

* ensure workers get started when media manager is used

* improved error setting in processing media, fix media test

* port changes from processingmedia to processing emoji

* finish code commenting

* finish code commenting and comment-out client API + federator worker pools until concurrency worker pools replaced

* linterrrrrrrrrrrrrrrr

---------

Signed-off-by: kim <grufwub@gmail.com>

5 files changed, 7 insertions, 10 deletions

diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go
index 370e661ed..dc4081915 100644
--- a/internal/processing/account/update.go
+++ b/internal/processing/account/update.go
@@ -194,7 +194,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead
 		Description: description,
 	}
 
-	processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, nil, accountID, ai)
+	processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, nil, accountID, ai)
 	if err != nil {
 		return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err)
 	}
@@ -221,10 +221,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead
 		Header: &isHeader,
 	}
 
-	processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, nil, accountID, ai)
-	if err != nil {
-		return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
-	}
+	processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, nil, accountID, ai)
 	if err != nil {
 		return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
 	}
diff --git a/internal/processing/admin/createemoji.go b/internal/processing/admin/createemoji.go
index 601ced336..b2a7bfc86 100644
--- a/internal/processing/admin/createemoji.go
+++ b/internal/processing/admin/createemoji.go
@@ -70,7 +70,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 		}
 	}
 
-	processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, ai, false)
+	processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, ai, false)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji")
 	}
diff --git a/internal/processing/admin/updateemoji.go b/internal/processing/admin/updateemoji.go
index 370e6e27f..41ccd609c 100644
--- a/internal/processing/admin/updateemoji.go
+++ b/internal/processing/admin/updateemoji.go
@@ -107,7 +107,7 @@ func (p *processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji,
 		}
 	}
 
-	processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, *shortcode, newEmojiID, newEmojiURI, ai, false)
+	processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, *shortcode, newEmojiID, newEmojiURI, ai, false)
 	if err != nil {
 		err = fmt.Errorf("emojiUpdateCopy: error processing emoji %s: %s", emoji.ID, err)
 		return nil, gtserror.NewErrorInternalError(err)
@@ -213,7 +213,7 @@ func (p *processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji
 			}
 		}
 
-		processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, emoji.Shortcode, emoji.ID, emoji.URI, ai, true)
+		processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, emoji.Shortcode, emoji.ID, emoji.URI, ai, true)
 		if err != nil {
 			err = fmt.Errorf("emojiUpdateModify: error processing emoji %s: %s", emoji.ID, err)
 			return nil, gtserror.NewErrorInternalError(err)
diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go
index dbe22fe33..494434acb 100644
--- a/internal/processing/media/create.go
+++ b/internal/processing/media/create.go
@@ -42,7 +42,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form
 	}
 
 	// process the media attachment and load it immediately
-	media, err := p.mediaManager.ProcessMedia(ctx, data, nil, account.ID, &media.AdditionalMediaInfo{
+	media, err := p.mediaManager.PreProcessMedia(ctx, data, nil, account.ID, &media.AdditionalMediaInfo{
 		Description: &form.Description,
 		FocusX:      &focusX,
 		FocusY:      &focusY,
diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go
index 9843d6714..41250b3f5 100644
--- a/internal/processing/media/getfile.go
+++ b/internal/processing/media/getfile.go
@@ -159,7 +159,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
 		}
 
 		// Start recaching this media with the prepared data function.
-		processingMedia, err := p.mediaManager.RecacheMedia(ctx, dataFn, nil, wantedMediaID)
+		processingMedia, err := p.mediaManager.PreProcessMediaRecache(ctx, dataFn, nil, wantedMediaID)
 		if err != nil {
 			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
 		}