From acc95923da555b2bf17a5638e62e533218c5840a Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 13 Feb 2023 18:40:48 +0000 Subject: [performance] processing media and scheduled jobs improvements (#1482) * replace media workers with just runners.WorkerPool, move to state structure, use go-sched for global task scheduling * improved code comment * fix worker tryUntil function, update go-runners/go-sched * make preprocess functions package public, use these where possible to stop doubled up processing * remove separate emoji worker pool * limit calls to time.Now() during media preprocessing * use Processor{} to manage singular runtime of processing media * ensure workers get started when media manager is used * improved error setting in processing media, fix media test * port changes from processingmedia to processing emoji * finish code commenting * finish code commenting and comment-out client API + federator worker pools until concurrency worker pools replaced * linterrrrrrrrrrrrrrrr --------- Signed-off-by: kim --- internal/media/refetch.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'internal/media/refetch.go') diff --git a/internal/media/refetch.go b/internal/media/refetch.go index e29b4587f..3d572e4b9 100644 --- a/internal/media/refetch.go +++ b/internal/media/refetch.go @@ -47,7 +47,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM // page through emojis 20 at a time, looking for those with missing images for { // Fetch next block of emojis from database - emojis, err := m.db.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) + emojis, err := m.state.DB.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) if err != nil { if !errors.Is(err, db.ErrNoEntries) { // an actual error has occurred @@ -86,7 +86,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM var totalRefetched int for _, emojiID := range refetchIDs { - emoji, err := m.db.GetEmojiByID(ctx, emojiID) + emoji, err := m.state.DB.GetEmojiByID(ctx, emojiID) if err != nil { // this shouldn't happen--since we know we have the emoji--so return if it does return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err) @@ -108,7 +108,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM return dereferenceMedia(ctx, emojiImageIRI) } - processingEmoji, err := m.ProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{ + processingEmoji, err := m.PreProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{ Domain: &emoji.Domain, ImageRemoteURL: &emoji.ImageRemoteURL, ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL, @@ -133,13 +133,13 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM } func (m *manager) emojiRequiresRefetch(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { - if has, err := m.storage.Has(ctx, emoji.ImagePath); err != nil { + if has, err := m.state.Storage.Has(ctx, emoji.ImagePath); err != nil { return false, err } else if !has { return true, nil } - if has, err := m.storage.Has(ctx, emoji.ImageStaticPath); err != nil { + if has, err := m.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil { return false, err } else if !has { return true, nil -- cgit v1.2.3