From 223025fc27ef636206027b360201877848d426a4 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sun, 15 May 2022 10:16:43 +0100 Subject: [security] transport.Controller{} and transport.Transport{} security and performance improvements (#564) * cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim * update code comments, defer mutex unlocks Signed-off-by: kim * add count to 'performing request' log message Signed-off-by: kim * reduce repeated conversions of same url.URL object Signed-off-by: kim * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim * fix merge conflicts Signed-off-by: kim * use singular mutex in transport rather than separate signer mus Signed-off-by: kim * improved useragent string Signed-off-by: kim * add note regarding missing test Signed-off-by: kim * remove useragent field from transport (instead store in controller) Signed-off-by: kim * shutup linter Signed-off-by: kim * reset other signing headers on each loop iteration Signed-off-by: kim * respect request ctx during retry-backoff sleep period Signed-off-by: kim * use external pkg with docs explaining performance "hack" Signed-off-by: kim * use http package constants instead of string method literals Signed-off-by: kim * add license file headers Signed-off-by: kim * update code comment to match new func names Signed-off-by: kim * updates to user-agent string Signed-off-by: kim * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim * fuck you linter Signed-off-by: kim --- internal/media/manager.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'internal/media') diff --git a/internal/media/manager.go b/internal/media/manager.go index 174fca8e2..5b4a01021 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -27,9 +27,9 @@ import ( "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/spf13/viper" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. @@ -79,8 +79,8 @@ type Manager interface { type manager struct { db db.DB storage *kv.KVStore - emojiWorker *worker.Worker[*ProcessingEmoji] - mediaWorker *worker.Worker[*ProcessingMedia] + emojiWorker *concurrency.WorkerPool[*ProcessingEmoji] + mediaWorker *concurrency.WorkerPool[*ProcessingMedia] stopCronJobs func() error } @@ -89,7 +89,7 @@ type manager struct { // A worker pool will also be initialized for the manager, to ensure that only // a limited number of media will be processed in parallel. The numbers of workers // is determined from the $GOMAXPROCS environment variable (usually no. CPU cores). -// See internal/worker.New() documentation for further information. +// See internal/concurrency.NewWorkerPool() documentation for further information. func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { m := &manager{ db: database, @@ -97,7 +97,7 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { } // Prepare the media worker pool - m.mediaWorker = worker.New[*ProcessingMedia](-1, 10) + m.mediaWorker = concurrency.NewWorkerPool[*ProcessingMedia](-1, 10) m.mediaWorker.SetProcessor(func(ctx context.Context, media *ProcessingMedia) error { if err := ctx.Err(); err != nil { return err @@ -109,7 +109,7 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { }) // Prepare the emoji worker pool - m.emojiWorker = worker.New[*ProcessingEmoji](-1, 10) + m.emojiWorker = concurrency.NewWorkerPool[*ProcessingEmoji](-1, 10) m.emojiWorker.SetProcessor(func(ctx context.Context, emoji *ProcessingEmoji) error { if err := ctx.Err(); err != nil { return err -- cgit v1.3