[performance] update remaining worker pools to use queues (#2865)

* start replacing client + federator + media workers with new worker + queue types

* refactor federatingDB.Delete(), drop queued messages when deleting account / status

* move all queue purging to the processor workers

* undo toolchain updates

* code comments, ensure dereferencer worker pool gets started

* update gruf libraries in readme

* start the job scheduler separately to the worker pools

* reshuffle ordering or server.go + remove duplicate worker start / stop

* update go-list version

* fix vendoring

* move queue invalidation to before wipeing / deletion, to ensure queued work not dropped

* add logging to worker processing functions in testrig, don't start workers in unexpected places

* update go-structr to add (+then rely on) QueueCtx{} type

* ensure more worker pools get started properly in tests

* fix remaining broken tests relying on worker queue logic

* fix account test suite queue popping logic, ensure noop workers do not pull from queue

* move back accidentally shuffled account deletion order

* ensure error (non nil!!) gets passed in refactored federatingDB{}.Delete()

* silently drop deletes from accounts not permitted to

* don't warn log on forwarded deletes

* make if else clauses easier to parse

* use getFederatorMsg()

* improved code comment

* improved code comment re: requesting account delete checks

* remove boolean result from worker start / stop since false = already running or already stopped

* remove optional passed-in http.client

* remove worker starting from the admin CLI commands (we don't need to handle side-effects)

* update prune cli to start scheduler but not all of the workers

* fix rebase issues

* remove redundant return statements

* i'm sorry sir linter

1 files changed, 22 insertions, 17 deletions

diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go
index 31c6df7d0..ed0949c9c 100644
--- a/internal/httpclient/client.go
+++ b/internal/httpclient/client.go
@@ -59,9 +59,27 @@ var (
 // configuration values passed to initialized http.Transport{}
 // and http.Client{}, along with httpclient.Client{} specific.
 type Config struct {
-	// MaxOpenConnsPerHost limits the max number of open connections to a host.
+
+	// MaxOpenConnsPerHost limits the max
+	// number of open connections to a host.
 	MaxOpenConnsPerHost int
 
+	// AllowRanges allows outgoing
+	// communications to given IP nets.
+	AllowRanges []netip.Prefix
+
+	// BlockRanges blocks outgoing
+	// communiciations to given IP nets.
+	BlockRanges []netip.Prefix
+
+	// TLSInsecureSkipVerify can be set to true to
+	// skip validation of remote TLS certificates.
+	//
+	// THIS SHOULD BE USED FOR TESTING ONLY, IF YOU
+	// TURN THIS ON WHILE RUNNING IN PRODUCTION YOU
+	// ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS!
+	TLSInsecureSkipVerify bool
+
 	// MaxIdleConns: see http.Transport{}.MaxIdleConns.
 	MaxIdleConns int
 
@@ -79,20 +97,6 @@ type Config struct {
 
 	// DisableCompression: see http.Transport{}.DisableCompression.
 	DisableCompression bool
-
-	// AllowRanges allows outgoing communications to given IP nets.
-	AllowRanges []netip.Prefix
-
-	// BlockRanges blocks outgoing communiciations to given IP nets.
-	BlockRanges []netip.Prefix
-
-	// TLSInsecureSkipVerify can be set to true to
-	// skip validation of remote TLS certificates.
-	//
-	// THIS SHOULD BE USED FOR TESTING ONLY, IF YOU
-	// TURN THIS ON WHILE RUNNING IN PRODUCTION YOU
-	// ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS!
-	TLSInsecureSkipVerify bool
 }
 
 // Client wraps an underlying http.Client{} to provide the following:
@@ -138,7 +142,8 @@ func New(cfg Config) *Client {
 		cfg.MaxBodySize = int64(40 * bytesize.MiB)
 	}
 
-	// Protect dialer with IP range sanitizer.
+	// Protect the dialer
+	// with IP range sanitizer.
 	d.Control = (&Sanitizer{
 		Allow: cfg.AllowRanges,
 		Block: cfg.BlockRanges,
@@ -148,7 +153,7 @@ func New(cfg Config) *Client {
 	c.client.Timeout = cfg.Timeout
 	c.bodyMax = cfg.MaxBodySize
 
-	// Prepare TLS config for transport.
+	// Prepare transport TLS config.
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec
 	}