From 223025fc27ef636206027b360201877848d426a4 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sun, 15 May 2022 10:16:43 +0100 Subject: [security] transport.Controller{} and transport.Transport{} security and performance improvements (#564) * cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim * update code comments, defer mutex unlocks Signed-off-by: kim * add count to 'performing request' log message Signed-off-by: kim * reduce repeated conversions of same url.URL object Signed-off-by: kim * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim * fix merge conflicts Signed-off-by: kim * use singular mutex in transport rather than separate signer mus Signed-off-by: kim * improved useragent string Signed-off-by: kim * add note regarding missing test Signed-off-by: kim * remove useragent field from transport (instead store in controller) Signed-off-by: kim * shutup linter Signed-off-by: kim * reset other signing headers on each loop iteration Signed-off-by: kim * respect request ctx during retry-backoff sleep period Signed-off-by: kim * use external pkg with docs explaining performance "hack" Signed-off-by: kim * use http package constants instead of string method literals Signed-off-by: kim * add license file headers Signed-off-by: kim * update code comment to match new func names Signed-off-by: kim * updates to user-agent string Signed-off-by: kim * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim * fuck you linter Signed-off-by: kim --- cmd/gotosocial/action/server/server.go | 13 ++++++++----- cmd/gotosocial/action/testrig/testrig.go | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'cmd') diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 29c05ba1a..4a8993598 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -21,7 +21,6 @@ package server import ( "context" "fmt" - "net/http" "os" "os/signal" "path" @@ -56,12 +55,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db/bundb" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" + "github.com/superseriousbusiness/gotosocial/internal/httpclient" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" @@ -71,7 +72,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/web" - "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Start creates and starts a gotosocial server @@ -93,8 +93,8 @@ var Start action.GTSAction = func(ctx context.Context) error { // NOTE: these MUST NOT be used until they are passed to the // processor and it is started. The reason being that the processor // sets the Worker process functions and start the underlying pools - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) federatingDB := federatingdb.New(dbService, fedWorker) @@ -120,13 +120,16 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error creating storage backend: %s", err) } + // Build HTTP client (TODO: add configurables here) + client := httpclient.New(httpclient.Config{}) + // build backend handlers mediaManager, err := media.NewManager(dbService, storage) if err != nil { return fmt.Errorf("error creating media manager: %s", err) } oauthServer := oauth.New(ctx, dbService) - transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, http.DefaultClient) + transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, client) federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) // decide whether to create a noop email sender (won't send emails) or a real one diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 010c730a0..cb587c51d 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -54,11 +54,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oidc" "github.com/superseriousbusiness/gotosocial/internal/web" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -74,8 +74,8 @@ var Start action.GTSAction = func(ctx context.Context) error { testrig.StandardStorageSetup(storageBackend, "./testrig/media") // Create client API and federator worker pools - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) // build backend handlers oauthServer := testrig.NewTestOauthServer(dbService) -- cgit v1.3