From 223025fc27ef636206027b360201877848d426a4 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Sun, 15 May 2022 10:16:43 +0100 Subject: [security] transport.Controller{} and transport.Transport{} security and performance improvements (#564) * cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim * update code comments, defer mutex unlocks Signed-off-by: kim * add count to 'performing request' log message Signed-off-by: kim * reduce repeated conversions of same url.URL object Signed-off-by: kim * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim * fix merge conflicts Signed-off-by: kim * use singular mutex in transport rather than separate signer mus Signed-off-by: kim * improved useragent string Signed-off-by: kim * add note regarding missing test Signed-off-by: kim * remove useragent field from transport (instead store in controller) Signed-off-by: kim * shutup linter Signed-off-by: kim * reset other signing headers on each loop iteration Signed-off-by: kim * respect request ctx during retry-backoff sleep period Signed-off-by: kim * use external pkg with docs explaining performance "hack" Signed-off-by: kim * use http package constants instead of string method literals Signed-off-by: kim * add license file headers Signed-off-by: kim * update code comment to match new func names Signed-off-by: kim * updates to user-agent string Signed-off-by: kim * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim * fuck you linter Signed-off-by: kim --- internal/api/client/account/account_test.go | 6 +++--- internal/api/client/admin/admin_test.go | 6 +++--- internal/api/client/fileserver/servefile_test.go | 6 +++--- internal/api/client/followrequest/followrequest_test.go | 6 +++--- internal/api/client/media/mediacreate_test.go | 6 +++--- internal/api/client/media/mediaupdate_test.go | 6 +++--- internal/api/client/status/status_test.go | 6 +++--- internal/api/client/user/user_test.go | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) (limited to 'internal/api/client') diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go index d65b49550..d6bb5a5c0 100644 --- a/internal/api/client/account/account_test.go +++ b/internal/api/client/account/account_test.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/account" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -20,7 +21,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -62,8 +62,8 @@ func (suite *AccountStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go index 578ab167c..11e2f8354 100644 --- a/internal/api/client/admin/admin_test.go +++ b/internal/api/client/admin/admin_test.go @@ -29,6 +29,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -38,7 +39,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -80,8 +80,8 @@ func (suite *AdminStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index 49d813981..d7de2f4f9 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -31,6 +31,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -40,7 +41,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -77,8 +77,8 @@ func (suite *ServeFileTestSuite) SetupSuite() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go index 072025931..14b5656b6 100644 --- a/internal/api/client/followrequest/followrequest_test.go +++ b/internal/api/client/followrequest/followrequest_test.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -37,7 +38,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -77,8 +77,8 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 4d08697ef..e16b9f5eb 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -37,6 +37,7 @@ import ( "github.com/stretchr/testify/suite" mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media" "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -47,7 +48,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -84,8 +84,8 @@ func (suite *MediaCreateTestSuite) SetupSuite() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index b87e6ec8d..a87718438 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -35,6 +35,7 @@ import ( "github.com/stretchr/testify/suite" mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media" "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -45,7 +46,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -82,8 +82,8 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { testrig.InitTestConfig() testrig.InitTestLog() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go index a4a56aa0b..e2e2819b5 100644 --- a/internal/api/client/status/status_test.go +++ b/internal/api/client/status/status_test.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -40,7 +41,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -90,8 +90,8 @@ func (suite *StatusStandardTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index b0fd2b2e9..6e9c46525 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -22,6 +22,7 @@ import ( "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -30,7 +31,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -58,8 +58,8 @@ type UserStandardTestSuite struct { func (suite *UserStandardTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() - fedWorker := worker.New[messages.FromFederator](-1, -1) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() -- cgit v1.3