diff options
author | 2022-05-15 10:16:43 +0100 | |
---|---|---|
committer | 2022-05-15 11:16:43 +0200 | |
commit | 223025fc27ef636206027b360201877848d426a4 (patch) | |
tree | d2f5f293caabdd82fbb87fed3730eb8f6f2e1c1f /internal/api/s2s | |
parent | [chore] Update LE server to use copy of main http.Server{} to maintain server... (diff) | |
download | gotosocial-223025fc27ef636206027b360201877848d426a4.tar.xz |
[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)
* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests
Signed-off-by: kim <grufwub@gmail.com>
* update code comments, defer mutex unlocks
Signed-off-by: kim <grufwub@gmail.com>
* add count to 'performing request' log message
Signed-off-by: kim <grufwub@gmail.com>
* reduce repeated conversions of same url.URL object
Signed-off-by: kim <grufwub@gmail.com>
* move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue
Signed-off-by: kim <grufwub@gmail.com>
* fix security advisories regarding max outgoing conns, max rsp body size
- implemented by a new httpclient.Client{} that wraps an underlying
client with a queue to limit connections, and limit reader wrapping
a response body with a configured maximum size
- update pub.HttpClient args passed around to be this new httpclient.Client{}
Signed-off-by: kim <grufwub@gmail.com>
* add httpclient tests, move ip validation to separate package + change mechanism
Signed-off-by: kim <grufwub@gmail.com>
* fix merge conflicts
Signed-off-by: kim <grufwub@gmail.com>
* use singular mutex in transport rather than separate signer mus
Signed-off-by: kim <grufwub@gmail.com>
* improved useragent string
Signed-off-by: kim <grufwub@gmail.com>
* add note regarding missing test
Signed-off-by: kim <grufwub@gmail.com>
* remove useragent field from transport (instead store in controller)
Signed-off-by: kim <grufwub@gmail.com>
* shutup linter
Signed-off-by: kim <grufwub@gmail.com>
* reset other signing headers on each loop iteration
Signed-off-by: kim <grufwub@gmail.com>
* respect request ctx during retry-backoff sleep period
Signed-off-by: kim <grufwub@gmail.com>
* use external pkg with docs explaining performance "hack"
Signed-off-by: kim <grufwub@gmail.com>
* use http package constants instead of string method literals
Signed-off-by: kim <grufwub@gmail.com>
* add license file headers
Signed-off-by: kim <grufwub@gmail.com>
* update code comment to match new func names
Signed-off-by: kim <grufwub@gmail.com>
* updates to user-agent string
Signed-off-by: kim <grufwub@gmail.com>
* update signed testrig models to fit with new transport logic (instead uses separate signer now)
Signed-off-by: kim <grufwub@gmail.com>
* fuck you linter
Signed-off-by: kim <grufwub@gmail.com>
Diffstat (limited to 'internal/api/s2s')
-rw-r--r-- | internal/api/s2s/user/inboxpost_test.go | 18 | ||||
-rw-r--r-- | internal/api/s2s/user/outboxget_test.go | 14 | ||||
-rw-r--r-- | internal/api/s2s/user/repliesget_test.go | 14 | ||||
-rw-r--r-- | internal/api/s2s/user/statusget_test.go | 10 | ||||
-rw-r--r-- | internal/api/s2s/user/user_test.go | 6 | ||||
-rw-r--r-- | internal/api/s2s/user/userget_test.go | 10 | ||||
-rw-r--r-- | internal/api/s2s/webfinger/webfinger_test.go | 6 | ||||
-rw-r--r-- | internal/api/s2s/webfinger/webfingerget_test.go | 10 |
8 files changed, 44 insertions, 44 deletions
diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go index 6f2909430..388a9fbbb 100644 --- a/internal/api/s2s/user/inboxpost_test.go +++ b/internal/api/s2s/user/inboxpost_test.go @@ -33,11 +33,11 @@ import ( "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -85,8 +85,8 @@ func (suite *InboxPostTestSuite) TestPostBlock() { suite.NoError(err) body := bytes.NewReader(bodyJson) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -188,8 +188,8 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { suite.NoError(err) body := bytes.NewReader(bodyJson) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -281,8 +281,8 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { suite.NoError(err) body := bytes.NewReader(bodyJson) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -403,8 +403,8 @@ func (suite *InboxPostTestSuite) TestPostDelete() { suite.NoError(err) body := bytes.NewReader(bodyJson) - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/s2s/user/outboxget_test.go b/internal/api/s2s/user/outboxget_test.go index ea9259b0f..79122731f 100644 --- a/internal/api/s2s/user/outboxget_test.go +++ b/internal/api/s2s/user/outboxget_test.go @@ -31,8 +31,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -46,8 +46,8 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox"] targetAccount := suite.testAccounts["local_account_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -104,8 +104,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox_first"] targetAccount := suite.testAccounts["local_account_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -162,8 +162,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox_next"] targetAccount := suite.testAccounts["local_account_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go index 4b8364318..845c07bdb 100644 --- a/internal/api/s2s/user/repliesget_test.go +++ b/internal/api/s2s/user/repliesget_test.go @@ -33,8 +33,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -49,8 +49,8 @@ func (suite *RepliesGetTestSuite) TestGetReplies() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -113,8 +113,8 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -180,8 +180,8 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/s2s/user/statusget_test.go b/internal/api/s2s/user/statusget_test.go index c28e4e567..6696bd7e9 100644 --- a/internal/api/s2s/user/statusget_test.go +++ b/internal/api/s2s/user/statusget_test.go @@ -32,8 +32,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -48,8 +48,8 @@ func (suite *StatusGetTestSuite) TestGetStatus() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -116,8 +116,8 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 1ed960544..e8d305d06 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -32,7 +33,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -78,8 +78,8 @@ func (suite *UserStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go index 5c9e4f0d8..5ac2197ff 100644 --- a/internal/api/s2s/user/userget_test.go +++ b/internal/api/s2s/user/userget_test.go @@ -33,9 +33,9 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -49,8 +49,8 @@ func (suite *UserGetTestSuite) TestGetUser() { signedRequest := derefRequests["foss_satan_dereference_zork"] targetAccount := suite.testAccounts["local_account_1"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) @@ -130,8 +130,8 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() { derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) signedRequest := derefRequests["foss_satan_dereference_zork_public_key"] - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go index 1f597d3f9..0df50c503 100644 --- a/internal/api/s2s/webfinger/webfinger_test.go +++ b/internal/api/s2s/webfinger/webfinger_test.go @@ -28,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -37,7 +38,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -81,8 +81,8 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go index 55de30f34..7871b6a3f 100644 --- a/internal/api/s2s/webfinger/webfingerget_test.go +++ b/internal/api/s2s/webfinger/webfingerget_test.go @@ -31,10 +31,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" - "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -71,8 +71,8 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() { func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) @@ -107,8 +107,8 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - clientWorker := worker.New[messages.FromClientAPI](-1, -1) - fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) |