[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)

* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests

Signed-off-by: kim <grufwub@gmail.com>

* update code comments, defer mutex unlocks

Signed-off-by: kim <grufwub@gmail.com>

* add count to 'performing request' log message

Signed-off-by: kim <grufwub@gmail.com>

* reduce repeated conversions of same url.URL object

Signed-off-by: kim <grufwub@gmail.com>

* move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue

Signed-off-by: kim <grufwub@gmail.com>

* fix security advisories regarding max outgoing conns, max rsp body size

- implemented by a new httpclient.Client{} that wraps an underlying
  client with a queue to limit connections, and limit reader wrapping
  a response body with a configured maximum size
- update pub.HttpClient args passed around to be this new httpclient.Client{}

Signed-off-by: kim <grufwub@gmail.com>

* add httpclient tests, move ip validation to separate package + change mechanism

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts

Signed-off-by: kim <grufwub@gmail.com>

* use singular mutex in transport rather than separate signer mus

Signed-off-by: kim <grufwub@gmail.com>

* improved useragent string

Signed-off-by: kim <grufwub@gmail.com>

* add note regarding missing test

Signed-off-by: kim <grufwub@gmail.com>

* remove useragent field from transport (instead store in controller)

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* reset other signing headers on each loop iteration

Signed-off-by: kim <grufwub@gmail.com>

* respect request ctx during retry-backoff sleep period

Signed-off-by: kim <grufwub@gmail.com>

* use external pkg with docs explaining performance "hack"

Signed-off-by: kim <grufwub@gmail.com>

* use http package constants instead of string method literals

Signed-off-by: kim <grufwub@gmail.com>

* add license file headers

Signed-off-by: kim <grufwub@gmail.com>

* update code comment to match new func names

Signed-off-by: kim <grufwub@gmail.com>

* updates to user-agent string

Signed-off-by: kim <grufwub@gmail.com>

* update signed testrig models to fit with new transport logic (instead uses separate signer now)

Signed-off-by: kim <grufwub@gmail.com>

* fuck you linter

Signed-off-by: kim <grufwub@gmail.com>

5 files changed, 50 insertions, 69 deletions

diff --git a/testrig/federatingdb.go b/testrig/federatingdb.go
index 3f71274ca..468cfbfd1 100644
--- a/testrig/federatingdb.go
+++ b/testrig/federatingdb.go
@@ -1,13 +1,13 @@
 package testrig
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
-	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestFederatingDB returns a federating DB with the underlying db
-func NewTestFederatingDB(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) federatingdb.DB {
+func NewTestFederatingDB(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) federatingdb.DB {
 	return federatingdb.New(db, fedWorker)
 }
diff --git a/testrig/federator.go b/testrig/federator.go
index 475ed3346..0546325ab 100644
--- a/testrig/federator.go
+++ b/testrig/federator.go
@@ -20,15 +20,15 @@ package testrig
 
 import (
 	"codeberg.org/gruf/go-store/kv"
+	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
-	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestFederator returns a federator with the given database and (mock!!) transport controller.
-func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *worker.Worker[messages.FromFederator]) federation.Federator {
+func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *concurrency.WorkerPool[messages.FromFederator]) federation.Federator {
 	return federation.NewFederator(db, NewTestFederatingDB(db, fedWorker), tc, NewTestTypeConverter(db), mediaManager)
 }
diff --git a/testrig/processor.go b/testrig/processor.go
index c0fbd8a74..15f9040f7 100644
--- a/testrig/processor.go
+++ b/testrig/processor.go
@@ -20,16 +20,16 @@ package testrig
 
 import (
 	"codeberg.org/gruf/go-store/kv"
+	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
-	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestProcessor returns a Processor suitable for testing purposes
-func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI], fedWorker *worker.Worker[messages.FromFederator]) processing.Processor {
+func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *concurrency.WorkerPool[messages.FromClientAPI], fedWorker *concurrency.WorkerPool[messages.FromFederator]) processing.Processor {
 	return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender, clientWorker, fedWorker)
 }
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index 8894e562d..a74357feb 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -20,8 +20,6 @@ package testrig
 
 import (
 	"bytes"
-	"context"
-	"crypto"
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
@@ -29,7 +27,6 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -42,8 +39,7 @@ import (
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
-	"github.com/superseriousbusiness/gotosocial/internal/worker"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@@ -1855,86 +1851,71 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
 	}
 }
 
-// GetSignatureForActivity does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
-// the HTTP Signature for the given activity, public key ID, private key, and destination.
-func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
-	// create a client that basically just pulls the signature out of the request and sets it
-	client := &mockHTTPClient{
-		do: func(req *http.Request) (*http.Response, error) {
-			signatureHeader = req.Header.Get("Signature")
-			digestHeader = req.Header.Get("Digest")
-			dateHeader = req.Header.Get("Date")
-			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
-			return &http.Response{
-				StatusCode: 200,
-				Body:       r,
-			}, nil
-		},
-	}
-
-	// Create temporary federator worker for transport controller
-	fedWorker := worker.New[messages.FromFederator](-1, -1)
-	_ = fedWorker.Start()
-	defer func() { _ = fedWorker.Stop() }()
-
-	// use the client to create a new transport
-	c := NewTestTransportController(client, NewTestDB(), fedWorker)
-	tp, err := c.NewTransport(pubKeyID, privkey)
+// GetSignatureForActivity prepares a mock HTTP request as if it were going to deliver activity to destination signed for privkey and pubKeyID, signs the request and returns the header values.
+func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
+	// convert the activity into json bytes
+	m, err := activity.Serialize()
+	if err != nil {
+		panic(err)
+	}
+	b, err := json.Marshal(m)
 	if err != nil {
 		panic(err)
 	}
 
-	// convert the activity into json bytes
-	m, err := activity.Serialize()
+	// Prepare HTTP request signer
+	sig, err := transport.NewPOSTSigner(120)
 	if err != nil {
 		panic(err)
 	}
-	bytes, err := json.Marshal(m)
+
+	// Prepare a mock request ready for signing
+	r, err := http.NewRequest("POST", destination.String(), bytes.NewReader(b))
 	if err != nil {
 		panic(err)
 	}
+	r.Header.Set("Host", destination.Host)
+	r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
 
-	// trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
-	if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
+	// Sign this new HTTP request
+	if err := sig.SignRequest(privkey, pubKeyID, r, b); err != nil {
 		panic(err)
 	}
 
+	// Load signed data from request
+	signatureHeader = r.Header.Get("Signature")
+	digestHeader = r.Header.Get("Digest")
+	dateHeader = r.Header.Get("Date")
+
 	// headers should now be populated
 	return
 }
 
-// GetSignatureForDereference does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
-// the HTTP Signature for the given derefence GET request using public key ID, private key, and destination.
-func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
-	// create a client that basically just pulls the signature out of the request and sets it
-	client := &mockHTTPClient{
-		do: func(req *http.Request) (*http.Response, error) {
-			signatureHeader = req.Header.Get("Signature")
-			dateHeader = req.Header.Get("Date")
-			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
-			return &http.Response{
-				StatusCode: 200,
-				Body:       r,
-			}, nil
-		},
-	}
-
-	// Create temporary federator worker for transport controller
-	fedWorker := worker.New[messages.FromFederator](-1, -1)
-	_ = fedWorker.Start()
-	defer func() { _ = fedWorker.Stop() }()
-
-	// use the client to create a new transport
-	c := NewTestTransportController(client, NewTestDB(), fedWorker)
-	tp, err := c.NewTransport(pubKeyID, privkey)
+// GetSignatureForDereference prepares a mock HTTP request as if it were going to dereference destination signed for privkey and pubKeyID, signs the request and returns the header values.
+func GetSignatureForDereference(pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
+	// Prepare HTTP request signer
+	sig, err := transport.NewGETSigner(120)
 	if err != nil {
 		panic(err)
 	}
 
-	// trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
-	if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
+	// Prepare a mock request ready for signing
+	r, err := http.NewRequest("GET", destination.String(), nil)
+	if err != nil {
 		panic(err)
 	}
+	r.Header.Set("Host", destination.Host)
+	r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+
+	// Sign this new HTTP request
+	if err := sig.SignRequest(privkey, pubKeyID, r, nil); err != nil {
+		panic(err)
+	}
+
+	// Load signed data from request
+	signatureHeader = r.Header.Get("Signature")
+	digestHeader = r.Header.Get("Digest")
+	dateHeader = r.Header.Get("Date")
 
 	// headers should now be populated
 	return
diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go
index 943be7a61..7f4c7f890 100644
--- a/testrig/transportcontroller.go
+++ b/testrig/transportcontroller.go
@@ -24,11 +24,11 @@ import (
 	"net/http"
 
 	"github.com/superseriousbusiness/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
-	"github.com/superseriousbusiness/gotosocial/internal/worker"
 )
 
 // NewTestTransportController returns a test transport controller with the given http client.
@@ -40,7 +40,7 @@ import (
 // Unlike the other test interfaces provided in this package, you'll probably want to call this function
 // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
 // basis.
-func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *worker.Worker[messages.FromFederator]) transport.Controller {
+func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) transport.Controller {
 	return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client)
 }