[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)

* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests

Signed-off-by: kim <grufwub@gmail.com>

* update code comments, defer mutex unlocks

Signed-off-by: kim <grufwub@gmail.com>

* add count to 'performing request' log message

Signed-off-by: kim <grufwub@gmail.com>

* reduce repeated conversions of same url.URL object

Signed-off-by: kim <grufwub@gmail.com>

* move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue

Signed-off-by: kim <grufwub@gmail.com>

* fix security advisories regarding max outgoing conns, max rsp body size

- implemented by a new httpclient.Client{} that wraps an underlying
  client with a queue to limit connections, and limit reader wrapping
  a response body with a configured maximum size
- update pub.HttpClient args passed around to be this new httpclient.Client{}

Signed-off-by: kim <grufwub@gmail.com>

* add httpclient tests, move ip validation to separate package + change mechanism

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts

Signed-off-by: kim <grufwub@gmail.com>

* use singular mutex in transport rather than separate signer mus

Signed-off-by: kim <grufwub@gmail.com>

* improved useragent string

Signed-off-by: kim <grufwub@gmail.com>

* add note regarding missing test

Signed-off-by: kim <grufwub@gmail.com>

* remove useragent field from transport (instead store in controller)

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* reset other signing headers on each loop iteration

Signed-off-by: kim <grufwub@gmail.com>

* respect request ctx during retry-backoff sleep period

Signed-off-by: kim <grufwub@gmail.com>

* use external pkg with docs explaining performance "hack"

Signed-off-by: kim <grufwub@gmail.com>

* use http package constants instead of string method literals

Signed-off-by: kim <grufwub@gmail.com>

* add license file headers

Signed-off-by: kim <grufwub@gmail.com>

* update code comment to match new func names

Signed-off-by: kim <grufwub@gmail.com>

* updates to user-agent string

Signed-off-by: kim <grufwub@gmail.com>

* update signed testrig models to fit with new transport logic (instead uses separate signer now)

Signed-off-by: kim <grufwub@gmail.com>

* fuck you linter

Signed-off-by: kim <grufwub@gmail.com>

1 files changed, 109 insertions, 87 deletions

diff --git a/internal/transport/controller.go b/internal/transport/controller.go
index 56a922a8b..280d4bc0b 100644
--- a/internal/transport/controller.go
+++ b/internal/transport/controller.go
@@ -20,13 +20,17 @@ package transport
 
 import (
 	"context"
-	"crypto"
+	"crypto/rsa"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net/url"
-	"sync"
+	"runtime/debug"
+	"time"
 
-	"github.com/go-fed/httpsig"
+	"codeberg.org/gruf/go-byteutil"
+	"codeberg.org/gruf/go-cache/v2"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams"
@@ -37,109 +41,85 @@ import (
 
 // Controller generates transports for use in making federation requests to other servers.
 type Controller interface {
-	NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error)
+	// NewTransport returns an http signature transport with the given public key ID (URL location of pubkey), and the given private key.
+	NewTransport(pubKeyID string, privkey *rsa.PrivateKey) (Transport, error)
+
+	// NewTransportForUsername searches for account with username, and returns result of .NewTransport().
 	NewTransportForUsername(ctx context.Context, username string) (Transport, error)
 }
 
 type controller struct {
-	db       db.DB
-	clock    pub.Clock
-	client   pub.HttpClient
-	appAgent string
-
-	// dereferenceFollowersShortcut is a shortcut to dereference followers of an
-	// account on this instance, without making any external api/http calls.
-	//
-	// It is passed to new transports, and should only be invoked when the iri.Host == this host.
-	dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
-
-	// dereferenceUserShortcut is a shortcut to dereference followers an account on
-	// this instance, without making any external api/http calls.
-	//
-	// It is passed to new transports, and should only be invoked when the iri.Host == this host.
-	dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+	db        db.DB
+	fedDB     federatingdb.DB
+	clock     pub.Clock
+	client    pub.HttpClient
+	cache     cache.Cache[string, *transport]
+	userAgent string
 }
 
-func dereferenceFollowersShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
-	return func(ctx context.Context, iri *url.URL) ([]byte, error) {
-		followers, err := federatingDB.Followers(ctx, iri)
-		if err != nil {
-			return nil, err
-		}
+// NewController returns an implementation of the Controller interface for creating new transports
+func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
+	applicationName := viper.GetString(config.Keys.ApplicationName)
+	host := viper.GetString(config.Keys.Host)
 
-		i, err := streams.Serialize(followers)
-		if err != nil {
-			return nil, err
-		}
+	// Determine build information
+	build, _ := debug.ReadBuildInfo()
 
-		return json.Marshal(i)
+	c := &controller{
+		db:        db,
+		fedDB:     federatingDB,
+		clock:     clock,
+		client:    client,
+		cache:     cache.New[string, *transport](),
+		userAgent: fmt.Sprintf("%s; %s (gofed/activity gotosocial-%s)", applicationName, host, build.Main.Version),
 	}
-}
 
-func dereferenceUserShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
-	return func(ctx context.Context, iri *url.URL) ([]byte, error) {
-		user, err := federatingDB.Get(ctx, iri)
-		if err != nil {
-			return nil, err
-		}
-
-		i, err := streams.Serialize(user)
-		if err != nil {
-			return nil, err
-		}
-
-		return json.Marshal(i)
+	// Transport cache has TTL=1hr freq=1m
+	c.cache.SetTTL(time.Hour, false)
+	if !c.cache.Start(time.Minute) {
+		logrus.Panic("failed to start transport controller cache")
 	}
-}
 
-// NewController returns an implementation of the Controller interface for creating new transports
-func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
-	applicationName := viper.GetString(config.Keys.ApplicationName)
-	host := viper.GetString(config.Keys.Host)
-	appAgent := fmt.Sprintf("%s %s", applicationName, host)
-
-	return &controller{
-		db:                           db,
-		clock:                        clock,
-		client:                       client,
-		appAgent:                     appAgent,
-		dereferenceFollowersShortcut: dereferenceFollowersShortcut(federatingDB),
-		dereferenceUserShortcut:      dereferenceUserShortcut(federatingDB),
-	}
+	return c
 }
 
-// NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key.
-func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) {
-	prefs := []httpsig.Algorithm{httpsig.RSA_SHA256}
-	digestAlgo := httpsig.DigestSha256
-	getHeaders := []string{httpsig.RequestTarget, "host", "date"}
-	postHeaders := []string{httpsig.RequestTarget, "host", "date", "digest"}
+func (c *controller) NewTransport(pubKeyID string, privkey *rsa.PrivateKey) (Transport, error) {
+	// Generate public key string for cache key
+	//
+	// NOTE: it is safe to use the public key as the cache
+	// key here as we are generating it ourselves from the
+	// private key. If we were simply using a public key
+	// provided as argument that would absolutely NOT be safe.
+	pubStr := privkeyToPublicStr(privkey)
+
+	// First check for cached transport
+	transp, ok := c.cache.Get(pubStr)
+	if ok {
+		return transp, nil
+	}
 
-	getSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, getHeaders, httpsig.Signature, 120)
-	if err != nil {
-		return nil, fmt.Errorf("error creating get signer: %s", err)
+	// Create the transport
+	transp = &transport{
+		controller: c,
+		pubKeyID:   pubKeyID,
+		privkey:    privkey,
 	}
 
-	postSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, postHeaders, httpsig.Signature, 120)
-	if err != nil {
-		return nil, fmt.Errorf("error creating post signer: %s", err)
+	// Cache this transport under pubkey
+	if !c.cache.Put(pubStr, transp) {
+		var cached *transport
+
+		cached, ok = c.cache.Get(pubStr)
+		if !ok {
+			// Some ridiculous race cond.
+			c.cache.Set(pubStr, transp)
+		} else {
+			// Use already cached
+			transp = cached
+		}
 	}
 
-	sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey)
-
-	return &transport{
-		client:                       c.client,
-		appAgent:                     c.appAgent,
-		gofedAgent:                   "(go-fed/activity v1.0.0)",
-		clock:                        c.clock,
-		pubKeyID:                     pubKeyID,
-		privkey:                      privkey,
-		sigTransport:                 sigTransport,
-		getSigner:                    getSigner,
-		getSignerMu:                  &sync.Mutex{},
-		dereferenceFollowersShortcut: c.dereferenceFollowersShortcut,
-		dereferenceUserShortcut:      c.dereferenceUserShortcut,
-	}, nil
+	return transp, nil
 }
 
 func (c *controller) NewTransportForUsername(ctx context.Context, username string) (Transport, error) {
@@ -164,3 +144,45 @@ func (c *controller) NewTransportForUsername(ctx context.Context, username strin
 	}
 	return transport, nil
 }
+
+// dereferenceLocalFollowers is a shortcut to dereference followers of an
+// account on this instance, without making any external api/http calls.
+//
+// It is passed to new transports, and should only be invoked when the iri.Host == this host.
+func (c *controller) dereferenceLocalFollowers(ctx context.Context, iri *url.URL) ([]byte, error) {
+	followers, err := c.fedDB.Followers(ctx, iri)
+	if err != nil {
+		return nil, err
+	}
+
+	i, err := streams.Serialize(followers)
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(i)
+}
+
+// dereferenceLocalUser is a shortcut to dereference followers an account on
+// this instance, without making any external api/http calls.
+//
+// It is passed to new transports, and should only be invoked when the iri.Host == this host.
+func (c *controller) dereferenceLocalUser(ctx context.Context, iri *url.URL) ([]byte, error) {
+	user, err := c.fedDB.Get(ctx, iri)
+	if err != nil {
+		return nil, err
+	}
+
+	i, err := streams.Serialize(user)
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(i)
+}
+
+// privkeyToPublicStr will create a string representation of RSA public key from private.
+func privkeyToPublicStr(privkey *rsa.PrivateKey) string {
+	b := x509.MarshalPKCS1PublicKey(&privkey.PublicKey)
+	return byteutil.B2S(b)
+}