[feature] support processing of (many) more media types (#3090)

* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables

1 files changed, 5 insertions, 32 deletions

diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go
index ba8760091..b78dbc2d9 100644
--- a/internal/httpclient/client.go
+++ b/internal/httpclient/client.go
@@ -31,7 +31,6 @@ import (
 	"strings"
 	"time"
 
-	"codeberg.org/gruf/go-bytesize"
 	"codeberg.org/gruf/go-cache/v3"
 	errorsv2 "codeberg.org/gruf/go-errors/v2"
 	"codeberg.org/gruf/go-iotools"
@@ -89,9 +88,6 @@ type Config struct {
 	// WriteBufferSize: see http.Transport{}.WriteBufferSize.
 	WriteBufferSize int
 
-	// MaxBodySize determines the maximum fetchable body size.
-	MaxBodySize int64
-
 	// Timeout: see http.Client{}.Timeout.
 	Timeout time.Duration
 
@@ -111,7 +107,6 @@ type Config struct {
 type Client struct {
 	client   http.Client
 	badHosts cache.TTLCache[string, struct{}]
-	bodyMax  int64
 	retries  uint
 }
 
@@ -137,11 +132,6 @@ func New(cfg Config) *Client {
 		cfg.MaxIdleConns = cfg.MaxOpenConnsPerHost * 10
 	}
 
-	if cfg.MaxBodySize <= 0 {
-		// By default set this to a reasonable 40MB.
-		cfg.MaxBodySize = int64(40 * bytesize.MiB)
-	}
-
 	// Protect the dialer
 	// with IP range sanitizer.
 	d.Control = (&Sanitizer{
@@ -151,7 +141,6 @@ func New(cfg Config) *Client {
 
 	// Prepare client fields.
 	c.client.Timeout = cfg.Timeout
-	c.bodyMax = cfg.MaxBodySize
 
 	// Prepare transport TLS config.
 	tlsClientConfig := &tls.Config{
@@ -377,31 +366,15 @@ func (c *Client) do(r *Request) (rsp *http.Response, retry bool, err error) {
 	rbody := (io.Reader)(rsp.Body)
 	cbody := (io.Closer)(rsp.Body)
 
-	var limit int64
-
-	if limit = rsp.ContentLength; limit < 0 {
-		// If unknown, use max as reader limit.
-		limit = c.bodyMax
-	}
-
-	// Don't trust them, limit body reads.
-	rbody = io.LimitReader(rbody, limit)
-
-	// Wrap closer to ensure entire body drained BEFORE close.
+	// Wrap closer to ensure body drained BEFORE close.
 	cbody = iotools.CloserAfterCallback(cbody, func() {
 		_, _ = discard.ReadFrom(rbody)
 	})
 
-	// Wrap body with limit.
-	rsp.Body = &struct {
-		io.Reader
-		io.Closer
-	}{rbody, cbody}
-
-	// Check response body not too large.
-	if rsp.ContentLength > c.bodyMax {
-		_ = rsp.Body.Close()
-		return nil, false, ErrBodyTooLarge
+	// Set the wrapped response body.
+	rsp.Body = &iotools.ReadCloserType{
+		Reader: rbody,
+		Closer: cbody,
 	}
 
 	return rsp, true, nil