[performance] move thumbnail generation to go code where possible (#3183)

* wrap thumbnailing code to handle generation natively where possible

* more code comments!

* add even more code comments!

* add code comments about blurhash generation

* maintain image rotation if contained in exif data

* move rotation before resizing

* ensure pix_fmt actually selected by ffprobe, check for alpha layer with gifs

* use linear instead of nearest-neighbour for resizing

* work with image "orientation" instead of "rotation". use default 75% quality for both webp and jpeg generation

* add header to new file

* use thumb extension when getting thumb mime type

* update test models and tests with new media processing

* add suggested code comments

* add note about thumbnail filter count reducing memory usage

1 files changed, 380 insertions, 0 deletions

diff --git a/internal/media/thumbnail.go b/internal/media/thumbnail.go
new file mode 100644
index 000000000..36ef24a01
--- /dev/null
+++ b/internal/media/thumbnail.go
@@ -0,0 +1,380 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package media
+
+import (
+	"context"
+	"image"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/buckket/go-blurhash"
+	"github.com/disintegration/imaging"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"golang.org/x/image/webp"
+)
+
+// generateThumb generates a thumbnail for the
+// input file at path, resizing it to the given
+// dimensions and generating a blurhash if needed.
+// This wraps much of the complex thumbnailing
+// logic in which where possible we use native
+// Go libraries for generating thumbnails, else
+// always falling back to slower but much more
+// widely supportive ffmpeg.
+func generateThumb(
+	ctx context.Context,
+	filepath string,
+	width, height int,
+	orientation int,
+	pixfmt string,
+	needBlurhash bool,
+) (
+	outpath string,
+	blurhash string,
+	err error,
+) {
+	var ext string
+
+	// Generate thumb output path REPLACING extension.
+	if i := strings.IndexByte(filepath, '.'); i != -1 {
+		outpath = filepath[:i] + "_thumb.webp"
+		ext = filepath[i+1:] // old extension
+	} else {
+		return "", "", gtserror.New("input file missing extension")
+	}
+
+	// Check for the few media types we
+	// have native Go decoding that allow
+	// us to generate thumbs natively.
+	switch {
+
+	case ext == "jpeg":
+		// Replace the "webp" with "jpeg", as we'll
+		// use our native Go thumbnailing generation.
+		outpath = outpath[:len(outpath)-4] + "jpeg"
+
+		log.Debug(ctx, "generating thumb from jpeg")
+		blurhash, err := generateNativeThumb(
+			filepath,
+			outpath,
+			width,
+			height,
+			orientation,
+			jpeg.Decode,
+			needBlurhash,
+		)
+		return outpath, blurhash, err
+
+	// We specifically only allow generating native
+	// thumbnails from gif IF it doesn't contain an
+	// alpha channel. We'll ultimately be encoding to
+	// jpeg which doesn't support transparency layers.
+	case ext == "gif" && !containsAlpha(pixfmt):
+
+		// Replace the "webp" with "jpeg", as we'll
+		// use our native Go thumbnailing generation.
+		outpath = outpath[:len(outpath)-4] + "jpeg"
+
+		log.Debug(ctx, "generating thumb from gif")
+		blurhash, err := generateNativeThumb(
+			filepath,
+			outpath,
+			width,
+			height,
+			orientation,
+			gif.Decode,
+			needBlurhash,
+		)
+		return outpath, blurhash, err
+
+	// We specifically only allow generating native
+	// thumbnails from png IF it doesn't contain an
+	// alpha channel. We'll ultimately be encoding to
+	// jpeg which doesn't support transparency layers.
+	case ext == "png" && !containsAlpha(pixfmt):
+
+		// Replace the "webp" with "jpeg", as we'll
+		// use our native Go thumbnailing generation.
+		outpath = outpath[:len(outpath)-4] + "jpeg"
+
+		log.Debug(ctx, "generating thumb from png")
+		blurhash, err := generateNativeThumb(
+			filepath,
+			outpath,
+			width,
+			height,
+			orientation,
+			png.Decode,
+			needBlurhash,
+		)
+		return outpath, blurhash, err
+
+	// We specifically only allow generating native
+	// thumbnails from webp IF it doesn't contain an
+	// alpha channel. We'll ultimately be encoding to
+	// jpeg which doesn't support transparency layers.
+	case ext == "webp" && !containsAlpha(pixfmt):
+
+		// Replace the "webp" with "jpeg", as we'll
+		// use our native Go thumbnailing generation.
+		outpath = outpath[:len(outpath)-4] + "jpeg"
+
+		log.Debug(ctx, "generating thumb from webp")
+		blurhash, err := generateNativeThumb(
+			filepath,
+			outpath,
+			width,
+			height,
+			orientation,
+			webp.Decode,
+			needBlurhash,
+		)
+		return outpath, blurhash, err
+	}
+
+	// The fallback for thumbnail generation, which
+	// encompasses most media types is with ffmpeg.
+	log.Debug(ctx, "generating thumb with ffmpeg")
+	if err := ffmpegGenerateWebpThumb(ctx,
+		filepath,
+		outpath,
+		width,
+		height,
+		pixfmt,
+	); err != nil {
+		return outpath, "", err
+	}
+
+	if needBlurhash {
+		// Generate new blurhash from webp output thumb.
+		blurhash, err = generateWebpBlurhash(outpath)
+		if err != nil {
+			return outpath, "", gtserror.Newf("error generating blurhash: %w", err)
+		}
+	}
+
+	return outpath, blurhash, err
+}
+
+// generateNativeThumb generates a thumbnail
+// using native Go code, using given decode
+// function to get image, resize to given dimens,
+// and write to output filepath as JPEG. If a
+// blurhash is required it will also generate
+// this from the image.Image while in-memory.
+func generateNativeThumb(
+	inpath, outpath string,
+	width, height int,
+	orientation int,
+	decode func(io.Reader) (image.Image, error),
+	needBlurhash bool,
+) (
+	string, // blurhash
+	error,
+) {
+	// Open input file at given path.
+	infile, err := os.Open(inpath)
+	if err != nil {
+		return "", gtserror.Newf("error opening input file %s: %w", inpath, err)
+	}
+
+	// Decode image into memory.
+	img, err := decode(infile)
+
+	// Done with file.
+	_ = infile.Close()
+
+	if err != nil {
+		return "", gtserror.Newf("error decoding file %s: %w", inpath, err)
+	}
+
+	// Apply orientation BEFORE any resize,
+	// as our image dimensions are calculated
+	// taking orientation into account.
+	switch orientation {
+	case orientationFlipH:
+		img = imaging.FlipH(img)
+	case orientationFlipV:
+		img = imaging.FlipV(img)
+	case orientationRotate90:
+		img = imaging.Rotate90(img)
+	case orientationRotate180:
+		img = imaging.Rotate180(img)
+	case orientationRotate270:
+		img = imaging.Rotate270(img)
+	case orientationTranspose:
+		img = imaging.Transpose(img)
+	case orientationTransverse:
+		img = imaging.Transverse(img)
+	}
+
+	// Resize image to dimens.
+	img = imaging.Resize(img,
+		width, height,
+		imaging.Linear,
+	)
+
+	// Open output file at given path.
+	outfile, err := os.Create(outpath)
+	if err != nil {
+		return "", gtserror.Newf("error opening output file %s: %w", outpath, err)
+	}
+
+	// Encode in-memory image to output file.
+	// (nil uses defaults, i.e. quality=75).
+	err = jpeg.Encode(outfile, img, nil)
+
+	// Done with file.
+	_ = outfile.Close()
+
+	if err != nil {
+		return "", gtserror.Newf("error encoding image: %w", err)
+	}
+
+	if needBlurhash {
+		// for generating blurhashes, it's more cost effective to
+		// lose detail since it's blurry, so make a tiny version.
+		tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor)
+
+		// Drop the larger image
+		// ref as soon as possible
+		// to allow GC to claim.
+		img = nil //nolint
+
+		// Generate blurhash for the tiny thumbnail.
+		blurhash, err := blurhash.Encode(4, 3, tiny)
+		if err != nil {
+			return "", gtserror.Newf("error generating blurhash: %w", err)
+		}
+
+		return blurhash, nil
+	}
+
+	return "", nil
+}
+
+// generateWebpBlurhash generates a blurhash for Webp at filepath.
+func generateWebpBlurhash(filepath string) (string, error) {
+	// Open the file at given path.
+	file, err := os.Open(filepath)
+	if err != nil {
+		return "", gtserror.Newf("error opening input file %s: %w", filepath, err)
+	}
+
+	// Decode image from file.
+	img, err := webp.Decode(file)
+
+	// Done with file.
+	_ = file.Close()
+
+	if err != nil {
+		return "", gtserror.Newf("error decoding file %s: %w", filepath, err)
+	}
+
+	// for generating blurhashes, it's more cost effective to
+	// lose detail since it's blurry, so make a tiny version.
+	tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor)
+
+	// Drop the larger image
+	// ref as soon as possible
+	// to allow GC to claim.
+	img = nil //nolint
+
+	// Generate blurhash for the tiny thumbnail.
+	blurhash, err := blurhash.Encode(4, 3, tiny)
+	if err != nil {
+		return "", gtserror.Newf("error generating blurhash: %w", err)
+	}
+
+	return blurhash, nil
+}
+
+// List of pixel formats that have an alpha layer.
+// Derived from the following very messy command:
+//
+//	for res in $(ffprobe -show_entries pixel_format=name:flags=alpha | grep -B1 alpha=1 | grep name); do echo $res | sed 's/name=//g' | sed 's/^/"/g' | sed 's/$/",/g'; done
+var alphaPixelFormats = []string{
+	"pal8",
+	"argb",
+	"rgba",
+	"abgr",
+	"bgra",
+	"yuva420p",
+	"ya8",
+	"yuva422p",
+	"yuva444p",
+	"yuva420p9be",
+	"yuva420p9le",
+	"yuva422p9be",
+	"yuva422p9le",
+	"yuva444p9be",
+	"yuva444p9le",
+	"yuva420p10be",
+	"yuva420p10le",
+	"yuva422p10be",
+	"yuva422p10le",
+	"yuva444p10be",
+	"yuva444p10le",
+	"yuva420p16be",
+	"yuva420p16le",
+	"yuva422p16be",
+	"yuva422p16le",
+	"yuva444p16be",
+	"yuva444p16le",
+	"rgba64be",
+	"rgba64le",
+	"bgra64be",
+	"bgra64le",
+	"ya16be",
+	"ya16le",
+	"gbrap",
+	"gbrap16be",
+	"gbrap16le",
+	"ayuv64le",
+	"ayuv64be",
+	"gbrap12be",
+	"gbrap12le",
+	"gbrap10be",
+	"gbrap10le",
+	"gbrapf32be",
+	"gbrapf32le",
+	"yuva422p12be",
+	"yuva422p12le",
+	"yuva444p12be",
+	"yuva444p12le",
+}
+
+// containsAlpha returns whether given pixfmt
+// (i.e. colorspace) contains an alpha channel.
+func containsAlpha(pixfmt string) bool {
+	if pixfmt == "" {
+		return false
+	}
+	for _, checkfmt := range alphaPixelFormats {
+		if pixfmt == checkfmt {
+			return true
+		}
+	}
+	return false
+}