[feature] support processing of (many) more media types (#3090)

* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables

1 files changed, 189 insertions, 248 deletions

diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index 466c3443f..43e153a4d 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -18,18 +18,12 @@
 package media
 
 import (
-	"bytes"
-	"cmp"
 	"context"
-	"image/jpeg"
-	"io"
 	"time"
 
 	errorsv2 "codeberg.org/gruf/go-errors/v2"
 	"codeberg.org/gruf/go-runners"
-	terminator "codeberg.org/superseriousbusiness/exif-terminator"
-	"github.com/disintegration/imaging"
-	"github.com/h2non/filetype"
+
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -145,19 +139,8 @@ func (p *ProcessingMedia) load(ctx context.Context) (
 		// full-size media attachment details.
 		//
 		// This will update p.media as it goes.
-		if err = p.store(ctx); err != nil {
-			return err
-		}
-
-		// Finish processing by reloading media into
-		// memory to get dimension and generate a thumb.
-		//
-		// This will update p.media as it goes.
-		if err = p.finish(ctx); err != nil {
-			return err //nolint:revive
-		}
-
-		return nil
+		err = p.store(ctx)
+		return err
 	})
 	media = p.media
 	return
@@ -167,286 +150,244 @@ func (p *ProcessingMedia) load(ctx context.Context) (
 // and updates the underlying attachment fields as necessary. It will then stream
 // bytes from p's reader directly into storage so that it can be retrieved later.
 func (p *ProcessingMedia) store(ctx context.Context) error {
-	// Load media from provided data fun
-	rc, sz, err := p.dataFn(ctx)
+	// Load media from data func.
+	rc, err := p.dataFn(ctx)
 	if err != nil {
 		return gtserror.Newf("error executing data function: %w", err)
 	}
 
+	var (
+		// predfine temporary media
+		// file path variables so we
+		// can remove them on error.
+		temppath  string
+		thumbpath string
+	)
+
 	defer func() {
-		// Ensure data reader gets closed on return.
-		if err := rc.Close(); err != nil {
-			log.Errorf(ctx, "error closing data reader: %v", err)
+		if err := remove(temppath, thumbpath); err != nil {
+			log.Errorf(ctx, "error(s) cleaning up files: %v", err)
 		}
 	}()
 
-	// Assume we're given correct file
-	// size, we can overwrite this later
-	// once we know THE TRUTH.
-	fileSize := int(sz)
-	p.media.File.FileSize = fileSize
-
-	// Prepare to read bytes from
-	// file header or magic number.
-	hdrBuf := newHdrBuf(fileSize)
-
-	// Read into buffer as much as possible.
-	//
-	// UnexpectedEOF means we couldn't read up to the
-	// given size, but we may still have read something.
-	//
-	// EOF means we couldn't read anything at all.
-	//
-	// Any other error likely means the connection messed up.
-	//
-	// In other words, rather counterintuitively, we
-	// can only proceed on no error or unexpected error!
-	n, err := io.ReadFull(rc, hdrBuf)
+	// Drain reader to tmp file
+	// (this reader handles close).
+	temppath, err = drainToTmp(rc)
 	if err != nil {
-		if err != io.ErrUnexpectedEOF {
-			return gtserror.Newf("error reading first bytes of incoming media: %w", err)
-		}
-
-		// Initial file size was misreported, so we didn't read
-		// fully into hdrBuf. Reslice it to the size we did read.
-		hdrBuf = hdrBuf[:n]
-		fileSize = n
-		p.media.File.FileSize = fileSize
+		return gtserror.Newf("error draining data to tmp: %w", err)
 	}
 
-	// Parse file type info from header buffer.
-	// This should only ever error if the buffer
-	// is empty (ie., the attachment is 0 bytes).
-	info, err := filetype.Match(hdrBuf)
+	// Pass input file through ffprobe to
+	// parse further metadata information.
+	result, err := ffprobe(ctx, temppath)
 	if err != nil {
-		return gtserror.Newf("error parsing file type: %w", err)
+		return gtserror.Newf("error ffprobing data: %w", err)
 	}
 
-	// Recombine header bytes with remaining stream
-	r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
-
-	// Assume we'll put
-	// this file in storage.
-	store := true
+	switch {
+	// No errors parsing data.
+	case result.Error == nil:
 
-	switch info.Extension {
-	case "mp4":
-		// No problem.
-
-	case "gif":
-		// No problem
-
-	case "jpg", "jpeg", "png", "webp":
-		if fileSize > 0 {
-			// A file size was provided so we can clean
-			// exif data from image as we're streaming it.
-			r, err = terminator.Terminate(r, fileSize, info.Extension)
-			if err != nil {
-				return gtserror.Newf("error cleaning exif data: %w", err)
-			}
-		}
+	// Data type unhandleable by ffprobe.
+	case result.Error.Code == -1094995529:
+		log.Warn(ctx, "unsupported data type")
+		return nil
 
 	default:
-		// The file is not a supported format that we can process, so we can't do much with it.
-		log.Warnf(ctx, "unsupported media extension '%s'; not caching locally", info.Extension)
-		store = false
+		return gtserror.Newf("ffprobe error: %w", err)
 	}
 
-	// Fill in correct attachment
-	// data now we've parsed it.
-	p.media.URL = uris.URIForAttachment(
-		p.media.AccountID,
-		string(TypeAttachment),
-		string(SizeOriginal),
-		p.media.ID,
-		info.Extension,
-	)
-
-	// Prefer discovered MIME, fallback to generic data stream.
-	mime := cmp.Or(info.MIME.Value, "application/octet-stream")
-	p.media.File.ContentType = mime
+	var ext string
 
-	// Calculate final media attachment file path.
-	p.media.File.Path = uris.StoragePathForAttachment(
-		p.media.AccountID,
-		string(TypeAttachment),
-		string(SizeOriginal),
-		p.media.ID,
-		info.Extension,
-	)
+	// Set the media type from ffprobe format data.
+	p.media.Type, ext = result.Format.GetFileType()
+	if p.media.Type == gtsmodel.FileTypeUnknown {
 
-	// We should only try to store the file if it's
-	// a format we can keep processing, otherwise be
-	// a bit cheeky: don't store it and let users
-	// click through to the remote server instead.
-	if !store {
+		// Return early (deleting file)
+		// for unhandled file types.
 		return nil
 	}
 
-	// File shouldn't already exist in storage at this point,
-	// but we do a check as it's worth logging / cleaning up.
-	if have, _ := p.mgr.state.Storage.Has(ctx, p.media.File.Path); have {
-		log.Warnf(ctx, "media already exists at: %s", p.media.File.Path)
-
-		// Attempt to remove existing media at storage path (might be broken / out-of-date)
-		if err := p.mgr.state.Storage.Delete(ctx, p.media.File.Path); err != nil {
-			return gtserror.Newf("error removing media %s from storage: %v", p.media.File.Path, err)
+	switch p.media.Type {
+	case gtsmodel.FileTypeImage:
+		// Pass file through ffmpeg clearing
+		// any excess metadata (e.g. EXIF).
+		if err := ffmpegClearMetadata(ctx,
+			temppath, ext,
+		); err != nil {
+			return gtserror.Newf("error cleaning metadata: %w", err)
 		}
-	}
-
-	// Write the final reader stream to our storage driver.
-	sz, err = p.mgr.state.Storage.PutStream(ctx, p.media.File.Path, r)
-	if err != nil {
-		return gtserror.Newf("error writing media to storage: %w", err)
-	}
-
-	// Set actual written size
-	// as authoritative file size.
-	p.media.File.FileSize = int(sz)
-
-	// We can now consider this cached.
-	p.media.Cached = util.Ptr(true)
-
-	return nil
-}
-
-func (p *ProcessingMedia) finish(ctx context.Context) error {
-	// Nothing else to do if
-	// media was not cached.
-	if !*p.media.Cached {
-		return nil
-	}
-
-	// Get a stream to the original file for further processing.
-	rc, err := p.mgr.state.Storage.GetStream(ctx, p.media.File.Path)
-	if err != nil {
-		return gtserror.Newf("error loading file from storage: %w", err)
-	}
-	defer rc.Close()
-
-	// fullImg is the processed version of
-	// the original (stripped + reoriented).
-	var fullImg *gtsImage
 
-	// Depending on the content type, we
-	// can do various types of decoding.
-	switch p.media.File.ContentType {
-
-	// .jpeg, .gif, .webp image type
-	case mimeImageJpeg, mimeImageGif, mimeImageWebp:
-		fullImg, err = decodeImage(rc,
-			imaging.AutoOrientation(true),
-		)
+		// Extract image metadata from streams.
+		width, height, err := result.ImageMeta()
 		if err != nil {
-			return gtserror.Newf("error decoding image: %w", err)
+			return err
 		}
-
-		// Mark as no longer unknown type now
-		// we know for sure we can decode it.
-		p.media.Type = gtsmodel.FileTypeImage
-
-	// .png image (requires ancillary chunk stripping)
-	case mimeImagePng:
-		fullImg, err = decodeImage(
-			&pngAncillaryChunkStripper{Reader: rc},
-			imaging.AutoOrientation(true),
+		p.media.FileMeta.Original.Width = width
+		p.media.FileMeta.Original.Height = height
+		p.media.FileMeta.Original.Size = (width * height)
+		p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
+
+		// Determine thumbnail dimensions to use.
+		thumbWidth, thumbHeight := thumbSize(width, height)
+		p.media.FileMeta.Small.Width = thumbWidth
+		p.media.FileMeta.Small.Height = thumbHeight
+		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
+		p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
+
+		// Generate a thumbnail image from input image path.
+		thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
+			thumbWidth,
+			thumbHeight,
 		)
 		if err != nil {
-			return gtserror.Newf("error decoding image: %w", err)
+			return gtserror.Newf("error generating image thumb: %w", err)
 		}
 
-		// Mark as no longer unknown type now
-		// we know for sure we can decode it.
-		p.media.Type = gtsmodel.FileTypeImage
+	case gtsmodel.FileTypeVideo:
+		// Pass file through ffmpeg clearing
+		// any excess metadata (e.g. EXIF).
+		if err := ffmpegClearMetadata(ctx,
+			temppath, ext,
+		); err != nil {
+			return gtserror.Newf("error cleaning metadata: %w", err)
+		}
 
-	// .mp4 video type
-	case mimeVideoMp4:
-		video, err := decodeVideoFrame(rc)
+		// Extract video metadata we can from streams.
+		width, height, framerate, err := result.VideoMeta()
 		if err != nil {
-			return gtserror.Newf("error decoding video: %w", err)
+			return err
+		}
+		p.media.FileMeta.Original.Width = width
+		p.media.FileMeta.Original.Height = height
+		p.media.FileMeta.Original.Size = (width * height)
+		p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
+		p.media.FileMeta.Original.Framerate = &framerate
+
+		// Extract total duration from format.
+		duration := result.Format.GetDuration()
+		p.media.FileMeta.Original.Duration = &duration
+
+		// Extract total bitrate from format.
+		bitrate := result.Format.GetBitRate()
+		p.media.FileMeta.Original.Bitrate = &bitrate
+
+		// Determine thumbnail dimensions to use.
+		thumbWidth, thumbHeight := thumbSize(width, height)
+		p.media.FileMeta.Small.Width = thumbWidth
+		p.media.FileMeta.Small.Height = thumbHeight
+		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
+		p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
+
+		// Extract a thumbnail frame from input video path.
+		thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
+			thumbWidth,
+			thumbHeight,
+		)
+		if err != nil {
+			return gtserror.Newf("error extracting video frame: %w", err)
 		}
 
-		// Set video frame as image.
-		fullImg = video.frame
-
-		// Set video metadata in attachment info.
-		p.media.FileMeta.Original.Duration = &video.duration
-		p.media.FileMeta.Original.Framerate = &video.framerate
-		p.media.FileMeta.Original.Bitrate = &video.bitrate
+	case gtsmodel.FileTypeAudio:
+		// Extract total duration from format.
+		duration := result.Format.GetDuration()
+		p.media.FileMeta.Original.Duration = &duration
+
+		// Extract total bitrate from format.
+		bitrate := result.Format.GetBitRate()
+		p.media.FileMeta.Original.Bitrate = &bitrate
+
+		// Extract image metadata from streams (if any),
+		// this will only exist for embedded album art.
+		width, height, _ := result.ImageMeta()
+		if width > 0 && height > 0 {
+
+			// Determine thumbnail dimensions to use.
+			thumbWidth, thumbHeight := thumbSize(width, height)
+			p.media.FileMeta.Small.Width = thumbWidth
+			p.media.FileMeta.Small.Height = thumbHeight
+			p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
+			p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
+
+			// Generate a thumbnail image from input image path.
+			thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
+				thumbWidth,
+				thumbHeight,
+			)
+			if err != nil {
+				return gtserror.Newf("error generating image thumb: %w", err)
+			}
+		}
 
-		// Mark as no longer unknown type now
-		// we know for sure we can decode it.
-		p.media.Type = gtsmodel.FileTypeVideo
+	default:
+		log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName)
+		return nil
 	}
 
-	// fullImg should be in-memory by
-	// now so we're done with storage.
-	if err := rc.Close(); err != nil {
-		return gtserror.Newf("error closing file: %w", err)
+	// Calculate final media attachment file path.
+	p.media.File.Path = uris.StoragePathForAttachment(
+		p.media.AccountID,
+		string(TypeAttachment),
+		string(SizeOriginal),
+		p.media.ID,
+		ext,
+	)
+
+	// Copy temporary file into storage at path.
+	filesz, err := p.mgr.state.Storage.PutFile(ctx,
+		p.media.File.Path,
+		temppath,
+	)
+	if err != nil {
+		return gtserror.Newf("error writing media to storage: %w", err)
 	}
 
-	// Set full-size dimensions in attachment info.
-	p.media.FileMeta.Original.Width = fullImg.Width()
-	p.media.FileMeta.Original.Height = fullImg.Height()
-	p.media.FileMeta.Original.Size = fullImg.Size()
-	p.media.FileMeta.Original.Aspect = fullImg.AspectRatio()
+	// Set final determined file size.
+	p.media.File.FileSize = int(filesz)
 
-	// Get smaller thumbnail image
-	thumbImg := fullImg.Thumbnail()
+	if thumbpath != "" {
+		// Note that neither thumbnail storage
+		// nor a blurhash are needed for audio.
 
-	// Garbage collector, you may
-	// now take our large son.
-	fullImg = nil
+		if p.media.Blurhash == "" {
+			// Generate blurhash (if not already) from thumbnail.
+			p.media.Blurhash, err = generateBlurhash(thumbpath)
+			if err != nil {
+				return gtserror.Newf("error generating thumb blurhash: %w", err)
+			}
+		}
 
-	// Only generate blurhash
-	// from thumb if necessary.
-	if p.media.Blurhash == "" {
-		hash, err := thumbImg.Blurhash()
+		// Copy thumbnail file into storage at path.
+		thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
+			p.media.Thumbnail.Path,
+			thumbpath,
+		)
 		if err != nil {
-			return gtserror.Newf("error generating blurhash: %w", err)
+			return gtserror.Newf("error writing thumb to storage: %w", err)
 		}
 
-		// Set the attachment blurhash.
-		p.media.Blurhash = hash
+		// Set final determined thumbnail size.
+		p.media.Thumbnail.FileSize = int(thumbsz)
 	}
 
-	// Thumbnail shouldn't exist in storage at this point,
-	// but we do a check as it's worth logging / cleaning up.
-	if have, _ := p.mgr.state.Storage.Has(ctx, p.media.Thumbnail.Path); have {
-		log.Warnf(ctx, "thumbnail already exists at: %s", p.media.Thumbnail.Path)
-
-		// Attempt to remove existing thumbnail (might be broken / out-of-date).
-		if err := p.mgr.state.Storage.Delete(ctx, p.media.Thumbnail.Path); err != nil {
-			return gtserror.Newf("error removing thumbnail %s from storage: %v", p.media.Thumbnail.Path, err)
-		}
-	}
-
-	// Create a thumbnail JPEG encoder stream.
-	enc := thumbImg.ToJPEG(&jpeg.Options{
-
-		// Good enough for
-		// a thumbnail.
-		Quality: 70,
-	})
-
-	// Stream-encode the JPEG thumbnail image into our storage driver.
-	sz, err := p.mgr.state.Storage.PutStream(ctx, p.media.Thumbnail.Path, enc)
-	if err != nil {
-		return gtserror.Newf("error stream-encoding thumbnail to storage: %w", err)
-	}
+	// Fill in correct attachment
+	// data now we've parsed it.
+	p.media.URL = uris.URIForAttachment(
+		p.media.AccountID,
+		string(TypeAttachment),
+		string(SizeOriginal),
+		p.media.ID,
+		ext,
+	)
 
-	// Set final written thumb size.
-	p.media.Thumbnail.FileSize = int(sz)
+	// Get mimetype for the file container
+	// type, falling back to generic data.
+	p.media.File.ContentType = getMimeType(ext)
 
-	// Set thumbnail dimensions in attachment info.
-	p.media.FileMeta.Small = gtsmodel.Small{
-		Width:  thumbImg.Width(),
-		Height: thumbImg.Height(),
-		Size:   thumbImg.Size(),
-		Aspect: thumbImg.AspectRatio(),
-	}
+	// We can now consider this cached.
+	p.media.Cached = util.Ptr(true)
 
-	// Finally set the attachment as processed.
+	// Finally set the attachment as finished processing.
 	p.media.Processing = gtsmodel.ProcessingStatusProcessed
 
 	return nil