[feature] For video attachments, store + return fps, bitrate, duration (#1282)

* start messing about with different mp4 metadata extraction

* heyyooo it works

* add test cow

* move useful multierror to gtserror package

* error out if video doesn't seem to be a real mp4

* test parsing mkv in disguise as mp4

* tidy up error handling

* remove extraneous line

* update framerate formatting

* use float32 for aspect

* fixy mctesterson

9 files changed, 201 insertions, 54 deletions

diff --git a/internal/media/image.go b/internal/media/image.go
index aedac5707..a03098930 100644
--- a/internal/media/image.go
+++ b/internal/media/image.go
@@ -48,7 +48,7 @@ func decodeGif(r io.Reader) (*mediaMeta, error) {
 	width := gif.Config.Width
 	height := gif.Config.Height
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -85,7 +85,7 @@ func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
 	width := i.Bounds().Size().X
 	height := i.Bounds().Size().Y
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -167,7 +167,7 @@ func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bo
 	thumbX := thumb.Bounds().Size().X
 	thumbY := thumb.Bounds().Size().Y
 	size := thumbX * thumbY
-	aspect := float64(thumbX) / float64(thumbY)
+	aspect := float32(thumbX) / float32(thumbY)
 
 	im := &mediaMeta{
 		width:  thumbX,
diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
index f9361a831..c61bdae28 100644
--- a/internal/media/manager_test.go
+++ b/internal/media/manager_test.go
@@ -407,9 +407,13 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	suite.Equal(accountID, attachment.AccountID)
 
 	// file meta should be correctly derived from the video
-	suite.EqualValues(gtsmodel.Original{
-		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
-	}, attachment.FileMeta.Original)
+	suite.Equal(338, attachment.FileMeta.Original.Width)
+	suite.Equal(240, attachment.FileMeta.Original.Height)
+	suite.Equal(81120, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.4083333, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(6.5862, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(29.000029, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0x3b3e1, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
 		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
 	}, attachment.FileMeta.Small)
@@ -448,6 +452,108 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
 }
 
+func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() {
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/longer-mp4-original.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// process the media with no additional info provided
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+	// fetch the attachment id from the processing media
+	attachmentID := processingMedia.AttachmentID()
+
+	// do a blocking call to fetch the attachment
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.NoError(err)
+	suite.NotNil(attachment)
+
+	// make sure it's got the stuff set on it that we expect
+	// the attachment ID and accountID we expect
+	suite.Equal(attachmentID, attachment.ID)
+	suite.Equal(accountID, attachment.AccountID)
+
+	// file meta should be correctly derived from the video
+	suite.Equal(600, attachment.FileMeta.Original.Width)
+	suite.Equal(330, attachment.FileMeta.Original.Height)
+	suite.Equal(198000, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.8181819, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(16.6, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(10, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate)
+	suite.EqualValues(gtsmodel.Small{
+		Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819,
+	}, attachment.FileMeta.Small)
+	suite.Equal("video/mp4", attachment.File.ContentType)
+	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal(109549, attachment.File.FileSize)
+	suite.Equal("", attachment.Blurhash)
+
+	// now make sure the attachment is in the database
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
+	suite.NoError(err)
+	suite.NotNil(dbAttachment)
+
+	// make sure the processed file is in storage
+	processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytes)
+
+	// load the processed bytes from our test folder, to compare
+	processedFullBytesExpected, err := os.ReadFile("./test/longer-mp4-processed.mp4")
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytesExpected)
+
+	// the bytes in storage should be what we expected
+	suite.Equal(processedFullBytesExpected, processedFullBytes)
+
+	// now do the same for the thumbnail and make sure it's what we expected
+	processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytes)
+
+	processedThumbnailBytesExpected, err := os.ReadFile("./test/longer-mp4-thumbnail.jpg")
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytesExpected)
+
+	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
+}
+
+func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() {
+	// try to load an 'mp4' that's actually an mkv in disguise
+
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/not-an.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// pre processing should go fine but...
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+
+	// we should get an error while loading
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"")
+	suite.Nil(attachment)
+}
+
 func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
 	ctx := context.Background()
 
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index a7ea4dbab..f22102d6d 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -249,16 +249,32 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
 		}
 
 		// set appropriate fields on the attachment based on the image we derived
+
+		// generic fields
+		p.attachment.File.UpdatedAt = time.Now()
 		p.attachment.FileMeta.Original = gtsmodel.Original{
 			Width:  decoded.width,
 			Height: decoded.height,
 			Size:   decoded.size,
 			Aspect: decoded.aspect,
 		}
-		p.attachment.File.UpdatedAt = time.Now()
-		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
+
+		// nullable fields
+		if decoded.duration != 0 {
+			i := decoded.duration
+			p.attachment.FileMeta.Original.Duration = &i
+		}
+		if decoded.framerate != 0 {
+			i := decoded.framerate
+			p.attachment.FileMeta.Original.Framerate = &i
+		}
+		if decoded.bitrate != 0 {
+			i := decoded.bitrate
+			p.attachment.FileMeta.Original.Bitrate = &i
+		}
 
 		// we're done processing the full-size image
+		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
 		atomic.StoreInt32(&p.fullSizeState, int32(complete))
 		log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
 		fallthrough
diff --git a/internal/media/test/longer-mp4-original.mp4 b/internal/media/test/longer-mp4-original.mp4
new file mode 100644
index 000000000..cfb596612
--- /dev/null
+++ b/internal/media/test/longer-mp4-original.mp4
diff --git a/internal/media/test/longer-mp4-processed.mp4 b/internal/media/test/longer-mp4-processed.mp4
new file mode 100644
index 000000000..cfb596612
--- /dev/null
+++ b/internal/media/test/longer-mp4-processed.mp4
diff --git a/internal/media/test/longer-mp4-thumbnail.jpg b/internal/media/test/longer-mp4-thumbnail.jpg
new file mode 100644
index 000000000..e77534950
--- /dev/null
+++ b/internal/media/test/longer-mp4-thumbnail.jpg
diff --git a/internal/media/test/not-an.mp4 b/internal/media/test/not-an.mp4
new file mode 100644
index 000000000..9bc8a7638
--- /dev/null
+++ b/internal/media/test/not-an.mp4
diff --git a/internal/media/types.go b/internal/media/types.go
index e7edfe643..47a545cb2 100644
--- a/internal/media/types.go
+++ b/internal/media/types.go
@@ -137,7 +137,12 @@ type mediaMeta struct {
 	width    int
 	height   int
 	size     int
-	aspect   float64
+	aspect   float32
 	blurhash string
 	small    []byte
+
+	// video-specific properties
+	duration  float32
+	framerate float32
+	bitrate   uint64
 }
diff --git a/internal/media/video.go b/internal/media/video.go
index ef486d63d..8db9061b5 100644
--- a/internal/media/video.go
+++ b/internal/media/video.go
@@ -20,7 +20,6 @@ package media
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"image"
 	"image/color"
@@ -30,6 +29,7 @@ import (
 	"os"
 
 	"github.com/abema/go-mp4"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
@@ -61,60 +61,80 @@ func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) {
 		return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err)
 	}
 
-	// define some vars we need to pull the width/height out of the video
 	var (
-		height      int
-		width       int
-		readHandler = getReadHandler(&height, &width)
+		width     int
+		height    int
+		duration  float32
+		framerate float32
+		bitrate   uint64
 	)
 
-	// do the actual decoding here, providing the temporary file we created as readseeker
-	if _, err := mp4.ReadBoxStructure(tempFile, readHandler); err != nil {
-		return nil, fmt.Errorf("parsing video data: %w", err)
+	// probe the video file to extract useful metadata from it; for methodology, see:
+	// https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154
+	info, err := mp4.Probe(tempFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not probe temporary video file %s: %w", tempFileName, err)
 	}
 
-	// width + height should now be updated by the readHandler
-	return &mediaMeta{
-		width:  width,
-		height: height,
-		size:   height * width,
-		aspect: float64(width) / float64(height),
-	}, nil
-}
+	for _, tr := range info.Tracks {
+		if tr.AVC == nil {
+			continue
+		}
 
-// getReadHandler returns a handler function that updates the underling
-// values of the given height and width int pointers to the hightest and
-// widest points of the video.
-func getReadHandler(height *int, width *int) func(h *mp4.ReadHandle) (interface{}, error) {
-	return func(rh *mp4.ReadHandle) (interface{}, error) {
-		if rh.BoxInfo.Type == mp4.BoxTypeTkhd() {
-			box, _, err := rh.ReadPayload()
-			if err != nil {
-				return nil, fmt.Errorf("could not read mp4 payload: %w", err)
-			}
-
-			tkhd, ok := box.(*mp4.Tkhd)
-			if !ok {
-				return nil, errors.New("box was not of type *mp4.Tkhd")
-			}
-
-			// if height + width of this box are greater than what
-			// we have stored, then update our stored values
-			if h := int(tkhd.GetHeight()); h > *height {
-				*height = h
-			}
-
-			if w := int(tkhd.GetWidth()); w > *width {
-				*width = w
-			}
+		if w := int(tr.AVC.Width); w > width {
+			width = w
 		}
 
-		if rh.BoxInfo.IsSupportedType() {
-			return rh.Expand()
+		if h := int(tr.AVC.Height); h > height {
+			height = h
 		}
 
-		return nil, nil
+		if br := tr.Samples.GetBitrate(tr.Timescale); br > bitrate {
+			bitrate = br
+		} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > bitrate {
+			bitrate = br
+		}
+
+		if d := float32(tr.Duration) / float32(tr.Timescale); d > duration {
+			duration = d
+			framerate = float32(len(tr.Samples)) / duration
+		}
 	}
+
+	var errs gtserror.MultiError
+	if width == 0 {
+		errs = append(errs, "video width could not be discovered")
+	}
+
+	if height == 0 {
+		errs = append(errs, "video height could not be discovered")
+	}
+
+	if duration == 0 {
+		errs = append(errs, "video duration could not be discovered")
+	}
+
+	if framerate == 0 {
+		errs = append(errs, "video framerate could not be discovered")
+	}
+
+	if bitrate == 0 {
+		errs = append(errs, "video bitrate could not be discovered")
+	}
+
+	if errs != nil {
+		return nil, errs.Combine()
+	}
+
+	return &mediaMeta{
+		width:     width,
+		height:    height,
+		duration:  duration,
+		framerate: framerate,
+		bitrate:   bitrate,
+		size:      height * width,
+		aspect:    float32(width) / float32(height),
+	}, nil
 }
 
 func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
@@ -134,7 +154,7 @@ func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
 		width:  width,
 		height: height,
 		size:   width * height,
-		aspect: float64(width) / float64(height),
+		aspect: float32(width) / float32(height),
 		small:  out.Bytes(),
 	}, nil
 }