summaryrefslogtreecommitdiff
path: root/internal/media
diff options
context:
space:
mode:
Diffstat (limited to 'internal/media')
-rw-r--r--internal/media/image.go85
-rw-r--r--internal/media/manager_test.go72
-rw-r--r--internal/media/processingmedia.go68
-rw-r--r--internal/media/test/test-mp4-original.mp4bin0 -> 312413 bytes
-rw-r--r--internal/media/test/test-mp4-processed.mp4bin0 -> 312413 bytes
-rw-r--r--internal/media/test/test-mp4-thumbnail.jpgbin0 -> 1912 bytes
-rw-r--r--internal/media/types.go13
-rw-r--r--internal/media/util.go15
-rw-r--r--internal/media/video.go140
9 files changed, 314 insertions, 79 deletions
diff --git a/internal/media/image.go b/internal/media/image.go
index b095a6c49..aedac5707 100644
--- a/internal/media/image.go
+++ b/internal/media/image.go
@@ -38,16 +38,7 @@ const (
thumbnailMaxHeight = 512
)
-type imageMeta struct {
- width int
- height int
- size int
- aspect float64
- blurhash string // defined only for calls to deriveThumbnail if createBlurhash is true
- small []byte // defined only for calls to deriveStaticEmoji or deriveThumbnail
-}
-
-func decodeGif(r io.Reader) (*imageMeta, error) {
+func decodeGif(r io.Reader) (*mediaMeta, error) {
gif, err := gif.DecodeAll(r)
if err != nil {
return nil, err
@@ -59,7 +50,7 @@ func decodeGif(r io.Reader) (*imageMeta, error) {
size := width * height
aspect := float64(width) / float64(height)
- return &imageMeta{
+ return &mediaMeta{
width: width,
height: height,
size: size,
@@ -67,7 +58,7 @@ func decodeGif(r io.Reader) (*imageMeta, error) {
}, nil
}
-func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
+func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
var i image.Image
var err error
@@ -96,7 +87,7 @@ func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
size := width * height
aspect := float64(width) / float64(height)
- return &imageMeta{
+ return &mediaMeta{
width: width,
height: height,
size: size,
@@ -104,8 +95,37 @@ func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
}, nil
}
-// deriveThumbnail returns a byte slice and metadata for a thumbnail
-// of a given jpeg, png, gif or webp, or an error if something goes wrong.
+// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
+func deriveStaticEmoji(r io.Reader, contentType string) (*mediaMeta, error) {
+ var i image.Image
+ var err error
+
+ switch contentType {
+ case mimeImagePng:
+ i, err = StrippedPngDecode(r)
+ if err != nil {
+ return nil, err
+ }
+ case mimeImageGif:
+ i, err = gif.Decode(r)
+ if err != nil {
+ return nil, err
+ }
+ default:
+ return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
+ }
+
+ out := &bytes.Buffer{}
+ if err := png.Encode(out, i); err != nil {
+ return nil, err
+ }
+ return &mediaMeta{
+ small: out.Bytes(),
+ }, nil
+}
+
+// deriveThumbnailFromImage returns a byte slice and metadata for a thumbnail
+// of a given piece of media, or an error if something goes wrong.
//
// If createBlurhash is true, then a blurhash will also be generated from a tiny
// version of the image. This costs precious CPU cycles, so only use it if you
@@ -113,7 +133,7 @@ func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
//
// If createBlurhash is false, then the blurhash field on the returned ImageAndMeta
// will be an empty string.
-func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*imageMeta, error) {
+func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bool) (*mediaMeta, error) {
var i image.Image
var err error
@@ -126,7 +146,7 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
})
i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true))
default:
- err = fmt.Errorf("content type %s can't be thumbnailed", contentType)
+ err = fmt.Errorf("content type %s can't be thumbnailed as an image", contentType)
}
if err != nil {
@@ -149,7 +169,7 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
size := thumbX * thumbY
aspect := float64(thumbX) / float64(thumbY)
- im := &imageMeta{
+ im := &mediaMeta{
width: thumbX,
height: thumbY,
size: size,
@@ -178,32 +198,3 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
return im, nil
}
-
-// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
-func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) {
- var i image.Image
- var err error
-
- switch contentType {
- case mimeImagePng:
- i, err = StrippedPngDecode(r)
- if err != nil {
- return nil, err
- }
- case mimeImageGif:
- i, err = gif.Decode(r)
- if err != nil {
- return nil, err
- }
- default:
- return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
- }
-
- out := &bytes.Buffer{}
- if err := png.Encode(out, i); err != nil {
- return nil, err
- }
- return &imageMeta{
- small: out.Bytes(),
- }, nil
-}
diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
index 659740af6..a8912bde0 100644
--- a/internal/media/manager_test.go
+++ b/internal/media/manager_test.go
@@ -376,6 +376,78 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
}
+func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
+ ctx := context.Background()
+
+ data := func(_ context.Context) (io.ReadCloser, int64, error) {
+ // load bytes from a test video
+ b, err := os.ReadFile("./test/test-mp4-original.mp4")
+ if err != nil {
+ panic(err)
+ }
+ return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+ }
+
+ accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+ // process the media with no additional info provided
+ processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+ suite.NoError(err)
+ // fetch the attachment id from the processing media
+ attachmentID := processingMedia.AttachmentID()
+
+ // do a blocking call to fetch the attachment
+ attachment, err := processingMedia.LoadAttachment(ctx)
+ suite.NoError(err)
+ suite.NotNil(attachment)
+
+ // make sure it's got the stuff set on it that we expect
+ // the attachment ID and accountID we expect
+ suite.Equal(attachmentID, attachment.ID)
+ suite.Equal(accountID, attachment.AccountID)
+
+ // file meta should be correctly derived from the video
+ suite.EqualValues(gtsmodel.Original{
+ Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
+ }, attachment.FileMeta.Original)
+ suite.EqualValues(gtsmodel.Small{
+ Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
+ }, attachment.FileMeta.Small)
+ suite.Equal("video/mp4", attachment.File.ContentType)
+ suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+ suite.Equal(312413, attachment.File.FileSize)
+ suite.Equal("", attachment.Blurhash)
+
+ // now make sure the attachment is in the database
+ dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
+ suite.NoError(err)
+ suite.NotNil(dbAttachment)
+
+ // make sure the processed file is in storage
+ processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
+ suite.NoError(err)
+ suite.NotEmpty(processedFullBytes)
+
+ // load the processed bytes from our test folder, to compare
+ processedFullBytesExpected, err := os.ReadFile("./test/test-mp4-processed.mp4")
+ suite.NoError(err)
+ suite.NotEmpty(processedFullBytesExpected)
+
+ // the bytes in storage should be what we expected
+ suite.Equal(processedFullBytesExpected, processedFullBytes)
+
+ // now do the same for the thumbnail and make sure it's what we expected
+ processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
+ suite.NoError(err)
+ suite.NotEmpty(processedThumbnailBytes)
+
+ processedThumbnailBytesExpected, err := os.ReadFile("./test/test-mp4-thumbnail.jpg")
+ suite.NoError(err)
+ suite.NotEmpty(processedThumbnailBytesExpected)
+
+ suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
+}
+
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
ctx := context.Background()
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index 94c8f9a7a..a7ea4dbab 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -88,11 +88,11 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
return nil, err
}
- if err := p.loadThumb(ctx); err != nil {
+ if err := p.loadFullSize(ctx); err != nil {
return nil, err
}
- if err := p.loadFullSize(ctx); err != nil {
+ if err := p.loadThumb(ctx); err != nil {
return nil, err
}
@@ -128,7 +128,6 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
switch processState(thumbState) {
case received:
// we haven't processed a thumbnail for this media yet so do it now
-
// check if we need to create a blurhash or if there's already one set
var createBlurhash bool
if p.attachment.Blurhash == "" {
@@ -136,28 +135,47 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
createBlurhash = true
}
- // stream the original file out of storage
- stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
- if err != nil {
- p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
- atomic.StoreInt32(&p.thumbState, int32(errored))
- return p.err
- }
- defer stored.Close()
+ var (
+ thumb *mediaMeta
+ err error
+ )
+ switch ct := p.attachment.File.ContentType; ct {
+ case mimeImageJpeg, mimeImagePng, mimeImageWebp, mimeImageGif:
+ // thumbnail the image from the original stored full size version
+ stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
+ if err != nil {
+ p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
+ atomic.StoreInt32(&p.thumbState, int32(errored))
+ return p.err
+ }
- // stream the file from storage straight into the derive thumbnail function
- thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash)
- if err != nil {
- p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
+ thumb, err = deriveThumbnailFromImage(stored, ct, createBlurhash)
+
+ // try to close the stored stream we had open, no matter what
+ if closeErr := stored.Close(); closeErr != nil {
+ log.Errorf("error closing stream: %s", closeErr)
+ }
+
+ // now check if we managed to get a thumbnail
+ if err != nil {
+ p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
+ atomic.StoreInt32(&p.thumbState, int32(errored))
+ return p.err
+ }
+ case mimeVideoMp4:
+ // create a generic thumbnail based on video height + width
+ thumb, err = deriveThumbnailFromVideo(p.attachment.FileMeta.Original.Height, p.attachment.FileMeta.Original.Width)
+ if err != nil {
+ p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
+ atomic.StoreInt32(&p.thumbState, int32(errored))
+ return p.err
+ }
+ default:
+ p.err = fmt.Errorf("loadThumb: content type %s not a processible image type", ct)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
- // Close stored media now we're done
- if err := stored.Close(); err != nil {
- log.Errorf("loadThumb: error closing stored full size: %s", err)
- }
-
// put the thumbnail in storage
if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil && err != storage.ErrAlreadyExists {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
@@ -195,7 +213,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
switch processState(fullSizeState) {
case received:
var err error
- var decoded *imageMeta
+ var decoded *mediaMeta
// stream the original file out of storage...
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
@@ -218,6 +236,8 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
decoded, err = decodeImage(stored, ct)
case mimeImageGif:
decoded, err = decodeGif(stored)
+ case mimeVideoMp4:
+ decoded, err = decodeVideo(stored, ct)
default:
err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct)
}
@@ -295,7 +315,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
}
// bail if this is a type we can't process
- if !supportedImage(contentType) {
+ if !supportedAttachment(contentType) {
return fmt.Errorf("store: media type %s not (yet) supported", contentType)
}
@@ -338,6 +358,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
// can't terminate if we don't know the file size, so just store the multiReader
readerToStore = multiReader
}
+ case mimeMp4:
+ p.attachment.Type = gtsmodel.FileTypeVideo
+ // nothing to terminate, we can just store the multireader
+ readerToStore = multiReader
default:
return fmt.Errorf("store: couldn't process %s", extension)
}
diff --git a/internal/media/test/test-mp4-original.mp4 b/internal/media/test/test-mp4-original.mp4
new file mode 100644
index 000000000..f78f51de6
--- /dev/null
+++ b/internal/media/test/test-mp4-original.mp4
Binary files differ
diff --git a/internal/media/test/test-mp4-processed.mp4 b/internal/media/test/test-mp4-processed.mp4
new file mode 100644
index 000000000..f78f51de6
--- /dev/null
+++ b/internal/media/test/test-mp4-processed.mp4
Binary files differ
diff --git a/internal/media/test/test-mp4-thumbnail.jpg b/internal/media/test/test-mp4-thumbnail.jpg
new file mode 100644
index 000000000..8bfdf1540
--- /dev/null
+++ b/internal/media/test/test-mp4-thumbnail.jpg
Binary files differ
diff --git a/internal/media/types.go b/internal/media/types.go
index b855d72b5..e7edfe643 100644
--- a/internal/media/types.go
+++ b/internal/media/types.go
@@ -34,6 +34,7 @@ const maxFileHeaderBytes = 261
// mime consts
const (
mimeImage = "image"
+ mimeVideo = "video"
mimeJpeg = "jpeg"
mimeImageJpeg = mimeImage + "/" + mimeJpeg
@@ -46,6 +47,9 @@ const (
mimeWebp = "webp"
mimeImageWebp = mimeImage + "/" + mimeWebp
+
+ mimeMp4 = "mp4"
+ mimeVideoMp4 = mimeVideo + "/" + mimeMp4
)
type processState int32
@@ -128,3 +132,12 @@ type DataFunc func(ctx context.Context) (reader io.ReadCloser, fileSize int64, e
//
// This can be set to nil, and will then not be executed.
type PostDataCallbackFunc func(ctx context.Context) error
+
+type mediaMeta struct {
+ width int
+ height int
+ size int
+ aspect float64
+ blurhash string
+ small []byte
+}
diff --git a/internal/media/util.go b/internal/media/util.go
index 60661cbc0..387f5d65a 100644
--- a/internal/media/util.go
+++ b/internal/media/util.go
@@ -37,6 +37,7 @@ func AllSupportedMIMETypes() []string {
mimeImageGif,
mimeImagePng,
mimeImageWebp,
+ mimeVideoMp4,
}
}
@@ -61,16 +62,10 @@ func parseContentType(fileHeader []byte) (string, error) {
return kind.MIME.Value, nil
}
-// supportedImage checks mime type of an image against a slice of accepted types,
-// and returns True if the mime type is accepted.
-func supportedImage(mimeType string) bool {
- acceptedImageTypes := []string{
- mimeImageJpeg,
- mimeImageGif,
- mimeImagePng,
- mimeImageWebp,
- }
- for _, accepted := range acceptedImageTypes {
+// supportedAttachment checks mime type of an attachment against a
+// slice of accepted types, and returns True if the mime type is accepted.
+func supportedAttachment(mimeType string) bool {
+ for _, accepted := range AllSupportedMIMETypes() {
if mimeType == accepted {
return true
}
diff --git a/internal/media/video.go b/internal/media/video.go
new file mode 100644
index 000000000..ef486d63d
--- /dev/null
+++ b/internal/media/video.go
@@ -0,0 +1,140 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "image"
+ "image/color"
+ "image/draw"
+ "image/jpeg"
+ "io"
+ "os"
+
+ "github.com/abema/go-mp4"
+ "github.com/superseriousbusiness/gotosocial/internal/log"
+)
+
+var thumbFill = color.RGBA{42, 43, 47, 0} // the color to fill video thumbnails with
+
+func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) {
+ // We'll need a readseeker to decode the video. We can get a readseeker
+ // without burning too much mem by first copying the reader into a temp file.
+ // First create the file in the temporary directory...
+ tempFile, err := os.CreateTemp(os.TempDir(), "gotosocial-")
+ if err != nil {
+ return nil, fmt.Errorf("could not create temporary file while decoding video: %w", err)
+ }
+ tempFileName := tempFile.Name()
+
+ // Make sure to clean up the temporary file when we're done with it
+ defer func() {
+ if err := tempFile.Close(); err != nil {
+ log.Errorf("could not close file %s: %s", tempFileName, err)
+ }
+ if err := os.Remove(tempFileName); err != nil {
+ log.Errorf("could not remove file %s: %s", tempFileName, err)
+ }
+ }()
+
+ // Now copy the entire reader we've been provided into the
+ // temporary file; we won't use the reader again after this.
+ if _, err := io.Copy(tempFile, r); err != nil {
+ return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err)
+ }
+
+ // define some vars we need to pull the width/height out of the video
+ var (
+ height int
+ width int
+ readHandler = getReadHandler(&height, &width)
+ )
+
+ // do the actual decoding here, providing the temporary file we created as readseeker
+ if _, err := mp4.ReadBoxStructure(tempFile, readHandler); err != nil {
+ return nil, fmt.Errorf("parsing video data: %w", err)
+ }
+
+ // width + height should now be updated by the readHandler
+ return &mediaMeta{
+ width: width,
+ height: height,
+ size: height * width,
+ aspect: float64(width) / float64(height),
+ }, nil
+}
+
+// getReadHandler returns a handler function that updates the underling
+// values of the given height and width int pointers to the hightest and
+// widest points of the video.
+func getReadHandler(height *int, width *int) func(h *mp4.ReadHandle) (interface{}, error) {
+ return func(rh *mp4.ReadHandle) (interface{}, error) {
+ if rh.BoxInfo.Type == mp4.BoxTypeTkhd() {
+ box, _, err := rh.ReadPayload()
+ if err != nil {
+ return nil, fmt.Errorf("could not read mp4 payload: %w", err)
+ }
+
+ tkhd, ok := box.(*mp4.Tkhd)
+ if !ok {
+ return nil, errors.New("box was not of type *mp4.Tkhd")
+ }
+
+ // if height + width of this box are greater than what
+ // we have stored, then update our stored values
+ if h := int(tkhd.GetHeight()); h > *height {
+ *height = h
+ }
+
+ if w := int(tkhd.GetWidth()); w > *width {
+ *width = w
+ }
+ }
+
+ if rh.BoxInfo.IsSupportedType() {
+ return rh.Expand()
+ }
+
+ return nil, nil
+ }
+}
+
+func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
+ // create a rectangle with the same dimensions as the video
+ img := image.NewRGBA(image.Rect(0, 0, width, height))
+
+ // fill the rectangle with our desired fill color
+ draw.Draw(img, img.Bounds(), &image.Uniform{thumbFill}, image.Point{}, draw.Src)
+
+ // we can get away with using extremely poor quality for this monocolor thumbnail
+ out := &bytes.Buffer{}
+ if err := jpeg.Encode(out, img, &jpeg.Options{Quality: 1}); err != nil {
+ return nil, fmt.Errorf("error encoding video thumbnail: %w", err)
+ }
+
+ return &mediaMeta{
+ width: width,
+ height: height,
+ size: width * height,
+ aspect: float64(width) / float64(height),
+ small: out.Bytes(),
+ }, nil
+}
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-13 05:46:39 +0000