diff options
Diffstat (limited to 'internal/media')
| -rw-r--r-- | internal/media/ffmpeg.go | 159 | ||||
| -rw-r--r-- | internal/media/manager_test.go | 50 | ||||
| -rw-r--r-- | internal/media/metadata.go | 9 | ||||
| -rw-r--r-- | internal/media/processingmedia.go | 65 | ||||
| -rw-r--r-- | internal/media/test/birdnest-thumbnail.webp | bin | 9446 -> 14478 bytes | |||
| -rw-r--r-- | internal/media/test/longer-mp4-thumbnail.webp | bin | 2128 -> 2976 bytes | |||
| -rw-r--r-- | internal/media/test/test-jpeg-thumbnail.jpeg | bin | 0 -> 22858 bytes | |||
| -rw-r--r-- | internal/media/test/test-jpeg-thumbnail.webp | bin | 8536 -> 0 bytes | |||
| -rw-r--r-- | internal/media/test/test-mp4-thumbnail.webp | bin | 3746 -> 5648 bytes | |||
| -rw-r--r-- | internal/media/test/test-png-alphachannel-thumbnail.jpeg | bin | 0 -> 3592 bytes | |||
| -rw-r--r-- | internal/media/test/test-png-alphachannel-thumbnail.webp | bin | 2630 -> 0 bytes | |||
| -rw-r--r-- | internal/media/test/test-png-noalphachannel-thumbnail.jpeg | bin | 0 -> 6446 bytes | |||
| -rw-r--r-- | internal/media/test/test-png-noalphachannel-thumbnail.webp | bin | 2630 -> 0 bytes | |||
| -rw-r--r-- | internal/media/thumbnail.go | 380 | ||||
| -rw-r--r-- | internal/media/util.go | 54 | 
15 files changed, 545 insertions, 172 deletions
diff --git a/internal/media/ffmpeg.go b/internal/media/ffmpeg.go index eb6dd9263..f6d74290c 100644 --- a/internal/media/ffmpeg.go +++ b/internal/media/ffmpeg.go @@ -66,26 +66,13 @@ func ffmpegClearMetadata(ctx context.Context, outpath, inpath string) error {  	)  } -// ffmpegGenerateThumb generates a thumbnail webp from input media of any type, useful for any media. -func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int) (string, error) { -	var outpath string - -	// Generate thumb output path REPLACING extension. -	if i := strings.IndexByte(filepath, '.'); i != -1 { -		outpath = filepath[:i] + "_thumb.webp" -	} else { -		return "", gtserror.New("input file missing extension") -	} - +// ffmpegGenerateWebpThumb generates a thumbnail webp from input media of any type, useful for any media. +func ffmpegGenerateWebpThumb(ctx context.Context, filepath, outpath string, width, height int, pixfmt string) error {  	// Get directory from filepath.  	dirpath := path.Dir(filepath) -	// Thumbnail size scaling argument. -	scale := strconv.Itoa(width) + ":" + -		strconv.Itoa(height) -  	// Generate thumb with ffmpeg. -	if err := ffmpeg(ctx, dirpath, +	return ffmpeg(ctx, dirpath,  		// Only log errors.  		"-loglevel", "error", @@ -97,36 +84,36 @@ func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int  		// (NOT as libwebp_anim).  		"-codec:v", "libwebp", -		// Select thumb from first 10 frames +		// Select thumb from first 7 frames. +		// (in particular <= 7 reduced memory usage, marginally)  		// (thumb filter: https://ffmpeg.org/ffmpeg-filters.html#thumbnail) -		"-filter:v", "thumbnail=n=10,"+ +		"-filter:v", "thumbnail=n=7,"+ -			// scale to dimensions +			// Scale to dimensions  			// (scale filter: https://ffmpeg.org/ffmpeg-filters.html#scale) -			"scale="+scale+","+ +			"scale="+strconv.Itoa(width)+ +			":"+strconv.Itoa(height)+","+ -			// YUVA 4:2:0 pixel format +			// Attempt to use original pixel format  			// (format filter: https://ffmpeg.org/ffmpeg-filters.html#format) -			"format=pix_fmts=yuva420p", +			"format=pix_fmts="+pixfmt,  		// Only one frame  		"-frames:v", "1", -		// ~40% webp quality +		// Quality not specified, +		// i.e. use default which +		// should be 75% webp quality.  		// (codec options: https://ffmpeg.org/ffmpeg-codecs.html#toc-Codec-Options)  		// (libwebp codec: https://ffmpeg.org/ffmpeg-codecs.html#Options-36) -		"-qscale:v", "40", +		// "-qscale:v", "75",  		// Overwrite.  		"-y",  		// Output.  		outpath, -	); err != nil { -		return "", err -	} - -	return outpath, nil +	)  }  // ffmpegGenerateStatic generates a static png from input image of any type, useful for emoji. @@ -219,12 +206,11 @@ func ffprobe(ctx context.Context, filepath string) (*result, error) {  			// Show specifically container format, total duration and bitrate.  			"-show_entries", "format=format_name,duration,bit_rate" + ":" + -				// Show specifically stream codec names, types, frame rate, duration and dimens. -				"stream=codec_name,codec_type,r_frame_rate,duration_ts,width,height" + ":" + +				// Show specifically stream codec names, types, frame rate, duration, dimens, and pixel format. +				"stream=codec_name,codec_type,r_frame_rate,duration_ts,width,height,pix_fmt" + ":" + -				// Show any rotation -				// side data stored. -				"side_data=rotation", +				// Show orientation. +				"tags=orientation",  			// Limit to reading the first  			// 1s of data looking for "rotation" @@ -262,15 +248,35 @@ func ffprobe(ctx context.Context, filepath string) (*result, error) {  	return res, nil  } +const ( +	// possible orientation values +	// specified in "orientation" +	// tag of images. +	// +	// FlipH      = flips horizontally +	// FlipV      = flips vertically +	// Transpose  = flips horizontally and rotates 90 counter-clockwise. +	// Transverse = flips vertically and rotates 90 counter-clockwise. +	orientationUnspecified = 0 +	orientationNormal      = 1 +	orientationFlipH       = 2 +	orientationRotate180   = 3 +	orientationFlipV       = 4 +	orientationTranspose   = 5 +	orientationRotate270   = 6 +	orientationTransverse  = 7 +	orientationRotate90    = 8 +) +  // result contains parsed ffprobe result  // data in a more useful data format.  type result struct { -	format   string -	audio    []audioStream -	video    []videoStream -	duration float64 -	bitrate  uint64 -	rotation int +	format      string +	audio       []audioStream +	video       []videoStream +	duration    float64 +	bitrate     uint64 +	orientation int  }  type stream struct { @@ -283,6 +289,7 @@ type audioStream struct {  type videoStream struct {  	stream +	pixfmt    string  	width     int  	height    int  	framerate float32 @@ -403,14 +410,28 @@ func (res *result) ImageMeta() (width int, height int, framerate float32) {  	// any odd multiples of 90,  	// flip width / height to  	// get the correct scale. -	switch res.rotation { -	case -90, 90, -270, 270: +	switch res.orientation { +	case orientationRotate90, +		orientationRotate270, +		orientationTransverse, +		orientationTranspose:  		width, height = height, width  	}  	return  } +// PixFmt returns the first valid pixel format +// contained among the result vidoe streams. +func (res *result) PixFmt() string { +	for _, str := range res.video { +		if str.pixfmt != "" { +			return str.pixfmt +		} +	} +	return "" +} +  // Process converts raw ffprobe result data into our more usable result{} type.  func (res *ffprobeResult) Process() (*result, error) {  	if res.Error != nil { @@ -446,37 +467,29 @@ func (res *ffprobeResult) Process() (*result, error) {  	// Check extra packet / frame information  	// for provided orientation (not always set).  	for _, pf := range res.PacketsAndFrames { -		for _, d := range pf.SideDataList { -			// Ensure frame side -			// data IS rotation data. -			if d.Rotation == 0 { -				continue -			} +		// Ensure frame contains tags. +		if pf.Tags.Orientation == "" { +			continue +		} -			// Ensure rotation not -			// already been specified. -			if r.rotation != 0 { -				return nil, errors.New("multiple sets of rotation data") -			} +		// Ensure orientation not +		// already been specified. +		if r.orientation != 0 { +			return nil, errors.New("multiple sets of orientation data") +		} -			// Drop any decimal -			// rotation value. -			rot := int(d.Rotation) +		// Trim any space from orientation value. +		str := strings.TrimSpace(pf.Tags.Orientation) -			// Round rotation to multiple of 90. -			// More granularity is not needed. -			if q := rot % 90; q > 45 { -				rot += (90 - q) -			} else { -				rot -= q -			} - -			// Drop any value above 360 -			// or below -360, these are -			// just repeat full turns. -			r.rotation = (rot % 360) +		// Parse as integer value. +		i, _ := strconv.Atoi(str) +		if i <= 0 || i >= 9 { +			return nil, errors.New("invalid orientation data")  		} + +		// Set orientation. +		r.orientation = i  	}  	// Preallocate streams to max possible lengths. @@ -519,6 +532,7 @@ func (res *ffprobeResult) Process() (*result, error) {  			// Append video stream data to result.  			r.video = append(r.video, videoStream{  				stream:    stream{codec: s.CodecName}, +				pixfmt:    s.PixFmt,  				width:     s.Width,  				height:    s.Height,  				framerate: framerate, @@ -539,17 +553,18 @@ type ffprobeResult struct {  }  type ffprobePacketOrFrame struct { -	Type         string            `json:"type"` -	SideDataList []ffprobeSideData `json:"side_data_list"` +	Type string      `json:"type"` +	Tags ffprobeTags `json:"tags"`  } -type ffprobeSideData struct { -	Rotation float64 `json:"rotation"` +type ffprobeTags struct { +	Orientation string `json:"orientation"`  }  type ffprobeStream struct {  	CodecName  string `json:"codec_name"`  	CodecType  string `json:"codec_type"` +	PixFmt     string `json:"pix_fmt"`  	RFrameRate string `json:"r_frame_rate"`  	DurationTS uint   `json:"duration_ts"`  	Width      int    `json:"width"` diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index d988ae274..87777ea30 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -273,10 +273,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcess() {  		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,  	}, attachment.FileMeta.Small)  	suite.Equal("image/jpeg", attachment.File.ContentType) -	suite.Equal("image/webp", attachment.Thumbnail.ContentType) +	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)  	suite.Equal(269739, attachment.File.FileSize) -	suite.Equal(8536, attachment.Thumbnail.FileSize) -	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash) +	suite.Equal(22858, attachment.Thumbnail.FileSize) +	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -285,7 +285,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcess() {  	// ensure the files contain the expected data.  	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg") -	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp") +	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpeg")  }  func (suite *ManagerTestSuite) TestSimpleJpegProcessTooLarge() { @@ -428,8 +428,8 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {  	suite.Equal("video/mp4", attachment.File.ContentType)  	suite.Equal("image/webp", attachment.Thumbnail.ContentType)  	suite.Equal(312453, attachment.File.FileSize) -	suite.Equal(3746, attachment.Thumbnail.FileSize) -	suite.Equal("LhIrNMt6Nsj[t7aybFj[_4WBspoe", attachment.Blurhash) +	suite.Equal(5648, attachment.Thumbnail.FileSize) +	suite.Equal("LhIrNMt6Nsj[t7ayW.j[_4WBsWkB", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -488,8 +488,8 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {  	suite.Equal("video/mp4", attachment.File.ContentType)  	suite.Equal("image/webp", attachment.Thumbnail.ContentType)  	suite.Equal(109569, attachment.File.FileSize) -	suite.Equal(2128, attachment.Thumbnail.FileSize) -	suite.Equal("L8Q0aP~qnM_3~qD%ozRjRiofWXRj", attachment.Blurhash) +	suite.Equal(2976, attachment.Thumbnail.FileSize) +	suite.Equal("L8QJfm~qD%_3_3D%t7RjM{j[ofRj", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -548,8 +548,8 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {  	suite.Equal("video/mp4", attachment.File.ContentType)  	suite.Equal("image/webp", attachment.Thumbnail.ContentType)  	suite.Equal(1409625, attachment.File.FileSize) -	suite.Equal(9446, attachment.Thumbnail.FileSize) -	suite.Equal("LKF~w1RjRO.99DRORPaetkV?WCMw", attachment.Blurhash) +	suite.Equal(14478, attachment.Thumbnail.FileSize) +	suite.Equal("LKF~w1RjRO.99DM_RPaetkV?WCMw", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -654,10 +654,10 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcess() {  		Width: 186, Height: 187, Size: 34782, Aspect: 0.9946524064171123,  	}, attachment.FileMeta.Small)  	suite.Equal("image/png", attachment.File.ContentType) -	suite.Equal("image/webp", attachment.Thumbnail.ContentType) +	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)  	suite.Equal(17471, attachment.File.FileSize) -	suite.Equal(2630, attachment.Thumbnail.FileSize) -	suite.Equal("LBOW$@%i-=aj%go#RSRP_1av~Tt2", attachment.Blurhash) +	suite.Equal(6446, attachment.Thumbnail.FileSize) +	suite.Equal("LDQcrD%i-?aj%ho#M~RP~nf3~nt2", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -666,7 +666,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcess() {  	// ensure the files contain the expected data.  	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-png-noalphachannel-processed.png") -	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-noalphachannel-thumbnail.webp") +	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-noalphachannel-thumbnail.jpeg")  }  func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() { @@ -712,8 +712,8 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() {  	suite.Equal("image/png", attachment.File.ContentType)  	suite.Equal("image/webp", attachment.Thumbnail.ContentType)  	suite.Equal(18832, attachment.File.FileSize) -	suite.Equal(2630, attachment.Thumbnail.FileSize) -	suite.Equal("LBOW$@%i-=aj%go#RSRP_1av~Tt2", attachment.Blurhash) +	suite.Equal(3592, attachment.Thumbnail.FileSize) +	suite.Equal("LBOW$@%i-rak%go#RSRP_1av~Ts+", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -722,7 +722,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() {  	// ensure the files contain the expected data.  	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-png-alphachannel-processed.png") -	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-alphachannel-thumbnail.webp") +	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-alphachannel-thumbnail.jpeg")  }  func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() { @@ -766,10 +766,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() {  		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,  	}, attachment.FileMeta.Small)  	suite.Equal("image/jpeg", attachment.File.ContentType) -	suite.Equal("image/webp", attachment.Thumbnail.ContentType) +	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)  	suite.Equal(269739, attachment.File.FileSize) -	suite.Equal(8536, attachment.Thumbnail.FileSize) -	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash) +	suite.Equal(22858, attachment.Thumbnail.FileSize) +	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -778,7 +778,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() {  	// ensure the files contain the expected data.  	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg") -	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp") +	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpeg")  }  func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() { @@ -844,10 +844,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() {  		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,  	}, attachment.FileMeta.Small)  	suite.Equal("image/jpeg", attachment.File.ContentType) -	suite.Equal("image/webp", attachment.Thumbnail.ContentType) +	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)  	suite.Equal(269739, attachment.File.FileSize) -	suite.Equal(8536, attachment.Thumbnail.FileSize) -	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash) +	suite.Equal(22858, attachment.Thumbnail.FileSize) +	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)  	// now make sure the attachment is in the database  	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID) @@ -856,7 +856,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() {  	// ensure the files contain the expected data.  	equalFiles(suite.T(), storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg") -	equalFiles(suite.T(), storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp") +	equalFiles(suite.T(), storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpeg")  }  func (suite *ManagerTestSuite) TestSmallSizedMediaTypeDetection_issue2263() { diff --git a/internal/media/metadata.go b/internal/media/metadata.go index cccfc8296..e9256f1b1 100644 --- a/internal/media/metadata.go +++ b/internal/media/metadata.go @@ -47,9 +47,14 @@ func clearMetadata(ctx context.Context, filepath string) error {  		// cleaning exif data using a native Go library.  		log.Debug(ctx, "cleaning with exif-terminator")  		err := terminateExif(outpath, filepath, ext) -		if err != nil { -			return err +		if err == nil { +			// No problem. +			break  		} + +		log.Warnf(ctx, "error cleaning with exif-terminator, falling back to ffmpeg: %v", err) +		fallthrough +  	default:  		// For all other types, best-effort clean with ffmpeg.  		log.Debug(ctx, "cleaning with ffmpeg -map_metadata -1") diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 1d286bda7..b89bbb41d 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -230,31 +230,26 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)  		p.media.FileMeta.Small.Aspect = aspect -		// Generate a thumbnail image from input image path. -		thumbpath, err = ffmpegGenerateThumb(ctx, temppath, +		// Determine if blurhash needs generating. +		needBlurhash := (p.media.Blurhash == "") +		var newBlurhash string + +		// Generate thumbnail, and new blurhash if need from media. +		thumbpath, newBlurhash, err = generateThumb(ctx, temppath,  			thumbWidth,  			thumbHeight, +			result.orientation, +			result.PixFmt(), +			needBlurhash,  		)  		if err != nil {  			return gtserror.Newf("error generating image thumb: %w", err)  		} -		if p.media.Blurhash == "" { -			// Generate blurhash (if not already) from thumbnail. -			p.media.Blurhash, err = generateBlurhash(thumbpath) -			if err != nil { -				return gtserror.Newf("error generating thumb blurhash: %w", err) -			} +		if needBlurhash { +			// Set newly determined blurhash. +			p.media.Blurhash = newBlurhash  		} - -		// Calculate final media attachment thumbnail path. -		p.media.Thumbnail.Path = uris.StoragePathForAttachment( -			p.media.AccountID, -			string(TypeAttachment), -			string(SizeSmall), -			p.media.ID, -			"webp", -		)  	}  	// Calculate final media attachment file path. @@ -279,6 +274,18 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  	p.media.File.FileSize = int(filesz)  	if thumbpath != "" { +		// Determine final thumbnail ext. +		thumbExt := getExtension(thumbpath) + +		// Calculate final media attachment thumbnail path. +		p.media.Thumbnail.Path = uris.StoragePathForAttachment( +			p.media.AccountID, +			string(TypeAttachment), +			string(SizeSmall), +			p.media.ID, +			thumbExt, +		) +  		// Copy thumbnail file into storage at path.  		thumbsz, err := p.mgr.state.Storage.PutFile(ctx,  			p.media.Thumbnail.Path, @@ -290,6 +297,18 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		// Set final determined thumbnail size.  		p.media.Thumbnail.FileSize = int(thumbsz) + +		// Determine thumbnail content-type from thumb ext. +		p.media.Thumbnail.ContentType = getMimeType(thumbExt) + +		// Generate a media attachment thumbnail URL. +		p.media.Thumbnail.URL = uris.URIForAttachment( +			p.media.AccountID, +			string(TypeAttachment), +			string(SizeSmall), +			p.media.ID, +			thumbExt, +		)  	}  	// Generate a media attachment URL. @@ -301,22 +320,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {  		ext,  	) -	// Generate a media attachment thumbnail URL. -	p.media.Thumbnail.URL = uris.URIForAttachment( -		p.media.AccountID, -		string(TypeAttachment), -		string(SizeSmall), -		p.media.ID, -		"webp", -	) -  	// Get mimetype for the file container  	// type, falling back to generic data.  	p.media.File.ContentType = getMimeType(ext) -	// Set the known thumbnail content type. -	p.media.Thumbnail.ContentType = "image/webp" -  	// We can now consider this cached.  	p.media.Cached = util.Ptr(true) diff --git a/internal/media/test/birdnest-thumbnail.webp b/internal/media/test/birdnest-thumbnail.webp Binary files differindex 882e813b6..d59e5c26e 100644 --- a/internal/media/test/birdnest-thumbnail.webp +++ b/internal/media/test/birdnest-thumbnail.webp diff --git a/internal/media/test/longer-mp4-thumbnail.webp b/internal/media/test/longer-mp4-thumbnail.webp Binary files differindex 4406f7f46..a7527c1ec 100644 --- a/internal/media/test/longer-mp4-thumbnail.webp +++ b/internal/media/test/longer-mp4-thumbnail.webp diff --git a/internal/media/test/test-jpeg-thumbnail.jpeg b/internal/media/test/test-jpeg-thumbnail.jpeg Binary files differnew file mode 100644 index 000000000..80170e7c8 --- /dev/null +++ b/internal/media/test/test-jpeg-thumbnail.jpeg diff --git a/internal/media/test/test-jpeg-thumbnail.webp b/internal/media/test/test-jpeg-thumbnail.webp Binary files differdeleted file mode 100644 index 5bc741037..000000000 --- a/internal/media/test/test-jpeg-thumbnail.webp +++ /dev/null diff --git a/internal/media/test/test-mp4-thumbnail.webp b/internal/media/test/test-mp4-thumbnail.webp Binary files differindex 7041837bf..8b28714c6 100644 --- a/internal/media/test/test-mp4-thumbnail.webp +++ b/internal/media/test/test-mp4-thumbnail.webp diff --git a/internal/media/test/test-png-alphachannel-thumbnail.jpeg b/internal/media/test/test-png-alphachannel-thumbnail.jpeg Binary files differnew file mode 100644 index 000000000..b70613f0b --- /dev/null +++ b/internal/media/test/test-png-alphachannel-thumbnail.jpeg diff --git a/internal/media/test/test-png-alphachannel-thumbnail.webp b/internal/media/test/test-png-alphachannel-thumbnail.webp Binary files differdeleted file mode 100644 index d78c45433..000000000 --- a/internal/media/test/test-png-alphachannel-thumbnail.webp +++ /dev/null diff --git a/internal/media/test/test-png-noalphachannel-thumbnail.jpeg b/internal/media/test/test-png-noalphachannel-thumbnail.jpeg Binary files differnew file mode 100644 index 000000000..ca62f4ea6 --- /dev/null +++ b/internal/media/test/test-png-noalphachannel-thumbnail.jpeg diff --git a/internal/media/test/test-png-noalphachannel-thumbnail.webp b/internal/media/test/test-png-noalphachannel-thumbnail.webp Binary files differdeleted file mode 100644 index d78c45433..000000000 --- a/internal/media/test/test-png-noalphachannel-thumbnail.webp +++ /dev/null diff --git a/internal/media/thumbnail.go b/internal/media/thumbnail.go new file mode 100644 index 000000000..36ef24a01 --- /dev/null +++ b/internal/media/thumbnail.go @@ -0,0 +1,380 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program.  If not, see <http://www.gnu.org/licenses/>. + +package media + +import ( +	"context" +	"image" +	"image/gif" +	"image/jpeg" +	"image/png" +	"io" +	"os" +	"strings" + +	"github.com/buckket/go-blurhash" +	"github.com/disintegration/imaging" +	"github.com/superseriousbusiness/gotosocial/internal/gtserror" +	"github.com/superseriousbusiness/gotosocial/internal/log" +	"golang.org/x/image/webp" +) + +// generateThumb generates a thumbnail for the +// input file at path, resizing it to the given +// dimensions and generating a blurhash if needed. +// This wraps much of the complex thumbnailing +// logic in which where possible we use native +// Go libraries for generating thumbnails, else +// always falling back to slower but much more +// widely supportive ffmpeg. +func generateThumb( +	ctx context.Context, +	filepath string, +	width, height int, +	orientation int, +	pixfmt string, +	needBlurhash bool, +) ( +	outpath string, +	blurhash string, +	err error, +) { +	var ext string + +	// Generate thumb output path REPLACING extension. +	if i := strings.IndexByte(filepath, '.'); i != -1 { +		outpath = filepath[:i] + "_thumb.webp" +		ext = filepath[i+1:] // old extension +	} else { +		return "", "", gtserror.New("input file missing extension") +	} + +	// Check for the few media types we +	// have native Go decoding that allow +	// us to generate thumbs natively. +	switch { + +	case ext == "jpeg": +		// Replace the "webp" with "jpeg", as we'll +		// use our native Go thumbnailing generation. +		outpath = outpath[:len(outpath)-4] + "jpeg" + +		log.Debug(ctx, "generating thumb from jpeg") +		blurhash, err := generateNativeThumb( +			filepath, +			outpath, +			width, +			height, +			orientation, +			jpeg.Decode, +			needBlurhash, +		) +		return outpath, blurhash, err + +	// We specifically only allow generating native +	// thumbnails from gif IF it doesn't contain an +	// alpha channel. We'll ultimately be encoding to +	// jpeg which doesn't support transparency layers. +	case ext == "gif" && !containsAlpha(pixfmt): + +		// Replace the "webp" with "jpeg", as we'll +		// use our native Go thumbnailing generation. +		outpath = outpath[:len(outpath)-4] + "jpeg" + +		log.Debug(ctx, "generating thumb from gif") +		blurhash, err := generateNativeThumb( +			filepath, +			outpath, +			width, +			height, +			orientation, +			gif.Decode, +			needBlurhash, +		) +		return outpath, blurhash, err + +	// We specifically only allow generating native +	// thumbnails from png IF it doesn't contain an +	// alpha channel. We'll ultimately be encoding to +	// jpeg which doesn't support transparency layers. +	case ext == "png" && !containsAlpha(pixfmt): + +		// Replace the "webp" with "jpeg", as we'll +		// use our native Go thumbnailing generation. +		outpath = outpath[:len(outpath)-4] + "jpeg" + +		log.Debug(ctx, "generating thumb from png") +		blurhash, err := generateNativeThumb( +			filepath, +			outpath, +			width, +			height, +			orientation, +			png.Decode, +			needBlurhash, +		) +		return outpath, blurhash, err + +	// We specifically only allow generating native +	// thumbnails from webp IF it doesn't contain an +	// alpha channel. We'll ultimately be encoding to +	// jpeg which doesn't support transparency layers. +	case ext == "webp" && !containsAlpha(pixfmt): + +		// Replace the "webp" with "jpeg", as we'll +		// use our native Go thumbnailing generation. +		outpath = outpath[:len(outpath)-4] + "jpeg" + +		log.Debug(ctx, "generating thumb from webp") +		blurhash, err := generateNativeThumb( +			filepath, +			outpath, +			width, +			height, +			orientation, +			webp.Decode, +			needBlurhash, +		) +		return outpath, blurhash, err +	} + +	// The fallback for thumbnail generation, which +	// encompasses most media types is with ffmpeg. +	log.Debug(ctx, "generating thumb with ffmpeg") +	if err := ffmpegGenerateWebpThumb(ctx, +		filepath, +		outpath, +		width, +		height, +		pixfmt, +	); err != nil { +		return outpath, "", err +	} + +	if needBlurhash { +		// Generate new blurhash from webp output thumb. +		blurhash, err = generateWebpBlurhash(outpath) +		if err != nil { +			return outpath, "", gtserror.Newf("error generating blurhash: %w", err) +		} +	} + +	return outpath, blurhash, err +} + +// generateNativeThumb generates a thumbnail +// using native Go code, using given decode +// function to get image, resize to given dimens, +// and write to output filepath as JPEG. If a +// blurhash is required it will also generate +// this from the image.Image while in-memory. +func generateNativeThumb( +	inpath, outpath string, +	width, height int, +	orientation int, +	decode func(io.Reader) (image.Image, error), +	needBlurhash bool, +) ( +	string, // blurhash +	error, +) { +	// Open input file at given path. +	infile, err := os.Open(inpath) +	if err != nil { +		return "", gtserror.Newf("error opening input file %s: %w", inpath, err) +	} + +	// Decode image into memory. +	img, err := decode(infile) + +	// Done with file. +	_ = infile.Close() + +	if err != nil { +		return "", gtserror.Newf("error decoding file %s: %w", inpath, err) +	} + +	// Apply orientation BEFORE any resize, +	// as our image dimensions are calculated +	// taking orientation into account. +	switch orientation { +	case orientationFlipH: +		img = imaging.FlipH(img) +	case orientationFlipV: +		img = imaging.FlipV(img) +	case orientationRotate90: +		img = imaging.Rotate90(img) +	case orientationRotate180: +		img = imaging.Rotate180(img) +	case orientationRotate270: +		img = imaging.Rotate270(img) +	case orientationTranspose: +		img = imaging.Transpose(img) +	case orientationTransverse: +		img = imaging.Transverse(img) +	} + +	// Resize image to dimens. +	img = imaging.Resize(img, +		width, height, +		imaging.Linear, +	) + +	// Open output file at given path. +	outfile, err := os.Create(outpath) +	if err != nil { +		return "", gtserror.Newf("error opening output file %s: %w", outpath, err) +	} + +	// Encode in-memory image to output file. +	// (nil uses defaults, i.e. quality=75). +	err = jpeg.Encode(outfile, img, nil) + +	// Done with file. +	_ = outfile.Close() + +	if err != nil { +		return "", gtserror.Newf("error encoding image: %w", err) +	} + +	if needBlurhash { +		// for generating blurhashes, it's more cost effective to +		// lose detail since it's blurry, so make a tiny version. +		tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor) + +		// Drop the larger image +		// ref as soon as possible +		// to allow GC to claim. +		img = nil //nolint + +		// Generate blurhash for the tiny thumbnail. +		blurhash, err := blurhash.Encode(4, 3, tiny) +		if err != nil { +			return "", gtserror.Newf("error generating blurhash: %w", err) +		} + +		return blurhash, nil +	} + +	return "", nil +} + +// generateWebpBlurhash generates a blurhash for Webp at filepath. +func generateWebpBlurhash(filepath string) (string, error) { +	// Open the file at given path. +	file, err := os.Open(filepath) +	if err != nil { +		return "", gtserror.Newf("error opening input file %s: %w", filepath, err) +	} + +	// Decode image from file. +	img, err := webp.Decode(file) + +	// Done with file. +	_ = file.Close() + +	if err != nil { +		return "", gtserror.Newf("error decoding file %s: %w", filepath, err) +	} + +	// for generating blurhashes, it's more cost effective to +	// lose detail since it's blurry, so make a tiny version. +	tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor) + +	// Drop the larger image +	// ref as soon as possible +	// to allow GC to claim. +	img = nil //nolint + +	// Generate blurhash for the tiny thumbnail. +	blurhash, err := blurhash.Encode(4, 3, tiny) +	if err != nil { +		return "", gtserror.Newf("error generating blurhash: %w", err) +	} + +	return blurhash, nil +} + +// List of pixel formats that have an alpha layer. +// Derived from the following very messy command: +// +//	for res in $(ffprobe -show_entries pixel_format=name:flags=alpha | grep -B1 alpha=1 | grep name); do echo $res | sed 's/name=//g' | sed 's/^/"/g' | sed 's/$/",/g'; done +var alphaPixelFormats = []string{ +	"pal8", +	"argb", +	"rgba", +	"abgr", +	"bgra", +	"yuva420p", +	"ya8", +	"yuva422p", +	"yuva444p", +	"yuva420p9be", +	"yuva420p9le", +	"yuva422p9be", +	"yuva422p9le", +	"yuva444p9be", +	"yuva444p9le", +	"yuva420p10be", +	"yuva420p10le", +	"yuva422p10be", +	"yuva422p10le", +	"yuva444p10be", +	"yuva444p10le", +	"yuva420p16be", +	"yuva420p16le", +	"yuva422p16be", +	"yuva422p16le", +	"yuva444p16be", +	"yuva444p16le", +	"rgba64be", +	"rgba64le", +	"bgra64be", +	"bgra64le", +	"ya16be", +	"ya16le", +	"gbrap", +	"gbrap16be", +	"gbrap16le", +	"ayuv64le", +	"ayuv64be", +	"gbrap12be", +	"gbrap12le", +	"gbrap10be", +	"gbrap10le", +	"gbrapf32be", +	"gbrapf32le", +	"yuva422p12be", +	"yuva422p12le", +	"yuva444p12be", +	"yuva444p12le", +} + +// containsAlpha returns whether given pixfmt +// (i.e. colorspace) contains an alpha channel. +func containsAlpha(pixfmt string) bool { +	if pixfmt == "" { +		return false +	} +	for _, checkfmt := range alphaPixelFormats { +		if pixfmt == checkfmt { +			return true +		} +	} +	return false +} diff --git a/internal/media/util.go b/internal/media/util.go index fa170965f..17d396a0b 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -21,20 +21,24 @@ import (  	"cmp"  	"errors"  	"fmt" -	"image"  	"io"  	"os" -	"golang.org/x/image/webp" -  	"codeberg.org/gruf/go-bytesize"  	"codeberg.org/gruf/go-iotools"  	"codeberg.org/gruf/go-mimetypes" - -	"github.com/buckket/go-blurhash" -	"github.com/disintegration/imaging"  ) +// getExtension splits file extension from path. +func getExtension(path string) string { +	for i := len(path) - 1; i >= 0 && path[i] != '/'; i-- { +		if path[i] == '.' { +			return path[i+1:] +		} +	} +	return "" +} +  // thumbSize returns the dimensions to use for an input  // image of given width / height, for its outgoing thumbnail.  // This attempts to maintains the original image aspect ratio. @@ -68,44 +72,6 @@ func thumbSize(width, height int, aspect float32) (int, int) {  	}  } -// webpDecode decodes the WebP at filepath into parsed image.Image. -func webpDecode(filepath string) (image.Image, error) { -	// Open the file at given path. -	file, err := os.Open(filepath) -	if err != nil { -		return nil, err -	} - -	// Decode image from file. -	img, err := webp.Decode(file) - -	// Done with file. -	_ = file.Close() - -	return img, err -} - -// generateBlurhash generates a blurhash for JPEG at filepath. -func generateBlurhash(filepath string) (string, error) { -	// Decode JPEG file at given path. -	img, err := webpDecode(filepath) -	if err != nil { -		return "", err -	} - -	// for generating blurhashes, it's more cost effective to -	// lose detail since it's blurry, so make a tiny version. -	tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor) - -	// Drop the larger image -	// ref as soon as possible -	// to allow GC to claim. -	img = nil //nolint - -	// Generate blurhash for thumbnail. -	return blurhash.Encode(4, 3, tiny) -} -  // getMimeType returns a suitable mimetype for file extension.  func getMimeType(ext string) string {  	const defaultType = "application/octet-stream"  | 
