From 6803c1682b26a0d78326dbe45b06f61ac0476d0d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 27 Dec 2021 18:03:36 +0100 Subject: start refactor of media package --- internal/media/handler.go | 53 ++----- internal/media/image.go | 133 +++++++++++++++++ internal/media/processicon.go | 6 +- internal/media/processimage.go | 133 ----------------- internal/media/processing.go | 190 ++++++++++++++++++++++++ internal/media/types.go | 149 +++++++++++++++++++ internal/media/util.go | 322 ----------------------------------------- internal/media/util_test.go | 4 +- 8 files changed, 489 insertions(+), 501 deletions(-) create mode 100644 internal/media/image.go delete mode 100644 internal/media/processimage.go create mode 100644 internal/media/processing.go create mode 100644 internal/media/types.go delete mode 100644 internal/media/util.go diff --git a/internal/media/handler.go b/internal/media/handler.go index e6c7369b6..b64e583b3 100644 --- a/internal/media/handler.go +++ b/internal/media/handler.go @@ -35,45 +35,16 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/uris" ) -// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) -const EmojiMaxBytes = 51200 -type Size string -const ( - SizeSmall Size = "small" // SizeSmall is the key for small/thumbnail versions of media - SizeOriginal Size = "original" // SizeOriginal is the key for original/fullsize versions of media and emoji - SizeStatic Size = "static" // SizeStatic is the key for static (non-animated) versions of emoji -) - -type Type string - -const ( - TypeAttachment Type = "attachment" // TypeAttachment is the key for media attachments - TypeHeader Type = "header" // TypeHeader is the key for profile header requests - TypeAvatar Type = "avatar" // TypeAvatar is the key for profile avatar requests - TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests -) +type ProcessedCallback func(*gtsmodel.MediaAttachment) error // Handler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs. type Handler interface { - // ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, - // puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, - // and then returns information to the caller about the new header. - ProcessHeaderOrAvatar(ctx context.Context, attachment []byte, accountID string, mediaType Type, remoteURL string) (*gtsmodel.MediaAttachment, error) - - // ProcessLocalAttachment takes a new attachment and the requesting account, checks it out, removes exif data from it, - // puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media, - // and then returns information to the caller about the attachment. It's the caller's responsibility to put the returned struct - // in the database. - ProcessAttachment(ctx context.Context, attachmentBytes []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) - - // ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new - // *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct - // in the database. - ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) - - ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) + ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) + ProcessAvatar(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) + ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) + ProcessEmoji(ctx context.Context, data []byte, shortcode string) (*gtsmodel.Emoji, error) } type mediaHandler struct { @@ -108,7 +79,7 @@ func (mh *mediaHandler) ProcessHeaderOrAvatar(ctx context.Context, attachment [] if err != nil { return nil, err } - if !SupportedImageType(contentType) { + if !supportedImage(contentType) { return nil, fmt.Errorf("%s is not an accepted image type", contentType) } @@ -152,8 +123,8 @@ func (mh *mediaHandler) ProcessAttachment(ctx context.Context, attachmentBytes [ // return nil, errors.New("video was of size 0") // } // return mh.processVideoAttachment(attachment, accountID, contentType, remoteURL) - case MIMEImage: - if !SupportedImageType(contentType) { + case mimeImage: + if !supportedImage(contentType) { return nil, fmt.Errorf("image type %s not supported", contentType) } if len(attachmentBytes) == 0 { @@ -180,7 +151,7 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte if err != nil { return nil, err } - if !supportedEmojiType(contentType) { + if !supportedEmoji(contentType) { return nil, fmt.Errorf("content type %s not supported for emojis", contentType) } @@ -193,11 +164,11 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte // clean any exif data from png but leave gifs alone switch contentType { - case MIMEPng: + case mimePng: if clean, err = purgeExif(emojiBytes); err != nil { return nil, fmt.Errorf("error cleaning exif data: %s", err) } - case MIMEGif: + case mimeGif: clean = emojiBytes default: return nil, errors.New("media type unrecognized") @@ -266,7 +237,7 @@ func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte ImagePath: emojiPath, ImageStaticPath: emojiStaticPath, ImageContentType: contentType, - ImageStaticContentType: MIMEPng, // static version will always be a png + ImageStaticContentType: mimePng, // static version will always be a png ImageFileSize: len(original.image), ImageStaticFileSize: len(static.image), ImageUpdatedAt: time.Now(), diff --git a/internal/media/image.go b/internal/media/image.go new file mode 100644 index 000000000..f1cc03bb6 --- /dev/null +++ b/internal/media/image.go @@ -0,0 +1,133 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { + var clean []byte + var err error + var original *imageAndMeta + var small *imageAndMeta + + contentType := minAttachment.File.ContentType + + switch contentType { + case mimeJpeg, mimePng: + if clean, err = purgeExif(data); err != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", err) + } + original, err = deriveImage(clean, contentType) + if err != nil { + return nil, fmt.Errorf("error parsing image: %s", err) + } + case mimeGif: + clean = data + original, err = deriveGif(clean, contentType) + if err != nil { + return nil, fmt.Errorf("error parsing gif: %s", err) + } + default: + return nil, errors.New("media type unrecognized") + } + + small, err = deriveThumbnail(clean, contentType, 512, 512) + if err != nil { + return nil, fmt.Errorf("error deriving thumbnail: %s", err) + } + + // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it + extension := strings.Split(contentType, "/")[1] + newMediaID, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), newMediaID, extension) + smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), newMediaID, "jpeg") // all thumbnails/smalls are encoded as jpeg + + // we store the original... + originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, newMediaID, extension) + if err := mh.storage.Put(originalPath, original.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + // and a thumbnail... + smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, newMediaID) // all thumbnails/smalls are encoded as jpeg + if err := mh.storage.Put(smallPath, small.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + minAttachment.FileMeta.Original = gtsmodel.Original{ + Width: original.width, + Height: original.height, + Size: original.size, + Aspect: original.aspect, + } + + minAttachment.FileMeta.Small = gtsmodel.Small{ + Width: small.width, + Height: small.height, + Size: small.size, + Aspect: small.aspect, + } + + attachment := >smodel.MediaAttachment{ + ID: newMediaID, + StatusID: minAttachment.StatusID, + URL: originalURL, + RemoteURL: minAttachment.RemoteURL, + CreatedAt: minAttachment.CreatedAt, + UpdatedAt: minAttachment.UpdatedAt, + Type: gtsmodel.FileTypeImage, + FileMeta: minAttachment.FileMeta, + AccountID: minAttachment.AccountID, + Description: minAttachment.Description, + ScheduledStatusID: minAttachment.ScheduledStatusID, + Blurhash: small.blurhash, + Processing: 2, + File: gtsmodel.File{ + Path: originalPath, + ContentType: contentType, + FileSize: len(original.image), + UpdatedAt: time.Now(), + }, + Thumbnail: gtsmodel.Thumbnail{ + Path: smallPath, + ContentType: mimeJpeg, // all thumbnails/smalls are encoded as jpeg + FileSize: len(small.image), + UpdatedAt: time.Now(), + URL: smallURL, + RemoteURL: minAttachment.Thumbnail.RemoteURL, + }, + Avatar: minAttachment.Avatar, + Header: minAttachment.Header, + } + + return attachment, nil +} diff --git a/internal/media/processicon.go b/internal/media/processicon.go index 66cf1f999..faeae0ee6 100644 --- a/internal/media/processicon.go +++ b/internal/media/processicon.go @@ -47,17 +47,17 @@ func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string var original *imageAndMeta switch contentType { - case MIMEJpeg: + case mimeJpeg: if clean, err = purgeExif(imageBytes); err != nil { return nil, fmt.Errorf("error cleaning exif data: %s", err) } original, err = deriveImage(clean, contentType) - case MIMEPng: + case mimePng: if clean, err = purgeExif(imageBytes); err != nil { return nil, fmt.Errorf("error cleaning exif data: %s", err) } original, err = deriveImage(clean, contentType) - case MIMEGif: + case mimeGif: clean = imageBytes original, err = deriveGif(clean, contentType) default: diff --git a/internal/media/processimage.go b/internal/media/processimage.go deleted file mode 100644 index ca92c0660..000000000 --- a/internal/media/processimage.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -func (mh *mediaHandler) processImageAttachment(data []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - var clean []byte - var err error - var original *imageAndMeta - var small *imageAndMeta - - contentType := minAttachment.File.ContentType - - switch contentType { - case MIMEJpeg, MIMEPng: - if clean, err = purgeExif(data); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - original, err = deriveImage(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing image: %s", err) - } - case MIMEGif: - clean = data - original, err = deriveGif(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing gif: %s", err) - } - default: - return nil, errors.New("media type unrecognized") - } - - small, err = deriveThumbnail(clean, contentType, 512, 512) - if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) - } - - // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - extension := strings.Split(contentType, "/")[1] - newMediaID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), newMediaID, extension) - smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), newMediaID, "jpeg") // all thumbnails/smalls are encoded as jpeg - - // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, newMediaID, extension) - if err := mh.storage.Put(originalPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, newMediaID) // all thumbnails/smalls are encoded as jpeg - if err := mh.storage.Put(smallPath, small.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - minAttachment.FileMeta.Original = gtsmodel.Original{ - Width: original.width, - Height: original.height, - Size: original.size, - Aspect: original.aspect, - } - - minAttachment.FileMeta.Small = gtsmodel.Small{ - Width: small.width, - Height: small.height, - Size: small.size, - Aspect: small.aspect, - } - - attachment := >smodel.MediaAttachment{ - ID: newMediaID, - StatusID: minAttachment.StatusID, - URL: originalURL, - RemoteURL: minAttachment.RemoteURL, - CreatedAt: minAttachment.CreatedAt, - UpdatedAt: minAttachment.UpdatedAt, - Type: gtsmodel.FileTypeImage, - FileMeta: minAttachment.FileMeta, - AccountID: minAttachment.AccountID, - Description: minAttachment.Description, - ScheduledStatusID: minAttachment.ScheduledStatusID, - Blurhash: small.blurhash, - Processing: 2, - File: gtsmodel.File{ - Path: originalPath, - ContentType: contentType, - FileSize: len(original.image), - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - Path: smallPath, - ContentType: MIMEJpeg, // all thumbnails/smalls are encoded as jpeg - FileSize: len(small.image), - UpdatedAt: time.Now(), - URL: smallURL, - RemoteURL: minAttachment.Thumbnail.RemoteURL, - }, - Avatar: minAttachment.Avatar, - Header: minAttachment.Header, - } - - return attachment, nil -} diff --git a/internal/media/processing.go b/internal/media/processing.go new file mode 100644 index 000000000..ccd9ebfdb --- /dev/null +++ b/internal/media/processing.go @@ -0,0 +1,190 @@ +package media + +import ( + "bytes" + "errors" + "fmt" + "image" + "image/gif" + "image/jpeg" + "image/png" + + "github.com/buckket/go-blurhash" + "github.com/nfnt/resize" + "github.com/superseriousbusiness/exifremove/pkg/exifremove" +) + +// purgeExif is a little wrapper for the action of removing exif data from an image. +// Only pass pngs or jpegs to this function. +func purgeExif(data []byte) ([]byte, error) { + if len(data) == 0 { + return nil, errors.New("passed image was not valid") + } + + clean, err := exifremove.Remove(data) + if err != nil { + return nil, fmt.Errorf("could not purge exif from image: %s", err) + } + + if len(clean) == 0 { + return nil, errors.New("purged image was not valid") + } + + return clean, nil +} + +func deriveGif(b []byte, extension string) (*imageAndMeta, error) { + var g *gif.GIF + var err error + switch extension { + case mimeGif: + g, err = gif.DecodeAll(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("extension %s not recognised", extension) + } + + // use the first frame to get the static characteristics + width := g.Config.Width + height := g.Config.Height + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +func deriveImage(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not recognised", contentType) + } + + width := i.Bounds().Size().X + height := i.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, +// of a given jpeg, png, or gif, or an error if something goes wrong. +// +// Note that the aspect ratio of the image will be retained, +// so it will not necessarily be a square, even if x and y are set as the same value. +func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not recognised", contentType) + } + + thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) + width := thumb.Bounds().Size().X + height := thumb.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) + bh, err := blurhash.Encode(4, 3, tiny) + if err != nil { + return nil, err + } + + out := &bytes.Buffer{} + if err := jpeg.Encode(out, thumb, &jpeg.Options{ + Quality: 75, + }); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + width: width, + height: height, + size: size, + aspect: aspect, + blurhash: bh, + }, nil +} + +// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. +func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) + } + + out := &bytes.Buffer{} + if err := png.Encode(out, i); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + }, nil +} + +type imageAndMeta struct { + image []byte + width int + height int + size int + aspect float64 + blurhash string +} diff --git a/internal/media/types.go b/internal/media/types.go new file mode 100644 index 000000000..f1608f880 --- /dev/null +++ b/internal/media/types.go @@ -0,0 +1,149 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "bytes" + "errors" + "fmt" + + "github.com/h2non/filetype" +) + +// mime consts +const ( + mimeImage = "image" + + mimeJpeg = "jpeg" + mimeImageJpeg = mimeImage + "/" + mimeJpeg + + mimeGif = "gif" + mimeImageGif = mimeImage + "/" + mimeGif + + mimePng = "png" + mimeImagePng = mimeImage + "/" + mimePng +) + + +// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) +// const EmojiMaxBytes = 51200 + +// maxFileHeaderBytes represents the maximum amount of bytes we want +// to examine from the beginning of a file to determine its type. +// +// See: https://en.wikipedia.org/wiki/File_format#File_header +// and https://github.com/h2non/filetype +const maxFileHeaderBytes = 262 + +type Size string + +const ( + SizeSmall Size = "small" // SizeSmall is the key for small/thumbnail versions of media + SizeOriginal Size = "original" // SizeOriginal is the key for original/fullsize versions of media and emoji + SizeStatic Size = "static" // SizeStatic is the key for static (non-animated) versions of emoji +) + +type Type string + +const ( + TypeAttachment Type = "attachment" // TypeAttachment is the key for media attachments + TypeHeader Type = "header" // TypeHeader is the key for profile header requests + TypeAvatar Type = "avatar" // TypeAvatar is the key for profile avatar requests + TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests +) + +// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). +// Returns an error if the content type is not something we can process. +func parseContentType(content []byte) (string, error) { + + // read in the first bytes of the file + fileHeader := make([]byte, maxFileHeaderBytes) + if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { + return "", fmt.Errorf("could not read first magic bytes of file: %s", err) + } + + kind, err := filetype.Match(fileHeader) + if err != nil { + return "", err + } + + if kind == filetype.Unknown { + return "", errors.New("filetype unknown") + } + + return kind.MIME.Value, nil +} + +// supportedImage checks mime type of an image against a slice of accepted types, +// and returns True if the mime type is accepted. +func supportedImage(mimeType string) bool { + acceptedImageTypes := []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedImageTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. +func supportedEmoji(mimeType string) bool { + acceptedEmojiTypes := []string{ + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedEmojiTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized +func ParseMediaType(s string) (Type, error) { + switch s { + case string(TypeAttachment): + return TypeAttachment, nil + case string(TypeHeader): + return TypeHeader, nil + case string(TypeAvatar): + return TypeAvatar, nil + case string(TypeEmoji): + return TypeEmoji, nil + } + return "", fmt.Errorf("%s not a recognized MediaType", s) +} + +// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized +func ParseMediaSize(s string) (Size, error) { + switch s { + case string(SizeSmall): + return SizeSmall, nil + case string(SizeOriginal): + return SizeOriginal, nil + case string(SizeStatic): + return SizeStatic, nil + } + return "", fmt.Errorf("%s not a recognized MediaSize", s) +} diff --git a/internal/media/util.go b/internal/media/util.go deleted file mode 100644 index 348136c92..000000000 --- a/internal/media/util.go +++ /dev/null @@ -1,322 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "bytes" - "errors" - "fmt" - "image" - "image/gif" - "image/jpeg" - "image/png" - - "github.com/buckket/go-blurhash" - "github.com/h2non/filetype" - "github.com/nfnt/resize" - "github.com/superseriousbusiness/exifremove/pkg/exifremove" -) - -const ( - // MIMEImage is the mime type for image - MIMEImage = "image" - // MIMEJpeg is the jpeg image mime type - MIMEJpeg = "image/jpeg" - // MIMEGif is the gif image mime type - MIMEGif = "image/gif" - // MIMEPng is the png image mime type - MIMEPng = "image/png" - - // MIMEVideo is the mime type for video - MIMEVideo = "video" - // MIMEMp4 is the mp4 video mime type - MIMEMp4 = "video/mp4" - // MIMEMpeg is the mpeg video mime type - MIMEMpeg = "video/mpeg" - // MIMEWebm is the webm video mime type - MIMEWebm = "video/webm" -) - -// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). -// Returns an error if the content type is not something we can process. -func parseContentType(content []byte) (string, error) { - head := make([]byte, 261) - _, err := bytes.NewReader(content).Read(head) - if err != nil { - return "", fmt.Errorf("could not read first magic bytes of file: %s", err) - } - - kind, err := filetype.Match(head) - if err != nil { - return "", err - } - - if kind == filetype.Unknown { - return "", errors.New("filetype unknown") - } - - return kind.MIME.Value, nil -} - -// SupportedImageType checks mime type of an image against a slice of accepted types, -// and returns True if the mime type is accepted. -func SupportedImageType(mimeType string) bool { - acceptedImageTypes := []string{ - MIMEJpeg, - MIMEGif, - MIMEPng, - } - for _, accepted := range acceptedImageTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// SupportedVideoType checks mime type of a video against a slice of accepted types, -// and returns True if the mime type is accepted. -func SupportedVideoType(mimeType string) bool { - acceptedVideoTypes := []string{ - MIMEMp4, - MIMEMpeg, - MIMEWebm, - } - for _, accepted := range acceptedVideoTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// supportedEmojiType checks that the content type is image/png -- the only type supported for emoji. -func supportedEmojiType(mimeType string) bool { - acceptedEmojiTypes := []string{ - MIMEGif, - MIMEPng, - } - for _, accepted := range acceptedEmojiTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// purgeExif is a little wrapper for the action of removing exif data from an image. -// Only pass pngs or jpegs to this function. -func purgeExif(b []byte) ([]byte, error) { - if len(b) == 0 { - return nil, errors.New("passed image was not valid") - } - - clean, err := exifremove.Remove(b) - if err != nil { - return nil, fmt.Errorf("could not purge exif from image: %s", err) - } - if len(clean) == 0 { - return nil, errors.New("purged image was not valid") - } - return clean, nil -} - -func deriveGif(b []byte, extension string) (*imageAndMeta, error) { - var g *gif.GIF - var err error - switch extension { - case MIMEGif: - g, err = gif.DecodeAll(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("extension %s not recognised", extension) - } - - // use the first frame to get the static characteristics - width := g.Config.Width - height := g.Config.Height - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -func deriveImage(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case MIMEJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case MIMEPng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - width := i.Bounds().Size().X - height := i.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, -// of a given jpeg, png, or gif, or an error if something goes wrong. -// -// Note that the aspect ratio of the image will be retained, -// so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case MIMEJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case MIMEPng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case MIMEGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) - width := thumb.Bounds().Size().X - height := thumb.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, err - } - - out := &bytes.Buffer{} - if err := jpeg.Encode(out, thumb, &jpeg.Options{ - Quality: 75, - }); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - width: width, - height: height, - size: size, - aspect: aspect, - blurhash: bh, - }, nil -} - -// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case MIMEPng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case MIMEGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) - } - - out := &bytes.Buffer{} - if err := png.Encode(out, i); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - }, nil -} - -type imageAndMeta struct { - image []byte - width int - height int - size int - aspect float64 - blurhash string -} - -// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized -func ParseMediaType(s string) (Type, error) { - switch s { - case string(TypeAttachment): - return TypeAttachment, nil - case string(TypeHeader): - return TypeHeader, nil - case string(TypeAvatar): - return TypeAvatar, nil - case string(TypeEmoji): - return TypeEmoji, nil - } - return "", fmt.Errorf("%s not a recognized MediaType", s) -} - -// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized -func ParseMediaSize(s string) (Size, error) { - switch s { - case string(SizeSmall): - return SizeSmall, nil - case string(SizeOriginal): - return SizeOriginal, nil - case string(SizeStatic): - return SizeStatic, nil - } - return "", fmt.Errorf("%s not a recognized MediaSize", s) -} diff --git a/internal/media/util_test.go b/internal/media/util_test.go index cb299d50e..817b597cb 100644 --- a/internal/media/util_test.go +++ b/internal/media/util_test.go @@ -138,10 +138,10 @@ func (suite *MediaUtilTestSuite) TestDeriveThumbnailFromJPEG() { } func (suite *MediaUtilTestSuite) TestSupportedImageTypes() { - ok := SupportedImageType("image/jpeg") + ok := supportedImage("image/jpeg") suite.True(ok) - ok = SupportedImageType("image/bmp") + ok = supportedImage("image/bmp") suite.False(ok) } -- cgit v1.3 From c4d63d125b5a44c150a00b0b20b3638cad9221f8 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 28 Dec 2021 16:36:00 +0100 Subject: more refactoring, media handler => manager --- cmd/gotosocial/action/server/server.go | 6 +- internal/api/client/fileserver/servefile_test.go | 4 +- internal/api/client/media/mediacreate_test.go | 4 +- internal/api/s2s/webfinger/webfingerget_test.go | 4 +- internal/federation/dereferencing/account.go | 4 +- internal/federation/dereferencing/attachment.go | 2 +- internal/federation/dereferencing/dereferencer.go | 6 +- .../federation/dereferencing/dereferencer_test.go | 2 +- internal/federation/federator.go | 8 +- internal/federation/federator_test.go | 4 +- internal/media/handler.go | 289 --------------------- internal/media/image.go | 241 +++++++++++++++-- internal/media/manager.go | 274 +++++++++++++++++++ internal/media/processicon.go | 143 ---------- internal/media/processing.go | 190 -------------- internal/media/processvideo.go | 2 +- internal/media/types.go | 1 - internal/media/util_test.go | 2 +- internal/processing/account/account.go | 6 +- internal/processing/account/account_test.go | 6 +- internal/processing/account/update.go | 4 +- internal/processing/admin/admin.go | 6 +- internal/processing/admin/emoji.go | 4 +- internal/processing/media/create.go | 4 +- internal/processing/media/media.go | 6 +- internal/processing/processor.go | 12 +- internal/processing/processor_test.go | 6 +- testrig/federator.go | 2 +- testrig/mediahandler.go | 4 +- testrig/processor.go | 2 +- 30 files changed, 545 insertions(+), 703 deletions(-) delete mode 100644 internal/media/handler.go create mode 100644 internal/media/manager.go delete mode 100644 internal/media/processicon.go delete mode 100644 internal/media/processing.go diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index d55f38fc5..05c2e8974 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,10 +105,10 @@ var Start action.GTSAction = func(ctx context.Context) error { } // build backend handlers - mediaHandler := media.New(dbService, storage) + mediaManager := media.New(dbService, storage) oauthServer := oauth.New(ctx, dbService) transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient) - federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaHandler) + federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) // decide whether to create a noop email sender (won't send emails) or a real one var emailSender email.Sender @@ -128,7 +128,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } // create and start the message processor using the other services we've created so far - processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, timelineManager, dbService, emailSender) + processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, timelineManager, dbService, emailSender) if err := processor.Start(ctx); err != nil { return fmt.Errorf("error starting processor: %s", err) } diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index 18f54542a..109ed4eba 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -51,7 +51,7 @@ type ServeFileTestSuite struct { federator federation.Federator tc typeutils.TypeConverter processor processing.Processor - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender @@ -82,7 +82,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) // setup module being tested diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 2897d786b..72a377c25 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -53,7 +53,7 @@ type MediaCreateTestSuite struct { storage *kv.KVStore federator federation.Federator tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender processor processing.Processor @@ -81,7 +81,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go index 4b27ada42..c5df1f7e5 100644 --- a/internal/api/s2s/webfinger/webfingerget_test.go +++ b/internal/api/s2s/webfinger/webfingerget_test.go @@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() { func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() @@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index d06ad21c1..19c98e203 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -246,7 +246,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ + a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ RemoteURL: targetAccount.AvatarRemoteURL, Avatar: true, }, targetAccount.ID) @@ -257,7 +257,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ + a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ RemoteURL: targetAccount.HeaderRemoteURL, Header: true, }, targetAccount.ID) diff --git a/internal/federation/dereferencing/attachment.go b/internal/federation/dereferencing/attachment.go index 0c7005e23..30ab6da10 100644 --- a/internal/federation/dereferencing/attachment.go +++ b/internal/federation/dereferencing/attachment.go @@ -93,7 +93,7 @@ func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) } - a, err := d.mediaHandler.ProcessAttachment(ctx, attachmentBytes, minAttachment) + a, err := d.mediaManager.ProcessAttachment(ctx, attachmentBytes, minAttachment) if err != nil { return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index d0b653920..4f977b8c8 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -80,18 +80,18 @@ type deref struct { db db.DB typeConverter typeutils.TypeConverter transportController transport.Controller - mediaHandler media.Handler + mediaManager media.Manager handshakes map[string][]*url.URL handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewDereferencer returns a Dereferencer initialized with the given parameters. -func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler) Dereferencer { +func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer { return &deref{ db: db, typeConverter: typeConverter, transportController: transportController, - mediaHandler: mediaHandler, + mediaManager: mediaManager, handshakeSync: &sync.Mutex{}, } } diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index 569e8e93b..fe66abce4 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -64,7 +64,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaHandler(suite.db, suite.storage)) + suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 0a82f12bc..7edff1118 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -78,13 +78,13 @@ type federator struct { typeConverter typeutils.TypeConverter transportController transport.Controller dereferencer dereferencing.Dereferencer - mediaHandler media.Handler + mediaManager media.Manager actor pub.FederatingActor } // NewFederator returns a new federator -func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator { - dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaHandler) +func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaManager media.Manager) Federator { + dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaManager) clock := &Clock{} f := &federator{ @@ -94,7 +94,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr typeConverter: typeConverter, transportController: transportController, dereferencer: dereferencer, - mediaHandler: mediaHandler, + mediaManager: mediaManager, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 43f4904a5..6dac76c05 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -78,7 +78,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { return nil, nil }), suite.db) // setup module being tested - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) // setup request ctx := context.Background() @@ -107,7 +107,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) // now setup module being tested, with the mock transport controller - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil) // we need these headers for the request to be validated diff --git a/internal/media/handler.go b/internal/media/handler.go deleted file mode 100644 index b64e583b3..000000000 --- a/internal/media/handler.go +++ /dev/null @@ -1,289 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "context" - "errors" - "fmt" - "net/url" - "strings" - "time" - - "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - - - -type ProcessedCallback func(*gtsmodel.MediaAttachment) error - -// Handler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs. -type Handler interface { - ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessAvatar(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessedCallback) (*gtsmodel.MediaAttachment, error) - ProcessEmoji(ctx context.Context, data []byte, shortcode string) (*gtsmodel.Emoji, error) -} - -type mediaHandler struct { - db db.DB - storage *kv.KVStore -} - -// New returns a new handler with the given db and storage -func New(database db.DB, storage *kv.KVStore) Handler { - return &mediaHandler{ - db: database, - storage: storage, - } -} - -/* - INTERFACE FUNCTIONS -*/ - -// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, -// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, -// and then returns information to the caller about the new header. -func (mh *mediaHandler) ProcessHeaderOrAvatar(ctx context.Context, attachment []byte, accountID string, mediaType Type, remoteURL string) (*gtsmodel.MediaAttachment, error) { - l := logrus.WithField("func", "SetHeaderForAccountID") - - if mediaType != TypeHeader && mediaType != TypeAvatar { - return nil, errors.New("header or avatar not selected") - } - - // make sure we have a type we can handle - contentType, err := parseContentType(attachment) - if err != nil { - return nil, err - } - if !supportedImage(contentType) { - return nil, fmt.Errorf("%s is not an accepted image type", contentType) - } - - if len(attachment) == 0 { - return nil, fmt.Errorf("passed reader was of size 0") - } - l.Tracef("read %d bytes of file", len(attachment)) - - // process it - ma, err := mh.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL) - if err != nil { - return nil, fmt.Errorf("error processing %s: %s", mediaType, err) - } - - // set it in the database - if err := mh.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil { - return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err) - } - - return ma, nil -} - -// ProcessAttachment takes a new attachment and the owning account, checks it out, removes exif data from it, -// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media, -// and then returns information to the caller about the attachment. -func (mh *mediaHandler) ProcessAttachment(ctx context.Context, attachmentBytes []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - contentType, err := parseContentType(attachmentBytes) - if err != nil { - return nil, err - } - - minAttachment.File.ContentType = contentType - - mainType := strings.Split(contentType, "/")[0] - switch mainType { - // case MIMEVideo: - // if !SupportedVideoType(contentType) { - // return nil, fmt.Errorf("video type %s not supported", contentType) - // } - // if len(attachment) == 0 { - // return nil, errors.New("video was of size 0") - // } - // return mh.processVideoAttachment(attachment, accountID, contentType, remoteURL) - case mimeImage: - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - if len(attachmentBytes) == 0 { - return nil, errors.New("image was of size 0") - } - return mh.processImageAttachment(attachmentBytes, minAttachment) - default: - break - } - return nil, fmt.Errorf("content type %s not (yet) supported", contentType) -} - -// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new -// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct -// in the database. -func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) { - var clean []byte - var err error - var original *imageAndMeta - var static *imageAndMeta - - // check content type of the submitted emoji and make sure it's supported by us - contentType, err := parseContentType(emojiBytes) - if err != nil { - return nil, err - } - if !supportedEmoji(contentType) { - return nil, fmt.Errorf("content type %s not supported for emojis", contentType) - } - - if len(emojiBytes) == 0 { - return nil, errors.New("emoji was of size 0") - } - if len(emojiBytes) > EmojiMaxBytes { - return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) - } - - // clean any exif data from png but leave gifs alone - switch contentType { - case mimePng: - if clean, err = purgeExif(emojiBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - case mimeGif: - clean = emojiBytes - default: - return nil, errors.New("media type unrecognized") - } - - // unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc - original = &imageAndMeta{ - image: clean, - } - - static, err = deriveStaticEmoji(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error deriving static emoji: %s", err) - } - - // since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver, - // (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created - // with the same username as the instance hostname, which doesn't belong to any particular user. - instanceAccount, err := mh.db.GetInstanceAccount(ctx, "") - if err != nil { - return nil, fmt.Errorf("error fetching instance account: %s", err) - } - - // the file extension (either png or gif) - extension := strings.Split(contentType, "/")[1] - - // generate a ulid for the new emoji - newEmojiID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - // activitypub uri for the emoji -- unrelated to actually serving the image - // will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC - emojiURI := uris.GenerateURIForEmoji(newEmojiID) - - // serve url and storage path for the original emoji -- can be png or gif - emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension) - emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension) - - // serve url and storage path for the static version -- will always be png - emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png") - emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID) - - // Store the original emoji - if err := mh.storage.Put(emojiPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // Store the static emoji - if err := mh.storage.Put(emojiStaticPath, static.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and finally return the new emoji data to the caller -- it's up to them what to do with it - e := >smodel.Emoji{ - ID: newEmojiID, - Shortcode: shortcode, - Domain: "", // empty because this is a local emoji - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - ImageRemoteURL: "", // empty because this is a local emoji - ImageStaticRemoteURL: "", // empty because this is a local emoji - ImageURL: emojiURL, - ImageStaticURL: emojiStaticURL, - ImagePath: emojiPath, - ImageStaticPath: emojiStaticPath, - ImageContentType: contentType, - ImageStaticContentType: mimePng, // static version will always be a png - ImageFileSize: len(original.image), - ImageStaticFileSize: len(static.image), - ImageUpdatedAt: time.Now(), - Disabled: false, - URI: emojiURI, - VisibleInPicker: true, - CategoryID: "", // empty because this is a new emoji -- no category yet - } - return e, nil -} - -func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { - if !currentAttachment.Header && !currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to neither header nor avatar") - } - - if currentAttachment.Header && currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to both header and avatar") - } - - var headerOrAvi Type - if currentAttachment.Header { - headerOrAvi = TypeHeader - } else if currentAttachment.Avatar { - headerOrAvi = TypeAvatar - } - - if currentAttachment.RemoteURL == "" { - return nil, errors.New("no remote URL on media attachment to dereference") - } - remoteIRI, err := url.Parse(currentAttachment.RemoteURL) - if err != nil { - return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) - } - - // for content type, we assume we don't know what to expect... - expectedContentType := "*/*" - if currentAttachment.File.ContentType != "" { - // ... and then narrow it down if we do - expectedContentType = currentAttachment.File.ContentType - } - - attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType) - if err != nil { - return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) - } - - return mh.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL) -} diff --git a/internal/media/image.go b/internal/media/image.go index f1cc03bb6..87b5d70b7 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -19,67 +19,88 @@ package media import ( + "bytes" "errors" "fmt" + "image" + "image/gif" + "image/jpeg" + "image/png" "strings" "time" + "github.com/buckket/go-blurhash" + "github.com/nfnt/resize" + "github.com/superseriousbusiness/exifremove/pkg/exifremove" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/uris" ) -func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { +const ( + thumbnailMaxWidth = 512 + thumbnailMaxHeight = 512 +) + +type imageAndMeta struct { + image []byte + width int + height int + size int + aspect float64 + blurhash string +} + +func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.MediaAttachment, error) { var clean []byte var err error var original *imageAndMeta var small *imageAndMeta - contentType := minAttachment.File.ContentType - switch contentType { - case mimeJpeg, mimePng: - if clean, err = purgeExif(data); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(data); exifErr != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) } - original, err = deriveImage(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing image: %s", err) - } - case mimeGif: + original, err = decodeImage(clean, contentType) + case mimeImageGif: + // gifs are already clean - no exif data to remove clean = data - original, err = deriveGif(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error parsing gif: %s", err) - } + original, err = decodeGif(clean, contentType) default: - return nil, errors.New("media type unrecognized") + err = fmt.Errorf("content type %s not a recognized image type", contentType) + } + + if err != nil { + return nil, err } - small, err = deriveThumbnail(clean, contentType, 512, 512) + small, err = deriveThumbnail(clean, contentType, thumbnailMaxWidth, thumbnailMaxHeight) if err != nil { return nil, fmt.Errorf("error deriving thumbnail: %s", err) } // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it extension := strings.Split(contentType, "/")[1] - newMediaID, err := id.NewRandomULID() + attachmentID, err := id.NewRandomULID() if err != nil { return nil, err } - originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), newMediaID, extension) - smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), newMediaID, "jpeg") // all thumbnails/smalls are encoded as jpeg + originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), attachmentID, extension) + smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), attachmentID, "jpeg") // all thumbnails/smalls are encoded as jpeg // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, newMediaID, extension) - if err := mh.storage.Put(originalPath, original.image); err != nil { + originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, attachmentID, extension) + if err := m.storage.Put(originalPath, original.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, newMediaID) // all thumbnails/smalls are encoded as jpeg - if err := mh.storage.Put(smallPath, small.image); err != nil { + smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, attachmentID) // all thumbnails/smalls are encoded as jpeg + if err := m.storage.Put(smallPath, small.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } @@ -98,7 +119,7 @@ func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaA } attachment := >smodel.MediaAttachment{ - ID: newMediaID, + ID: attachmentID, StatusID: minAttachment.StatusID, URL: originalURL, RemoteURL: minAttachment.RemoteURL, @@ -131,3 +152,173 @@ func (mh *mediaHandler) processImage(data []byte, minAttachment *gtsmodel.MediaA return attachment, nil } + +func decodeGif(b []byte, extension string) (*imageAndMeta, error) { + var g *gif.GIF + var err error + + switch extension { + case mimeGif: + g, err = gif.DecodeAll(bytes.NewReader(b)) + default: + err = fmt.Errorf("extension %s not recognised", extension) + } + + if err != nil { + return nil, err + } + + // use the first frame to get the static characteristics + width := g.Config.Width + height := g.Config.Height + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +func decodeImage(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + default: + err = fmt.Errorf("content type %s not recognised", contentType) + } + + if err != nil { + return nil, err + } + + if i == nil { + return nil, errors.New("processed image was nil") + } + + width := i.Bounds().Size().X + height := i.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + return &imageAndMeta{ + image: b, + width: width, + height: height, + size: size, + aspect: aspect, + }, nil +} + +// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, +// of a given jpeg, png, or gif, or an error if something goes wrong. +// +// Note that the aspect ratio of the image will be retained, +// so it will not necessarily be a square, even if x and y are set as the same value. +func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImageJpeg: + i, err = jpeg.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not recognised", contentType) + } + + thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) + width := thumb.Bounds().Size().X + height := thumb.Bounds().Size().Y + size := width * height + aspect := float64(width) / float64(height) + + tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) + bh, err := blurhash.Encode(4, 3, tiny) + if err != nil { + return nil, err + } + + out := &bytes.Buffer{} + if err := jpeg.Encode(out, thumb, &jpeg.Options{ + Quality: 75, + }); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + width: width, + height: height, + size: size, + aspect: aspect, + blurhash: bh, + }, nil +} + +// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. +func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { + var i image.Image + var err error + + switch contentType { + case mimeImagePng: + i, err = png.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + case mimeImageGif: + i, err = gif.Decode(bytes.NewReader(b)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) + } + + out := &bytes.Buffer{} + if err := png.Encode(out, i); err != nil { + return nil, err + } + return &imageAndMeta{ + image: out.Bytes(), + }, nil +} + +// purgeExif is a little wrapper for the action of removing exif data from an image. +// Only pass pngs or jpegs to this function. +func purgeExif(data []byte) ([]byte, error) { + if len(data) == 0 { + return nil, errors.New("passed image was not valid") + } + + clean, err := exifremove.Remove(data) + if err != nil { + return nil, fmt.Errorf("could not purge exif from image: %s", err) + } + + if len(clean) == 0 { + return nil, errors.New("purged image was not valid") + } + + return clean, nil +} diff --git a/internal/media/manager.go b/internal/media/manager.go new file mode 100644 index 000000000..782542ca9 --- /dev/null +++ b/internal/media/manager.go @@ -0,0 +1,274 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +// ProcessCallback is triggered by the media manager when an attachment has finished undergoing +// image processing (generation of a blurhash, thumbnail etc) but hasn't yet been inserted into +// the database. It is provided to allow callers to a) access the processed media attachment and b) +// make any last-minute changes to the media attachment before it enters the database. +type ProcessCallback func(*gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment + +// defaultCB will be used when a nil ProcessCallback is passed to one of the manager's interface functions. +// It just returns the processed media attachment with no additional changes. +var defaultCB ProcessCallback = func(a *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment { + return a +} + +// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. +type Manager interface { + ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) +} + +type manager struct { + db db.DB + storage *kv.KVStore +} + +// New returns a media manager with the given db and underlying storage. +func New(database db.DB, storage *kv.KVStore) Manager { + return &manager{ + db: database, + storage: storage, + } +} + +/* + INTERFACE FUNCTIONS +*/ + +func (m *manager) ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { + contentType, err := parseContentType(data) + if err != nil { + return nil, err + } + + mainType := strings.Split(contentType, "/")[0] + switch mainType { + case mimeImage: + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + return m.processImage(attachmentBytes, minAttachment) + default: + return nil, fmt.Errorf("content type %s not (yet) supported", contentType) + } +} + +// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, +// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, +// and then returns information to the caller about the new header. +func (m *manager) ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { + + // make sure we have a type we can handle + contentType, err := parseContentType(data) + if err != nil { + return nil, err + } + + if !supportedImage(contentType) { + return nil, fmt.Errorf("%s is not an accepted image type", contentType) + } + + if len(data) == 0 { + return nil, fmt.Errorf("passed reader was of size 0") + } + + // process it + ma, err := m.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL) + if err != nil { + return nil, fmt.Errorf("error processing %s: %s", mediaType, err) + } + + // set it in the database + if err := m.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil { + return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err) + } + + return ma, nil +} + +// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new +// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct +// in the database. +func (m *manager) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) { + var clean []byte + var err error + var original *imageAndMeta + var static *imageAndMeta + + // check content type of the submitted emoji and make sure it's supported by us + contentType, err := parseContentType(emojiBytes) + if err != nil { + return nil, err + } + if !supportedEmoji(contentType) { + return nil, fmt.Errorf("content type %s not supported for emojis", contentType) + } + + if len(emojiBytes) == 0 { + return nil, errors.New("emoji was of size 0") + } + if len(emojiBytes) > EmojiMaxBytes { + return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) + } + + // clean any exif data from png but leave gifs alone + switch contentType { + case mimePng: + if clean, err = purgeExif(emojiBytes); err != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", err) + } + case mimeGif: + clean = emojiBytes + default: + return nil, errors.New("media type unrecognized") + } + + // unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc + original = &imageAndMeta{ + image: clean, + } + + static, err = deriveStaticEmoji(clean, contentType) + if err != nil { + return nil, fmt.Errorf("error deriving static emoji: %s", err) + } + + // since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver, + // (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created + // with the same username as the instance hostname, which doesn't belong to any particular user. + instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + if err != nil { + return nil, fmt.Errorf("error fetching instance account: %s", err) + } + + // the file extension (either png or gif) + extension := strings.Split(contentType, "/")[1] + + // generate a ulid for the new emoji + newEmojiID, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + // activitypub uri for the emoji -- unrelated to actually serving the image + // will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC + emojiURI := uris.GenerateURIForEmoji(newEmojiID) + + // serve url and storage path for the original emoji -- can be png or gif + emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension) + emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension) + + // serve url and storage path for the static version -- will always be png + emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png") + emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID) + + // Store the original emoji + if err := m.storage.Put(emojiPath, original.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + // Store the static emoji + if err := m.storage.Put(emojiStaticPath, static.image); err != nil { + return nil, fmt.Errorf("storage error: %s", err) + } + + // and finally return the new emoji data to the caller -- it's up to them what to do with it + e := >smodel.Emoji{ + ID: newEmojiID, + Shortcode: shortcode, + Domain: "", // empty because this is a local emoji + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + ImageRemoteURL: "", // empty because this is a local emoji + ImageStaticRemoteURL: "", // empty because this is a local emoji + ImageURL: emojiURL, + ImageStaticURL: emojiStaticURL, + ImagePath: emojiPath, + ImageStaticPath: emojiStaticPath, + ImageContentType: contentType, + ImageStaticContentType: mimePng, // static version will always be a png + ImageFileSize: len(original.image), + ImageStaticFileSize: len(static.image), + ImageUpdatedAt: time.Now(), + Disabled: false, + URI: emojiURI, + VisibleInPicker: true, + CategoryID: "", // empty because this is a new emoji -- no category yet + } + return e, nil +} + +func (m *manager) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { + if !currentAttachment.Header && !currentAttachment.Avatar { + return nil, errors.New("provided attachment was set to neither header nor avatar") + } + + if currentAttachment.Header && currentAttachment.Avatar { + return nil, errors.New("provided attachment was set to both header and avatar") + } + + var headerOrAvi Type + if currentAttachment.Header { + headerOrAvi = TypeHeader + } else if currentAttachment.Avatar { + headerOrAvi = TypeAvatar + } + + if currentAttachment.RemoteURL == "" { + return nil, errors.New("no remote URL on media attachment to dereference") + } + remoteIRI, err := url.Parse(currentAttachment.RemoteURL) + if err != nil { + return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) + } + + // for content type, we assume we don't know what to expect... + expectedContentType := "*/*" + if currentAttachment.File.ContentType != "" { + // ... and then narrow it down if we do + expectedContentType = currentAttachment.File.ContentType + } + + attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType) + if err != nil { + return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) + } + + return m.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL) +} diff --git a/internal/media/processicon.go b/internal/media/processicon.go deleted file mode 100644 index faeae0ee6..000000000 --- a/internal/media/processicon.go +++ /dev/null @@ -1,143 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" -) - -func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string, mediaType Type, accountID string, remoteURL string) (*gtsmodel.MediaAttachment, error) { - var isHeader bool - var isAvatar bool - - switch mediaType { - case TypeHeader: - isHeader = true - case TypeAvatar: - isAvatar = true - default: - return nil, errors.New("header or avatar not selected") - } - - var clean []byte - var err error - - var original *imageAndMeta - switch contentType { - case mimeJpeg: - if clean, err = purgeExif(imageBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - original, err = deriveImage(clean, contentType) - case mimePng: - if clean, err = purgeExif(imageBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - original, err = deriveImage(clean, contentType) - case mimeGif: - clean = imageBytes - original, err = deriveGif(clean, contentType) - default: - return nil, errors.New("media type unrecognized") - } - - if err != nil { - return nil, fmt.Errorf("error parsing image: %s", err) - } - - small, err := deriveThumbnail(clean, contentType, 256, 256) - if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) - } - - // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - extension := strings.Split(contentType, "/")[1] - newMediaID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - originalURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeOriginal), newMediaID, extension) - smallURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeSmall), newMediaID, extension) - // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeOriginal, newMediaID, extension) - if err := mh.storage.Put(originalPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeSmall, newMediaID, extension) - if err := mh.storage.Put(smallPath, small.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - ma := >smodel.MediaAttachment{ - ID: newMediaID, - StatusID: "", - URL: originalURL, - RemoteURL: remoteURL, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Original: gtsmodel.Original{ - Width: original.width, - Height: original.height, - Size: original.size, - Aspect: original.aspect, - }, - Small: gtsmodel.Small{ - Width: small.width, - Height: small.height, - Size: small.size, - Aspect: small.aspect, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: small.blurhash, - Processing: 2, - File: gtsmodel.File{ - Path: originalPath, - ContentType: contentType, - FileSize: len(original.image), - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - Path: smallPath, - ContentType: contentType, - FileSize: len(small.image), - UpdatedAt: time.Now(), - URL: smallURL, - RemoteURL: "", - }, - Avatar: isAvatar, - Header: isHeader, - } - - return ma, nil -} diff --git a/internal/media/processing.go b/internal/media/processing.go deleted file mode 100644 index ccd9ebfdb..000000000 --- a/internal/media/processing.go +++ /dev/null @@ -1,190 +0,0 @@ -package media - -import ( - "bytes" - "errors" - "fmt" - "image" - "image/gif" - "image/jpeg" - "image/png" - - "github.com/buckket/go-blurhash" - "github.com/nfnt/resize" - "github.com/superseriousbusiness/exifremove/pkg/exifremove" -) - -// purgeExif is a little wrapper for the action of removing exif data from an image. -// Only pass pngs or jpegs to this function. -func purgeExif(data []byte) ([]byte, error) { - if len(data) == 0 { - return nil, errors.New("passed image was not valid") - } - - clean, err := exifremove.Remove(data) - if err != nil { - return nil, fmt.Errorf("could not purge exif from image: %s", err) - } - - if len(clean) == 0 { - return nil, errors.New("purged image was not valid") - } - - return clean, nil -} - -func deriveGif(b []byte, extension string) (*imageAndMeta, error) { - var g *gif.GIF - var err error - switch extension { - case mimeGif: - g, err = gif.DecodeAll(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("extension %s not recognised", extension) - } - - // use the first frame to get the static characteristics - width := g.Config.Width - height := g.Config.Height - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -func deriveImage(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - width := i.Bounds().Size().X - height := i.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - return &imageAndMeta{ - image: b, - width: width, - height: height, - size: size, - aspect: aspect, - }, nil -} - -// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, -// of a given jpeg, png, or gif, or an error if something goes wrong. -// -// Note that the aspect ratio of the image will be retained, -// so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not recognised", contentType) - } - - thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) - width := thumb.Bounds().Size().X - height := thumb.Bounds().Size().Y - size := width * height - aspect := float64(width) / float64(height) - - tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, err - } - - out := &bytes.Buffer{} - if err := jpeg.Encode(out, thumb, &jpeg.Options{ - Quality: 75, - }); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - width: width, - height: height, - size: size, - aspect: aspect, - blurhash: bh, - }, nil -} - -// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) - } - - out := &bytes.Buffer{} - if err := png.Encode(out, i); err != nil { - return nil, err - } - return &imageAndMeta{ - image: out.Bytes(), - }, nil -} - -type imageAndMeta struct { - image []byte - width int - height int - size int - aspect float64 - blurhash string -} diff --git a/internal/media/processvideo.go b/internal/media/processvideo.go index d0d11f779..e829c68c0 100644 --- a/internal/media/processvideo.go +++ b/internal/media/processvideo.go @@ -18,6 +18,6 @@ package media -// func (mh *mediaHandler) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) { +// func (mh *mediaManager) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) { // return nil, nil // } diff --git a/internal/media/types.go b/internal/media/types.go index f1608f880..d40f402d2 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -40,7 +40,6 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) - // EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) // const EmojiMaxBytes = 51200 diff --git a/internal/media/util_test.go b/internal/media/util_test.go index 817b597cb..1180bf2aa 100644 --- a/internal/media/util_test.go +++ b/internal/media/util_test.go @@ -103,7 +103,7 @@ func (suite *MediaUtilTestSuite) TestDeriveImageFromJPEG() { suite.NoError(err) // clean it up and validate the clean version - imageAndMeta, err := deriveImage(b, "image/jpeg") + imageAndMeta, err := decodeImage(b, "image/jpeg") suite.NoError(err) suite.Equal(1920, imageAndMeta.width) diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go index ae005f4f6..b2321f414 100644 --- a/internal/processing/account/account.go +++ b/internal/processing/account/account.go @@ -77,7 +77,7 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager fromClientAPI chan messages.FromClientAPI oauthServer oauth.Server filter visibility.Filter @@ -87,10 +87,10 @@ type processor struct { } // New returns a new account processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, fromClientAPI: fromClientAPI, oauthServer: oauthServer, filter: visibility.NewFilter(db), diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index e4611ba23..9c7f0fe67 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -41,7 +41,7 @@ type AccountStandardTestSuite struct { db db.DB tc typeutils.TypeConverter storage *kv.KVStore - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server fromClientAPIChan chan messages.FromClientAPI httpClient pub.HttpClient @@ -80,7 +80,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) suite.httpClient = testrig.NewMockHTTPClient(nil) @@ -88,7 +88,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) - suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaHandler, suite.oauthServer, suite.fromClientAPIChan, suite.federator) + suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") } diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index a32dd9ac0..8de6c83f0 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -159,7 +159,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead } // do the setting - avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "") + avatarInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "") if err != nil { return nil, fmt.Errorf("error processing avatar: %s", err) } @@ -193,7 +193,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead } // do the setting - headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "") + headerInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "") if err != nil { return nil, fmt.Errorf("error processing header: %s", err) } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 217d10dfe..27a7da47a 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -43,16 +43,16 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager fromClientAPI chan messages.FromClientAPI db db.DB } // New returns a new admin processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, fromClientAPI chan messages.FromClientAPI) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, fromClientAPI chan messages.FromClientAPI) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, fromClientAPI: fromClientAPI, db: db, } diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 4989d8e8d..5620374b8 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -49,8 +49,8 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, errors.New("could not read provided emoji: size 0 bytes") } - // allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using - emoji, err := p.mediaHandler.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode) + // allow the mediaManager to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using + emoji, err := p.mediaManager.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode) if err != nil { return nil, fmt.Errorf("error reading emoji: %s", err) } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index de15d3162..68a011683 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -65,8 +65,8 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form }, } - // allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using - attachment, err := p.mediaHandler.ProcessAttachment(ctx, buf.Bytes(), minAttachment) + // allow the mediaManager to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using + attachment, err := p.mediaManager.ProcessAttachment(ctx, buf.Bytes(), minAttachment) if err != nil { return nil, fmt.Errorf("error reading attachment: %s", err) } diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go index 9e050fe84..3d4ae5009 100644 --- a/internal/processing/media/media.go +++ b/internal/processing/media/media.go @@ -43,16 +43,16 @@ type Processor interface { type processor struct { tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager storage *kv.KVStore db db.DB } // New returns a new media processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, storage *kv.KVStore) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor { return &processor{ tc: tc, - mediaHandler: mediaHandler, + mediaManager: mediaManager, storage: storage, db: db, } diff --git a/internal/processing/processor.go b/internal/processing/processor.go index f5334a1ef..2626c1fea 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -235,7 +235,7 @@ type processor struct { stop chan interface{} tc typeutils.TypeConverter oauthServer oauth.Server - mediaHandler media.Handler + mediaManager media.Manager storage *kv.KVStore timelineManager timeline.Manager db db.DB @@ -259,7 +259,7 @@ func NewProcessor( tc typeutils.TypeConverter, federator federation.Federator, oauthServer oauth.Server, - mediaHandler media.Handler, + mediaManager media.Manager, storage *kv.KVStore, timelineManager timeline.Manager, db db.DB, @@ -269,9 +269,9 @@ func NewProcessor( statusProcessor := status.New(db, tc, fromClientAPI) streamingProcessor := streaming.New(db, oauthServer) - accountProcessor := account.New(db, tc, mediaHandler, oauthServer, fromClientAPI, federator) - adminProcessor := admin.New(db, tc, mediaHandler, fromClientAPI) - mediaProcessor := mediaProcessor.New(db, tc, mediaHandler, storage) + accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator) + adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI) + mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage) userProcessor := user.New(db, emailSender) federationProcessor := federationProcessor.New(db, tc, federator, fromFederator) @@ -282,7 +282,7 @@ func NewProcessor( stop: make(chan interface{}), tc: tc, oauthServer: oauthServer, - mediaHandler: mediaHandler, + mediaManager: mediaManager, storage: storage, timelineManager: timelineManager, db: db, diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index dc7562a2e..04793898f 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -51,7 +51,7 @@ type ProcessingStandardTestSuite struct { transportController transport.Controller federator federation.Federator oauthServer oauth.Server - mediaHandler media.Handler + mediaManager media.Manager timelineManager timeline.Manager emailSender email.Sender @@ -218,11 +218,11 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.transportController = testrig.NewTestTransportController(httpClient, suite.db) suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.timelineManager = testrig.NewTestTimelineManager(suite.db) suite.emailSender = testrig.NewEmailSender("../../web/template/", nil) - suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.timelineManager, suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.timelineManager, suite.db, suite.emailSender) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/testrig/federator.go b/testrig/federator.go index b04c01d63..b9c742832 100644 --- a/testrig/federator.go +++ b/testrig/federator.go @@ -27,5 +27,5 @@ import ( // NewTestFederator returns a federator with the given database and (mock!!) transport controller. func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore) federation.Federator { - return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaHandler(db, storage)) + return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaManager(db, storage)) } diff --git a/testrig/mediahandler.go b/testrig/mediahandler.go index ab7fee621..ba2148655 100644 --- a/testrig/mediahandler.go +++ b/testrig/mediahandler.go @@ -24,7 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -// NewTestMediaHandler returns a media handler with the default test config, and the given db and storage. -func NewTestMediaHandler(db db.DB, storage *kv.KVStore) media.Handler { +// NewTestMediaManager returns a media handler with the default test config, and the given db and storage. +func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager { return media.New(db, storage) } diff --git a/testrig/processor.go b/testrig/processor.go index 0baffd35b..d86dd8411 100644 --- a/testrig/processor.go +++ b/testrig/processor.go @@ -28,5 +28,5 @@ import ( // NewTestProcessor returns a Processor suitable for testing purposes func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor { - return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, NewTestTimelineManager(db), db, emailSender) + return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaManager(db, storage), storage, NewTestTimelineManager(db), db, emailSender) } -- cgit v1.3 From 2f57eb5ece9a5cb25b78284d01bd55b14d2e4580 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 2 Jan 2022 15:00:53 +0100 Subject: fiddle around with workers --- internal/media/image.go | 88 ++++++++-------- internal/media/manager.go | 224 ++++------------------------------------- internal/media/media.go | 7 ++ internal/media/pool.go | 65 ++++++++++++ internal/media/process.go | 5 + internal/media/processvideo.go | 23 ----- internal/media/util_test.go | 150 --------------------------- 7 files changed, 135 insertions(+), 427 deletions(-) create mode 100644 internal/media/media.go create mode 100644 internal/media/pool.go create mode 100644 internal/media/process.go delete mode 100644 internal/media/processvideo.go delete mode 100644 internal/media/util_test.go diff --git a/internal/media/image.go b/internal/media/image.go index 87b5d70b7..26bd5e8b6 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,6 +20,7 @@ package media import ( "bytes" + "context" "errors" "fmt" "image" @@ -51,7 +52,8 @@ type imageAndMeta struct { blurhash string } -func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.MediaAttachment, error) { +func (m *manager) processImage(ctx context.Context, data []byte, contentType string, accountID string) { + var clean []byte var err error var original *imageAndMeta @@ -68,9 +70,9 @@ func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.Media case mimeImageGif: // gifs are already clean - no exif data to remove clean = data - original, err = decodeGif(clean, contentType) + original, err = decodeGif(clean) default: - err = fmt.Errorf("content type %s not a recognized image type", contentType) + err = fmt.Errorf("content type %s not a processible image type", contentType) } if err != nil { @@ -89,47 +91,46 @@ func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.Media return nil, err } - originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), attachmentID, extension) - smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), attachmentID, "jpeg") // all thumbnails/smalls are encoded as jpeg + originalURL := uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), attachmentID, extension) + smallURL := uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), attachmentID, "jpeg") // all thumbnails/smalls are encoded as jpeg // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, attachmentID, extension) + originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, attachmentID, extension) if err := m.storage.Put(originalPath, original.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, attachmentID) // all thumbnails/smalls are encoded as jpeg + smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", accountID, TypeAttachment, SizeSmall, attachmentID) // all thumbnails/smalls are encoded as jpeg if err := m.storage.Put(smallPath, small.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } - minAttachment.FileMeta.Original = gtsmodel.Original{ - Width: original.width, - Height: original.height, - Size: original.size, - Aspect: original.aspect, - } - - minAttachment.FileMeta.Small = gtsmodel.Small{ - Width: small.width, - Height: small.height, - Size: small.size, - Aspect: small.aspect, - } - attachment := >smodel.MediaAttachment{ - ID: attachmentID, - StatusID: minAttachment.StatusID, - URL: originalURL, - RemoteURL: minAttachment.RemoteURL, - CreatedAt: minAttachment.CreatedAt, - UpdatedAt: minAttachment.UpdatedAt, - Type: gtsmodel.FileTypeImage, - FileMeta: minAttachment.FileMeta, - AccountID: minAttachment.AccountID, - Description: minAttachment.Description, - ScheduledStatusID: minAttachment.ScheduledStatusID, + ID: attachmentID, + StatusID: "", + URL: originalURL, + RemoteURL: "", + CreatedAt: time.Time{}, + UpdatedAt: time.Time{}, + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Original: gtsmodel.Original{ + Width: original.width, + Height: original.height, + Size: original.size, + Aspect: original.aspect, + }, + Small: gtsmodel.Small{ + Width: small.width, + Height: small.height, + Size: small.size, + Aspect: small.aspect, + }, + }, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", Blurhash: small.blurhash, Processing: 2, File: gtsmodel.File{ @@ -144,33 +145,24 @@ func (m *manager) processImage(data []byte, contentType string) (*gtsmodel.Media FileSize: len(small.image), UpdatedAt: time.Now(), URL: smallURL, - RemoteURL: minAttachment.Thumbnail.RemoteURL, + RemoteURL: "", }, - Avatar: minAttachment.Avatar, - Header: minAttachment.Header, + Avatar: false, + Header: false, } return attachment, nil } -func decodeGif(b []byte, extension string) (*imageAndMeta, error) { - var g *gif.GIF - var err error - - switch extension { - case mimeGif: - g, err = gif.DecodeAll(bytes.NewReader(b)) - default: - err = fmt.Errorf("extension %s not recognised", extension) - } - +func decodeGif(b []byte) (*imageAndMeta, error) { + gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { return nil, err } // use the first frame to get the static characteristics - width := g.Config.Width - height := g.Config.Height + width := gif.Config.Width + height := gif.Config.Height size := width * height aspect := float64(width) / float64(height) diff --git a/internal/media/manager.go b/internal/media/manager.go index 782542ca9..16465bb67 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -22,45 +22,32 @@ import ( "context" "errors" "fmt" - "net/url" + "runtime" "strings" - "time" "codeberg.org/gruf/go-store/kv" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) -// ProcessCallback is triggered by the media manager when an attachment has finished undergoing -// image processing (generation of a blurhash, thumbnail etc) but hasn't yet been inserted into -// the database. It is provided to allow callers to a) access the processed media attachment and b) -// make any last-minute changes to the media attachment before it enters the database. -type ProcessCallback func(*gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment - -// defaultCB will be used when a nil ProcessCallback is passed to one of the manager's interface functions. -// It just returns the processed media attachment with no additional changes. -var defaultCB ProcessCallback = func(a *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment { - return a -} - // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { - ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) + ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error) } type manager struct { db db.DB storage *kv.KVStore + pool *workerPool } // New returns a media manager with the given db and underlying storage. func New(database db.DB, storage *kv.KVStore) Manager { + workers := runtime.NumCPU() / 2 + return &manager{ db: database, storage: storage, + pool: newWorkerPool(workers), } } @@ -68,13 +55,19 @@ func New(database db.DB, storage *kv.KVStore) Manager { INTERFACE FUNCTIONS */ -func (m *manager) ProcessAttachment(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { +func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error) { contentType, err := parseContentType(data) if err != nil { return nil, err } - mainType := strings.Split(contentType, "/")[0] + split := strings.Split(contentType, "/") + if len(split) != 2 { + return nil, fmt.Errorf("content type %s malformed", contentType) + } + + mainType := split[0] + switch mainType { case mimeImage: if !supportedImage(contentType) { @@ -83,192 +76,11 @@ func (m *manager) ProcessAttachment(ctx context.Context, data []byte, accountID if len(data) == 0 { return nil, errors.New("image was of size 0") } - return m.processImage(attachmentBytes, minAttachment) - default: - return nil, fmt.Errorf("content type %s not (yet) supported", contentType) - } -} - -// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it, -// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image, -// and then returns information to the caller about the new header. -func (m *manager) ProcessHeader(ctx context.Context, data []byte, accountID string, cb ProcessCallback) (*gtsmodel.MediaAttachment, error) { - - // make sure we have a type we can handle - contentType, err := parseContentType(data) - if err != nil { - return nil, err - } - if !supportedImage(contentType) { - return nil, fmt.Errorf("%s is not an accepted image type", contentType) - } - - if len(data) == 0 { - return nil, fmt.Errorf("passed reader was of size 0") - } - - // process it - ma, err := m.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL) - if err != nil { - return nil, fmt.Errorf("error processing %s: %s", mediaType, err) - } - - // set it in the database - if err := m.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil { - return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err) - } - - return ma, nil -} - -// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new -// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct -// in the database. -func (m *manager) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) { - var clean []byte - var err error - var original *imageAndMeta - var static *imageAndMeta - - // check content type of the submitted emoji and make sure it's supported by us - contentType, err := parseContentType(emojiBytes) - if err != nil { - return nil, err - } - if !supportedEmoji(contentType) { - return nil, fmt.Errorf("content type %s not supported for emojis", contentType) - } - - if len(emojiBytes) == 0 { - return nil, errors.New("emoji was of size 0") - } - if len(emojiBytes) > EmojiMaxBytes { - return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) - } - - // clean any exif data from png but leave gifs alone - switch contentType { - case mimePng: - if clean, err = purgeExif(emojiBytes); err != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", err) - } - case mimeGif: - clean = emojiBytes + return m.pool.run(func(ctx context.Context, data []byte, contentType string, accountID string) { + m.processImage(ctx, data, contentType, accountID) + }) default: - return nil, errors.New("media type unrecognized") - } - - // unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc - original = &imageAndMeta{ - image: clean, - } - - static, err = deriveStaticEmoji(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error deriving static emoji: %s", err) - } - - // since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver, - // (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created - // with the same username as the instance hostname, which doesn't belong to any particular user. - instanceAccount, err := m.db.GetInstanceAccount(ctx, "") - if err != nil { - return nil, fmt.Errorf("error fetching instance account: %s", err) - } - - // the file extension (either png or gif) - extension := strings.Split(contentType, "/")[1] - - // generate a ulid for the new emoji - newEmojiID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - // activitypub uri for the emoji -- unrelated to actually serving the image - // will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC - emojiURI := uris.GenerateURIForEmoji(newEmojiID) - - // serve url and storage path for the original emoji -- can be png or gif - emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension) - emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension) - - // serve url and storage path for the static version -- will always be png - emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png") - emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID) - - // Store the original emoji - if err := m.storage.Put(emojiPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // Store the static emoji - if err := m.storage.Put(emojiStaticPath, static.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and finally return the new emoji data to the caller -- it's up to them what to do with it - e := >smodel.Emoji{ - ID: newEmojiID, - Shortcode: shortcode, - Domain: "", // empty because this is a local emoji - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - ImageRemoteURL: "", // empty because this is a local emoji - ImageStaticRemoteURL: "", // empty because this is a local emoji - ImageURL: emojiURL, - ImageStaticURL: emojiStaticURL, - ImagePath: emojiPath, - ImageStaticPath: emojiStaticPath, - ImageContentType: contentType, - ImageStaticContentType: mimePng, // static version will always be a png - ImageFileSize: len(original.image), - ImageStaticFileSize: len(static.image), - ImageUpdatedAt: time.Now(), - Disabled: false, - URI: emojiURI, - VisibleInPicker: true, - CategoryID: "", // empty because this is a new emoji -- no category yet - } - return e, nil -} - -func (m *manager) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) { - if !currentAttachment.Header && !currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to neither header nor avatar") - } - - if currentAttachment.Header && currentAttachment.Avatar { - return nil, errors.New("provided attachment was set to both header and avatar") - } - - var headerOrAvi Type - if currentAttachment.Header { - headerOrAvi = TypeHeader - } else if currentAttachment.Avatar { - headerOrAvi = TypeAvatar - } - - if currentAttachment.RemoteURL == "" { - return nil, errors.New("no remote URL on media attachment to dereference") - } - remoteIRI, err := url.Parse(currentAttachment.RemoteURL) - if err != nil { - return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err) - } - - // for content type, we assume we don't know what to expect... - expectedContentType := "*/*" - if currentAttachment.File.ContentType != "" { - // ... and then narrow it down if we do - expectedContentType = currentAttachment.File.ContentType - } - - attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType) - if err != nil { - return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err) + return nil, fmt.Errorf("content type %s not (yet) supported", contentType) } - - return m.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL) } diff --git a/internal/media/media.go b/internal/media/media.go new file mode 100644 index 000000000..e96c37020 --- /dev/null +++ b/internal/media/media.go @@ -0,0 +1,7 @@ +package media + +import gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init" + +type Media struct { + Attachment *gtsmodel.MediaAttachment +} diff --git a/internal/media/pool.go b/internal/media/pool.go new file mode 100644 index 000000000..19b31cde3 --- /dev/null +++ b/internal/media/pool.go @@ -0,0 +1,65 @@ +package media + +import "context" + +func newWorkerPool(workers int) *workerPool { + // make a pool with the given worker capacity + pool := &workerPool{ + workerQueue: make(chan *worker, workers), + } + + // fill the pool with workers + for i := 0; i < workers; i++ { + pool.workerQueue <- &worker{ + // give each worker a reference to the pool so it + // can put itself back in when it's finished + workerQueue: pool.workerQueue, + data: []byte{}, + contentType: "", + accountID: "", + } + } + + return pool +} + +type workerPool struct { + workerQueue chan *worker +} + +func (p *workerPool) run(fn func(ctx context.Context, data []byte, contentType string, accountID string)) (*Media, error) { + + m := &Media{} + + go func() { + // take a worker from the worker pool + worker := <-p.workerQueue + // tell it to work + worker.work(fn) + }() + + return m, nil +} + +type worker struct { + workerQueue chan *worker + data []byte + contentType string + accountID string +} + +func (w *worker) work(fn func(ctx context.Context, data []byte, contentType string, accountID string)) { + // return self to pool when finished + defer w.finish() + // do the work + fn(context.Background(), w.data, w.contentType, w.accountID) +} + +func (w *worker) finish() { + // clear self + w.data = []byte{} + w.contentType = "" + w.accountID = "" + // put self back in the worker pool + w.workerQueue <- w +} diff --git a/internal/media/process.go b/internal/media/process.go new file mode 100644 index 000000000..e921be6bc --- /dev/null +++ b/internal/media/process.go @@ -0,0 +1,5 @@ +package media + +import "context" + +type mediaProcessingFunction func(ctx context.Context, data []byte, contentType string, accountID string) diff --git a/internal/media/processvideo.go b/internal/media/processvideo.go deleted file mode 100644 index e829c68c0..000000000 --- a/internal/media/processvideo.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -// func (mh *mediaManager) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) { -// return nil, nil -// } diff --git a/internal/media/util_test.go b/internal/media/util_test.go deleted file mode 100644 index 1180bf2aa..000000000 --- a/internal/media/util_test.go +++ /dev/null @@ -1,150 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "io/ioutil" - "testing" - - "github.com/spf13/viper" - "github.com/superseriousbusiness/gotosocial/internal/config" - "github.com/superseriousbusiness/gotosocial/internal/log" - - "github.com/stretchr/testify/suite" -) - -type MediaUtilTestSuite struct { - suite.Suite -} - -/* - TEST INFRASTRUCTURE -*/ - -// SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout -func (suite *MediaUtilTestSuite) SetupSuite() { - // doesn't use testrig.InitTestLog() helper to prevent import cycle - viper.Set(config.Keys.LogLevel, "trace") - err := log.Initialize() - if err != nil { - panic(err) - } -} - -func (suite *MediaUtilTestSuite) TearDownSuite() { - -} - -// SetupTest creates a db connection and creates necessary tables before each test -func (suite *MediaUtilTestSuite) SetupTest() { - -} - -// TearDownTest drops tables to make sure there's no data in the db -func (suite *MediaUtilTestSuite) TearDownTest() { - -} - -/* - ACTUAL TESTS -*/ - -func (suite *MediaUtilTestSuite) TestParseContentTypeOK() { - f, err := ioutil.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - ct, err := parseContentType(f) - suite.NoError(err) - suite.Equal("image/jpeg", ct) -} - -func (suite *MediaUtilTestSuite) TestParseContentTypeNotOK() { - f, err := ioutil.ReadFile("./test/test-corrupted.jpg") - suite.NoError(err) - ct, err := parseContentType(f) - suite.NotNil(err) - suite.Equal("", ct) - suite.Equal("filetype unknown", err.Error()) -} - -func (suite *MediaUtilTestSuite) TestRemoveEXIF() { - // load and validate image - b, err := ioutil.ReadFile("./test/test-with-exif.jpg") - suite.NoError(err) - - // clean it up and validate the clean version - clean, err := purgeExif(b) - suite.NoError(err) - - // compare it to our stored sample - sampleBytes, err := ioutil.ReadFile("./test/test-without-exif.jpg") - suite.NoError(err) - suite.EqualValues(sampleBytes, clean) -} - -func (suite *MediaUtilTestSuite) TestDeriveImageFromJPEG() { - // load image - b, err := ioutil.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - - // clean it up and validate the clean version - imageAndMeta, err := decodeImage(b, "image/jpeg") - suite.NoError(err) - - suite.Equal(1920, imageAndMeta.width) - suite.Equal(1080, imageAndMeta.height) - suite.Equal(1.7777777777777777, imageAndMeta.aspect) - suite.Equal(2073600, imageAndMeta.size) - - // assert that the final image is what we would expect - sampleBytes, err := ioutil.ReadFile("./test/test-jpeg-processed.jpg") - suite.NoError(err) - suite.EqualValues(sampleBytes, imageAndMeta.image) -} - -func (suite *MediaUtilTestSuite) TestDeriveThumbnailFromJPEG() { - // load image - b, err := ioutil.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - - // clean it up and validate the clean version - imageAndMeta, err := deriveThumbnail(b, "image/jpeg", 512, 512) - suite.NoError(err) - - suite.Equal(512, imageAndMeta.width) - suite.Equal(288, imageAndMeta.height) - suite.Equal(1.7777777777777777, imageAndMeta.aspect) - suite.Equal(147456, imageAndMeta.size) - suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", imageAndMeta.blurhash) - - sampleBytes, err := ioutil.ReadFile("./test/test-jpeg-thumbnail.jpg") - suite.NoError(err) - suite.EqualValues(sampleBytes, imageAndMeta.image) -} - -func (suite *MediaUtilTestSuite) TestSupportedImageTypes() { - ok := supportedImage("image/jpeg") - suite.True(ok) - - ok = supportedImage("image/bmp") - suite.False(ok) -} - -func TestMediaUtilTestSuite(t *testing.T) { - suite.Run(t, new(MediaUtilTestSuite)) -} -- cgit v1.3 From e08c0e55eec777b9bf1eb907c78cecd4c8859c8e Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 3 Jan 2022 17:37:09 +0100 Subject: add gruf worker pool --- go.mod | 1 + go.sum | 2 + vendor/codeberg.org/gruf/go-runners/LICENSE | 9 ++ vendor/codeberg.org/gruf/go-runners/README.md | 3 + vendor/codeberg.org/gruf/go-runners/context.go | 36 ++++++ vendor/codeberg.org/gruf/go-runners/pool.go | 160 +++++++++++++++++++++++++ vendor/codeberg.org/gruf/go-runners/run.go | 130 ++++++++++++++++++++ vendor/codeberg.org/gruf/go-runners/service.go | 159 ++++++++++++++++++++++++ vendor/modules.txt | 3 + 9 files changed, 503 insertions(+) create mode 100644 vendor/codeberg.org/gruf/go-runners/LICENSE create mode 100644 vendor/codeberg.org/gruf/go-runners/README.md create mode 100644 vendor/codeberg.org/gruf/go-runners/context.go create mode 100644 vendor/codeberg.org/gruf/go-runners/pool.go create mode 100644 vendor/codeberg.org/gruf/go-runners/run.go create mode 100644 vendor/codeberg.org/gruf/go-runners/service.go diff --git a/go.mod b/go.mod index 019607977..71509ee6a 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/superseriousbusiness/gotosocial go 1.17 require ( + codeberg.org/gruf/go-runners v1.2.0 codeberg.org/gruf/go-store v1.1.5 github.com/ReneKroon/ttlcache v1.7.0 github.com/buckket/go-blurhash v1.1.0 diff --git a/go.sum b/go.sum index 5f09d42f1..05d7597e1 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ codeberg.org/gruf/go-nowish v1.1.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K codeberg.org/gruf/go-pools v1.0.2 h1:B0X6yoCL9FVmnvyoizb1SYRwMYPWwEJBjPnBMM5ILos= codeberg.org/gruf/go-pools v1.0.2/go.mod h1:MjUV3H6IASyBeBPCyCr7wjPpSNu8E2N87LG4r4TAyok= codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= +codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBsH4= +codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-store v1.1.5 h1:fp28vzGD15OsAF51CCwi7woH+Y3vb0aMl4OFh9JSjA0= codeberg.org/gruf/go-store v1.1.5/go.mod h1:Q6ev500ddKghDQ8KS4IstL/W9fptDKa2T9oeHP+tXsI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= diff --git a/vendor/codeberg.org/gruf/go-runners/LICENSE b/vendor/codeberg.org/gruf/go-runners/LICENSE new file mode 100644 index 000000000..b7c4417ac --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2021 gruf + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/codeberg.org/gruf/go-runners/README.md b/vendor/codeberg.org/gruf/go-runners/README.md new file mode 100644 index 000000000..91cc1528d --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/README.md @@ -0,0 +1,3 @@ +# go-runners + +Provides a means a simple means of managing long-running functions and services \ No newline at end of file diff --git a/vendor/codeberg.org/gruf/go-runners/context.go b/vendor/codeberg.org/gruf/go-runners/context.go new file mode 100644 index 000000000..edb695060 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/context.go @@ -0,0 +1,36 @@ +package runners + +import ( + "context" + "time" +) + +// ContextWithCancel returns a new context.Context impl with cancel. +func ContextWithCancel() (context.Context, context.CancelFunc) { + ctx := make(cancelctx) + return ctx, func() { close(ctx) } +} + +// cancelctx is the simplest possible cancellable context. +type cancelctx (chan struct{}) + +func (cancelctx) Deadline() (time.Time, bool) { + return time.Time{}, false +} + +func (ctx cancelctx) Done() <-chan struct{} { + return ctx +} + +func (ctx cancelctx) Err() error { + select { + case <-ctx: + return context.Canceled + default: + return nil + } +} + +func (cancelctx) Value(key interface{}) interface{} { + return nil +} diff --git a/vendor/codeberg.org/gruf/go-runners/pool.go b/vendor/codeberg.org/gruf/go-runners/pool.go new file mode 100644 index 000000000..49fc22038 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/pool.go @@ -0,0 +1,160 @@ +package runners + +import ( + "context" + "sync" +) + +// WorkerFunc represents a function processable by a worker in WorkerPool. Note +// that implementations absolutely MUST check whether passed context is Done() +// otherwise stopping the pool may block for large periods of time. +type WorkerFunc func(context.Context) + +// WorkerPool provides a means of enqueuing asynchronous work. +type WorkerPool struct { + queue chan WorkerFunc + free chan struct{} + wait sync.WaitGroup + svc Service +} + +// NewWorkerPool returns a new WorkerPool with provided worker count and WorkerFunc queue size. +// The number of workers represents how many WorkerFuncs can be executed simultaneously, and the +// queue size represents the max number of WorkerFuncs that can be queued at any one time. +func NewWorkerPool(workers int, queue int) WorkerPool { + return WorkerPool{ + queue: make(chan WorkerFunc, queue), + free: make(chan struct{}, workers), + } +} + +// Start will attempt to start the worker pool, asynchronously. Return is success state. +func (pool *WorkerPool) Start() bool { + ok := true + + done := make(chan struct{}) + go func() { + ok = pool.svc.Run(func(ctx context.Context) { + close(done) + pool.process(ctx) + }) + if !ok { + close(done) + } + }() + <-done + + return ok +} + +// Stop will attempt to stop the worker pool, this will block until stopped. Return is success state. +func (pool *WorkerPool) Stop() bool { + return pool.svc.Stop() +} + +// Running returns whether the worker pool is running. +func (pool *WorkerPool) Running() bool { + return pool.svc.Running() +} + +// execute will take a queued function and pass it to a free worker when available. +func (pool *WorkerPool) execute(ctx context.Context, fn WorkerFunc) { + // Set as running + pool.wait.Add(1) + + select { + // Pool context cancelled + case <-ctx.Done(): + pool.wait.Done() + + // Free worker acquired + case pool.free <- struct{}{}: + } + + go func() { + defer func() { + // defer in case panic + <-pool.free + pool.wait.Done() + }() + + // Run queued + fn(ctx) + }() +} + +// process is the background processing routine that passes queued functions to workers. +func (pool *WorkerPool) process(ctx context.Context) { + for { + select { + // Pool context cancelled + case <-ctx.Done(): + for { + select { + // Pop and execute queued + case fn := <-pool.queue: + fn(ctx) // ctx is closed + + // Empty, wait for workers + default: + pool.wait.Wait() + return + } + } + + // Queued func received + case fn := <-pool.queue: + pool.execute(ctx, fn) + } + } +} + +// Enqueue will add provided WorkerFunc to the queue to be performed when there is a free worker. +// Note that 'fn' will ALWAYS be executed, and the supplied context will specify whether this 'fn' +// is being executed during normal pool execution, or if the pool has been stopped with <-ctx.Done(). +func (pool *WorkerPool) Enqueue(fn WorkerFunc) { + // Check valid fn + if fn == nil { + return + } + + select { + // Pool context cancelled + case <-pool.svc.Done(): + + // Placed fn in queue + case pool.queue <- fn: + } +} + +// EnqueueNoBlock performs Enqueue but returns false if queue size is at max. Else, true. +func (pool *WorkerPool) EnqueueNoBlock(fn WorkerFunc) bool { + // Check valid fn + if fn == nil { + return false + } + + select { + // Pool context cancelled + case <-pool.svc.Done(): + return false + + // Placed fn in queue + case pool.queue <- fn: + return true + + // Queue is full + default: + return false + } +} + +// Queue returns the number of currently queued WorkerFuncs. +func (pool *WorkerPool) Queue() int { + return len(pool.queue) +} + +// Workers returns the number of currently active workers. +func (pool *WorkerPool) Workers() int { + return len(pool.free) +} diff --git a/vendor/codeberg.org/gruf/go-runners/run.go b/vendor/codeberg.org/gruf/go-runners/run.go new file mode 100644 index 000000000..27f7fb9b8 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/run.go @@ -0,0 +1,130 @@ +package runners + +import ( + "context" + "errors" + "fmt" + "sync" + "time" +) + +// FuncRunner provides a means of managing long-running functions e.g. main logic loops. +type FuncRunner struct { + // HandOff is the time after which a blocking function will be considered handed off + HandOff time.Duration + + // ErrorHandler is the function that errors are passed to when encountered by the + // provided function. This can be used both for logging, and for error filtering + ErrorHandler func(err error) error + + svc Service // underlying service to manage start/stop + err error // last-set error + mu sync.Mutex // protects err +} + +// Go will attempt to run 'fn' asynchronously. The provided context is used to propagate requested +// cancel if FuncRunner.Stop() is called. Any returned error will be passed to FuncRunner.ErrorHandler +// for filtering/logging/etc. Any blocking functions will be waited on for FuncRunner.HandOff amount of +// time before considering the function as handed off. Returned bool is success state, i.e. returns true +// if function is successfully handed off or returns within hand off time with nil error. +func (r *FuncRunner) Go(fn func(ctx context.Context) error) bool { + done := make(chan struct{}) + + go func() { + var cancelled bool + + has := r.svc.Run(func(ctx context.Context) { + // reset error + r.mu.Lock() + r.err = nil + r.mu.Unlock() + + // Run supplied func and set errror if returned + if err := Run(func() error { return fn(ctx) }); err != nil { + r.mu.Lock() + r.err = err + r.mu.Unlock() + } + + // signal done + close(done) + + // Check if cancelled + select { + case <-ctx.Done(): + cancelled = true + default: + cancelled = false + } + }) + + switch has { + // returned after starting + case true: + r.mu.Lock() + + // filter out errors due FuncRunner.Stop() being called + if cancelled && errors.Is(r.err, context.Canceled) { + // filter out errors from FuncRunner.Stop() being called + r.err = nil + } else if r.err != nil && r.ErrorHandler != nil { + // pass any non-nil error to set handler + r.err = r.ErrorHandler(r.err) + } + + r.mu.Unlock() + + // already running + case false: + close(done) + } + }() + + // get valid handoff to use + handoff := r.HandOff + if handoff < 1 { + handoff = time.Second * 5 + } + + select { + // handed off (long-run successful) + case <-time.After(handoff): + return true + + // 'fn' returned, check error + case <-done: + return (r.Err() == nil) + } +} + +// Stop will cancel the context supplied to the running function. +func (r *FuncRunner) Stop() bool { + return r.svc.Stop() +} + +// Err returns the last-set error value. +func (r *FuncRunner) Err() error { + r.mu.Lock() + err := r.err + r.mu.Unlock() + return err +} + +// Run will execute the supplied 'fn' catching any panics. Returns either function-returned error or formatted panic. +func Run(fn func() error) (err error) { + defer func() { + if r := recover(); r != nil { + if e, ok := r.(error); ok { + // wrap and preserve existing error + err = fmt.Errorf("caught panic: %w", e) + } else { + // simply create new error fromt iface + err = fmt.Errorf("caught panic: %v", r) + } + } + }() + + // run supplied func + err = fn() + return +} diff --git a/vendor/codeberg.org/gruf/go-runners/service.go b/vendor/codeberg.org/gruf/go-runners/service.go new file mode 100644 index 000000000..c0f878c45 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/service.go @@ -0,0 +1,159 @@ +package runners + +import ( + "context" + "sync" +) + +// Service provides a means of tracking a single long-running service, provided protected state +// changes and preventing multiple instances running. Also providing service state information. +type Service struct { + state uint32 // 0=stopped, 1=running, 2=stopping + wait sync.Mutex // wait is the mutex used as a single-entity wait-group, i.e. just a "wait" :p + cncl context.CancelFunc // cncl is the cancel function set for the current context + ctx context.Context // ctx is the current context for running function (or nil if not running) + mu sync.Mutex // mu protects state changes +} + +// Run will run the supplied function until completion, use given context to propagate cancel. +// Immediately returns false if the Service is already running, and true after completed run. +func (svc *Service) Run(fn func(context.Context)) bool { + // Attempt to start the svc + ctx, ok := svc.doStart() + if !ok { + return false + } + + defer func() { + // unlock single wait + svc.wait.Unlock() + + // ensure stopped + svc.Stop() + }() + + // Run user func + if fn != nil { + fn(ctx) + } + return true +} + +// Stop will attempt to stop the service, cancelling the running function's context. Immediately +// returns false if not running, and true only after Service is fully stopped. +func (svc *Service) Stop() bool { + // Attempt to stop the svc + cncl, ok := svc.doStop() + if !ok { + return false + } + + defer func() { + // Get svc lock + svc.mu.Lock() + + // Wait until stopped + svc.wait.Lock() + svc.wait.Unlock() + + // Reset the svc + svc.ctx = nil + svc.cncl = nil + svc.state = 0 + svc.mu.Unlock() + }() + + cncl() // cancel ctx + return true +} + +// doStart will safely set Service state to started, returning a ptr to this context insance. +func (svc *Service) doStart() (context.Context, bool) { + // Protect startup + svc.mu.Lock() + + if svc.state != 0 /* not stopped */ { + svc.mu.Unlock() + return nil, false + } + + // state started + svc.state = 1 + + // Take our own ptr + var ctx context.Context + + if svc.ctx == nil { + // Context required allocating + svc.ctx, svc.cncl = ContextWithCancel() + } + + // Start the waiter + svc.wait.Lock() + + // Set our ptr + unlock + ctx = svc.ctx + svc.mu.Unlock() + + return ctx, true +} + +// doStop will safely set Service state to stopping, returning a ptr to this cancelfunc instance. +func (svc *Service) doStop() (context.CancelFunc, bool) { + // Protect stop + svc.mu.Lock() + + if svc.state != 1 /* not started */ { + svc.mu.Unlock() + return nil, false + } + + // state stopping + svc.state = 2 + + // Take our own ptr + // and unlock state + cncl := svc.cncl + svc.mu.Unlock() + + return cncl, true +} + +// Running returns if Service is running (i.e. state NOT stopped / stopping). +func (svc *Service) Running() bool { + svc.mu.Lock() + state := svc.state + svc.mu.Unlock() + return (state == 1) +} + +// Done returns a channel that's closed when Service.Stop() is called. It is +// the same channel provided to the currently running service function. +func (svc *Service) Done() <-chan struct{} { + var done <-chan struct{} + + svc.mu.Lock() + switch svc.state { + // stopped + // (here we create a new context so that the + // returned 'done' channel here will still + // be valid for when Service is next started) + case 0: + if svc.ctx == nil { + // need to allocate new context + svc.ctx, svc.cncl = ContextWithCancel() + } + done = svc.ctx.Done() + + // started + case 1: + done = svc.ctx.Done() + + // stopping + case 2: + done = svc.ctx.Done() + } + svc.mu.Unlock() + + return done +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 051463c5e..c403b7d2d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -22,6 +22,9 @@ codeberg.org/gruf/go-nowish # codeberg.org/gruf/go-pools v1.0.2 ## explicit; go 1.16 codeberg.org/gruf/go-pools +# codeberg.org/gruf/go-runners v1.2.0 +## explicit; go 1.14 +codeberg.org/gruf/go-runners # codeberg.org/gruf/go-store v1.1.5 ## explicit; go 1.14 codeberg.org/gruf/go-store/kv -- cgit v1.3 From 8abfa7751ab4b80ced391f8a7bd16c5e6c432fee Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 3 Jan 2022 17:37:38 +0100 Subject: return very partial image on first upload --- internal/media/image.go | 143 ++++++++++++++++++---------------------------- internal/media/manager.go | 30 ++++++++-- internal/media/media.go | 31 +++++++++- internal/media/pool.go | 65 --------------------- internal/media/process.go | 5 -- 5 files changed, 109 insertions(+), 165 deletions(-) delete mode 100644 internal/media/pool.go delete mode 100644 internal/media/process.go diff --git a/internal/media/image.go b/internal/media/image.go index 26bd5e8b6..4ad68db5a 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -43,21 +43,55 @@ const ( thumbnailMaxHeight = 512 ) -type imageAndMeta struct { - image []byte - width int - height int - size int - aspect float64 - blurhash string +type ImageMeta struct { + image []byte + contentType string + width int + height int + size int + aspect float64 + blurhash string } -func (m *manager) processImage(ctx context.Context, data []byte, contentType string, accountID string) { +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + extension := strings.Split(contentType, "/")[1] + + attachment := >smodel.MediaAttachment{ + ID: id, + UpdatedAt: time.Now(), + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + Type: gtsmodel.FileTypeImage, + AccountID: accountID, + Processing: 0, + File: gtsmodel.File{ + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), + ContentType: contentType, + UpdatedAt: time.Now(), + }, + Thumbnail: gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + }, + Avatar: false, + Header: false, + } + + media := &Media{ + attachment: attachment, + } + + return media, nil var clean []byte - var err error - var original *imageAndMeta - var small *imageAndMeta + var original *ImageMeta + var small *ImageMeta switch contentType { case mimeImageJpeg, mimeImagePng: @@ -79,82 +113,17 @@ func (m *manager) processImage(ctx context.Context, data []byte, contentType str return nil, err } - small, err = deriveThumbnail(clean, contentType, thumbnailMaxWidth, thumbnailMaxHeight) + small, err = deriveThumbnail(clean, contentType) if err != nil { return nil, fmt.Errorf("error deriving thumbnail: %s", err) } // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - extension := strings.Split(contentType, "/")[1] - attachmentID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - originalURL := uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), attachmentID, extension) - smallURL := uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), attachmentID, "jpeg") // all thumbnails/smalls are encoded as jpeg - - // we store the original... - originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, attachmentID, extension) - if err := m.storage.Put(originalPath, original.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - // and a thumbnail... - smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", accountID, TypeAttachment, SizeSmall, attachmentID) // all thumbnails/smalls are encoded as jpeg - if err := m.storage.Put(smallPath, small.image); err != nil { - return nil, fmt.Errorf("storage error: %s", err) - } - - attachment := >smodel.MediaAttachment{ - ID: attachmentID, - StatusID: "", - URL: originalURL, - RemoteURL: "", - CreatedAt: time.Time{}, - UpdatedAt: time.Time{}, - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Original: gtsmodel.Original{ - Width: original.width, - Height: original.height, - Size: original.size, - Aspect: original.aspect, - }, - Small: gtsmodel.Small{ - Width: small.width, - Height: small.height, - Size: small.size, - Aspect: small.aspect, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: small.blurhash, - Processing: 2, - File: gtsmodel.File{ - Path: originalPath, - ContentType: contentType, - FileSize: len(original.image), - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - Path: smallPath, - ContentType: mimeJpeg, // all thumbnails/smalls are encoded as jpeg - FileSize: len(small.image), - UpdatedAt: time.Now(), - URL: smallURL, - RemoteURL: "", - }, - Avatar: false, - Header: false, - } return attachment, nil } -func decodeGif(b []byte) (*imageAndMeta, error) { +func decodeGif(b []byte) (*ImageMeta, error) { gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { return nil, err @@ -166,7 +135,7 @@ func decodeGif(b []byte) (*imageAndMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &imageAndMeta{ + return &ImageMeta{ image: b, width: width, height: height, @@ -175,7 +144,7 @@ func decodeGif(b []byte) (*imageAndMeta, error) { }, nil } -func decodeImage(b []byte, contentType string) (*imageAndMeta, error) { +func decodeImage(b []byte, contentType string) (*ImageMeta, error) { var i image.Image var err error @@ -201,7 +170,7 @@ func decodeImage(b []byte, contentType string) (*imageAndMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &imageAndMeta{ + return &ImageMeta{ image: b, width: width, height: height, @@ -210,12 +179,12 @@ func decodeImage(b []byte, contentType string) (*imageAndMeta, error) { }, nil } -// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y, +// deriveThumbnail returns a byte slice and metadata for a thumbnail // of a given jpeg, png, or gif, or an error if something goes wrong. // // Note that the aspect ratio of the image will be retained, // so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) { +func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { var i image.Image var err error @@ -239,7 +208,7 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet return nil, fmt.Errorf("content type %s not recognised", contentType) } - thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor) + thumb := resize.Thumbnail(thumbnailMaxWidth, thumbnailMaxHeight, i, resize.NearestNeighbor) width := thumb.Bounds().Size().X height := thumb.Bounds().Size().Y size := width * height @@ -257,7 +226,7 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet }); err != nil { return nil, err } - return &imageAndMeta{ + return &ImageMeta{ image: out.Bytes(), width: width, height: height, @@ -268,7 +237,7 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet } // deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { +func deriveStaticEmoji(b []byte, contentType string) (*ImageMeta, error) { var i image.Image var err error @@ -291,7 +260,7 @@ func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { if err := png.Encode(out, i); err != nil { return nil, err } - return &imageAndMeta{ + return &ImageMeta{ image: out.Bytes(), }, nil } diff --git a/internal/media/manager.go b/internal/media/manager.go index 16465bb67..54b964564 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -25,7 +25,9 @@ import ( "runtime" "strings" + "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" + "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" ) @@ -37,18 +39,27 @@ type Manager interface { type manager struct { db db.DB storage *kv.KVStore - pool *workerPool + pool runners.WorkerPool } // New returns a media manager with the given db and underlying storage. -func New(database db.DB, storage *kv.KVStore) Manager { +func New(database db.DB, storage *kv.KVStore) (Manager, error) { workers := runtime.NumCPU() / 2 + queue := workers * 10 + pool := runners.NewWorkerPool(workers, queue) - return &manager{ + if start := pool.Start(); !start { + return nil, errors.New("could not start worker pool") + } + logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", workers, queue) + + m := &manager{ db: database, storage: storage, - pool: newWorkerPool(workers), + pool: pool, } + + return m, nil } /* @@ -77,9 +88,16 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin return nil, errors.New("image was of size 0") } - return m.pool.run(func(ctx context.Context, data []byte, contentType string, accountID string) { - m.processImage(ctx, data, contentType, accountID) + media, err := m.preProcessImage(ctx, data, contentType, accountID) + if err != nil { + return nil, err + } + + m.pool.Enqueue(func(innerCtx context.Context) { + }) + + return nil, nil default: return nil, fmt.Errorf("content type %s not (yet) supported", contentType) } diff --git a/internal/media/media.go b/internal/media/media.go index e96c37020..0bd196b27 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -1,7 +1,34 @@ package media -import gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init" +import ( + "fmt" + "sync" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) type Media struct { - Attachment *gtsmodel.MediaAttachment + mu sync.Mutex + attachment *gtsmodel.MediaAttachment + rawData []byte +} + +func (m *Media) Thumb() (*ImageMeta, error) { + m.mu.Lock() + thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) + if err != nil { + return nil, fmt.Errorf("error deriving thumbnail: %s", err) + } + m.attachment.Blurhash = thumb.blurhash + aaaaaaaaaaaaaaaa +} + +func (m *Media) PreLoad() { + m.mu.Lock() + defer m.mu.Unlock() +} + +func (m *Media) Load() { + m.mu.Lock() + defer m.mu.Unlock() } diff --git a/internal/media/pool.go b/internal/media/pool.go deleted file mode 100644 index 19b31cde3..000000000 --- a/internal/media/pool.go +++ /dev/null @@ -1,65 +0,0 @@ -package media - -import "context" - -func newWorkerPool(workers int) *workerPool { - // make a pool with the given worker capacity - pool := &workerPool{ - workerQueue: make(chan *worker, workers), - } - - // fill the pool with workers - for i := 0; i < workers; i++ { - pool.workerQueue <- &worker{ - // give each worker a reference to the pool so it - // can put itself back in when it's finished - workerQueue: pool.workerQueue, - data: []byte{}, - contentType: "", - accountID: "", - } - } - - return pool -} - -type workerPool struct { - workerQueue chan *worker -} - -func (p *workerPool) run(fn func(ctx context.Context, data []byte, contentType string, accountID string)) (*Media, error) { - - m := &Media{} - - go func() { - // take a worker from the worker pool - worker := <-p.workerQueue - // tell it to work - worker.work(fn) - }() - - return m, nil -} - -type worker struct { - workerQueue chan *worker - data []byte - contentType string - accountID string -} - -func (w *worker) work(fn func(ctx context.Context, data []byte, contentType string, accountID string)) { - // return self to pool when finished - defer w.finish() - // do the work - fn(context.Background(), w.data, w.contentType, w.accountID) -} - -func (w *worker) finish() { - // clear self - w.data = []byte{} - w.contentType = "" - w.accountID = "" - // put self back in the worker pool - w.workerQueue <- w -} diff --git a/internal/media/process.go b/internal/media/process.go deleted file mode 100644 index e921be6bc..000000000 --- a/internal/media/process.go +++ /dev/null @@ -1,5 +0,0 @@ -package media - -import "context" - -type mediaProcessingFunction func(ctx context.Context, data []byte, contentType string, accountID string) -- cgit v1.3 From 7ebe0f6a15f1881e465b8e78bb8ef8b4982b00aa Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 4 Jan 2022 17:37:54 +0100 Subject: start working on thumb + full funcs --- internal/media/image.go | 24 +++----- internal/media/manager.go | 17 +++--- internal/media/media.go | 136 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 146 insertions(+), 31 deletions(-) diff --git a/internal/media/image.go b/internal/media/image.go index 4ad68db5a..157ae0f4a 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -54,6 +54,14 @@ type ImageMeta struct { } func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + id, err := id.NewRandomULID() if err != nil { return nil, err @@ -93,21 +101,7 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType var original *ImageMeta var small *ImageMeta - switch contentType { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(data); exifErr != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) - } - original, err = decodeImage(clean, contentType) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = data - original, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", contentType) - } + if err != nil { return nil, err diff --git a/internal/media/manager.go b/internal/media/manager.go index 54b964564..074ebdb58 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -81,23 +81,22 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin switch mainType { case mimeImage: - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - media, err := m.preProcessImage(ctx, data, contentType, accountID) if err != nil { return nil, err } m.pool.Enqueue(func(innerCtx context.Context) { - + select { + case <-innerCtx.Done(): + // if the inner context is done that means the worker pool is closing, so we should just return + return + default: + media.PreLoad(innerCtx) + } }) - return nil, nil + return media, nil default: return nil, fmt.Errorf("content type %s not (yet) supported", contentType) } diff --git a/internal/media/media.go b/internal/media/media.go index 0bd196b27..aa11787b2 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -1,29 +1,151 @@ package media import ( + "context" "fmt" "sync" + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + type Media struct { - mu sync.Mutex + mu sync.Mutex + + /* + below fields should be set on newly created media; + attachment will be updated incrementally as media goes through processing + */ + attachment *gtsmodel.MediaAttachment rawData []byte + + /* + below fields represent the processing state of the media thumbnail + */ + + thumbing processState + thumb *ImageMeta + + /* + below fields represent the processing state of the full-sized media + */ + + processing processState + processed *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any } -func (m *Media) Thumb() (*ImageMeta, error) { +func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.mu.Lock() - thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) + defer m.mu.Unlock() + + switch m.thumbing { + case received: + // we haven't processed a thumbnail for this media yet so do it now + thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) + if err != nil { + m.err = fmt.Errorf("error deriving thumbnail: %s", err) + m.thumbing = errored + return nil, m.err + } + + // put the thumbnail in storage + if err := m.storage.Put(m.attachment.Thumbnail.Path, thumb.image); err != nil { + m.err = fmt.Errorf("error storing thumbnail: %s", err) + m.thumbing = errored + return nil, m.err + } + + // set appropriate fields on the attachment based on the thumbnail we derived + m.attachment.Blurhash = thumb.blurhash + m.attachment.FileMeta.Small = gtsmodel.Small{ + Width: thumb.width, + Height: thumb.height, + Size: thumb.size, + Aspect: thumb.aspect, + } + m.attachment.Thumbnail.FileSize = thumb.size + + // put or update the attachment in the database + if err := m.database.Put(ctx, m.attachment); err != nil { + if err != db.ErrAlreadyExists { + m.err = fmt.Errorf("error putting attachment: %s", err) + m.thumbing = errored + return nil, m.err + } + if err := m.database.UpdateByPrimaryKey(ctx, m.attachment); err != nil { + m.err = fmt.Errorf("error updating attachment: %s", err) + m.thumbing = errored + return nil, m.err + } + } + + // set the thumbnail of this media + m.thumb = thumb + + // we're done processing the thumbnail! + m.thumbing = complete + fallthrough + case complete: + return m.thumb, nil + case errored: + return nil, m.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbing) +} + +func (m *Media) Full(ctx context.Context) (*ImageMeta, error) { + var clean []byte + var err error + var original *ImageMeta + + ct := m.attachment.File.ContentType +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(m.rawData); exifErr != nil { + return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) + } + original, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = m.rawData + original, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) + return nil, err } - m.attachment.Blurhash = thumb.blurhash - aaaaaaaaaaaaaaaa + + return original, nil } -func (m *Media) PreLoad() { +func (m *Media) PreLoad(ctx context.Context) { + go m.Thumb(ctx) m.mu.Lock() defer m.mu.Unlock() } -- cgit v1.3 From c2ff8f392b6320d45d8667d4e093e8eb8ddf59c1 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 8 Jan 2022 13:45:42 +0100 Subject: further refinements --- internal/media/image.go | 70 ----------------------- internal/media/manager.go | 56 ++++++++++++++++++ internal/media/media.go | 110 +++++++++++++++++++++++------------- internal/processing/media/create.go | 26 ++------- 4 files changed, 131 insertions(+), 131 deletions(-) diff --git a/internal/media/image.go b/internal/media/image.go index 157ae0f4a..acc62a28b 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,22 +20,16 @@ package media import ( "bytes" - "context" "errors" "fmt" "image" "image/gif" "image/jpeg" "image/png" - "strings" - "time" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" "github.com/superseriousbusiness/exifremove/pkg/exifremove" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) const ( @@ -53,70 +47,6 @@ type ImageMeta struct { blurhash string } -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - extension := strings.Split(contentType, "/")[1] - - attachment := >smodel.MediaAttachment{ - ID: id, - UpdatedAt: time.Now(), - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - Type: gtsmodel.FileTypeImage, - AccountID: accountID, - Processing: 0, - File: gtsmodel.File{ - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), - ContentType: contentType, - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeJpeg, - UpdatedAt: time.Now(), - }, - Avatar: false, - Header: false, - } - - media := &Media{ - attachment: attachment, - } - - return media, nil - - var clean []byte - var original *ImageMeta - var small *ImageMeta - - - - if err != nil { - return nil, err - } - - small, err = deriveThumbnail(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) - } - - // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - - return attachment, nil -} - func decodeGif(b []byte) (*ImageMeta, error) { gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { diff --git a/internal/media/manager.go b/internal/media/manager.go index 074ebdb58..8032ab42d 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -24,11 +24,15 @@ import ( "fmt" "runtime" "strings" + "time" "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. @@ -92,6 +96,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin // if the inner context is done that means the worker pool is closing, so we should just return return default: + // start preloading the media for the caller's convenience media.PreLoad(innerCtx) } }) @@ -101,3 +106,54 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin return nil, fmt.Errorf("content type %s not (yet) supported", contentType) } } + +// preProcessImage initializes processing +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + extension := strings.Split(contentType, "/")[1] + + attachment := >smodel.MediaAttachment{ + ID: id, + UpdatedAt: time.Now(), + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + Type: gtsmodel.FileTypeImage, + AccountID: accountID, + Processing: 0, + File: gtsmodel.File{ + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), + ContentType: contentType, + UpdatedAt: time.Now(), + }, + Thumbnail: gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + }, + Avatar: false, + Header: false, + } + + media := &Media{ + attachment: attachment, + rawData: data, + thumbstate: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return media, nil +} diff --git a/internal/media/media.go b/internal/media/media.go index aa11787b2..022de063e 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -33,15 +33,15 @@ type Media struct { below fields represent the processing state of the media thumbnail */ - thumbing processState - thumb *ImageMeta + thumbstate processState + thumb *ImageMeta /* below fields represent the processing state of the full-sized media */ - processing processState - processed *ImageMeta + fullSizeState processState + fullSize *ImageMeta /* below pointers to database and storage are maintained so that @@ -58,20 +58,20 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.mu.Lock() defer m.mu.Unlock() - switch m.thumbing { + switch m.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) if err != nil { m.err = fmt.Errorf("error deriving thumbnail: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } // put the thumbnail in storage if err := m.storage.Put(m.attachment.Thumbnail.Path, thumb.image); err != nil { m.err = fmt.Errorf("error storing thumbnail: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } @@ -89,12 +89,12 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { if err := m.database.Put(ctx, m.attachment); err != nil { if err != db.ErrAlreadyExists { m.err = fmt.Errorf("error putting attachment: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } if err := m.database.UpdateByPrimaryKey(ctx, m.attachment); err != nil { m.err = fmt.Errorf("error updating attachment: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } } @@ -103,7 +103,7 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.thumb = thumb // we're done processing the thumbnail! - m.thumbing = complete + m.thumbstate = complete fallthrough case complete: return m.thumb, nil @@ -111,46 +111,76 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { return nil, m.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbing) + return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbstate) } -func (m *Media) Full(ctx context.Context) (*ImageMeta, error) { - var clean []byte - var err error - var original *ImageMeta - - ct := m.attachment.File.ContentType -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(m.rawData); exifErr != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) +func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { + m.mu.Lock() + defer m.mu.Unlock() + + switch m.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := m.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(m.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = m.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) } - original, err = decodeImage(clean, ct) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = m.rawData - original, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", ct) - } - if err != nil { - return nil, err + if err != nil { + m.err = err + m.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + m.fullSize = decoded + + // we're done processing the full-size image + m.fullSizeState = complete + fallthrough + case complete: + return m.fullSize, nil + case errored: + return nil, m.err } - return original, nil + return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) } +// PreLoad begins the process of deriving the thumbnail and encoding the full-size image. +// It does this in a non-blocking way, so you can call it and then come back later and check +// if it's finished. func (m *Media) PreLoad(ctx context.Context) { go m.Thumb(ctx) - m.mu.Lock() - defer m.mu.Unlock() + go m.FullSize(ctx) } -func (m *Media) Load() { - m.mu.Lock() - defer m.mu.Unlock() +// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image +// have been processed, then it returns the full-size image. +func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { + if _, err := m.Thumb(ctx); err != nil { + return nil, err + } + + if _, err := m.FullSize(ctx); err != nil { + return nil, err + } + + return m.attachment, nil } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 68a011683..357278e64 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -24,11 +24,9 @@ import ( "errors" "fmt" "io" - "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/text" ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { @@ -46,29 +44,15 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, errors.New("could not read provided attachment: size 0 bytes") } - // now parse the focus parameter - focusx, focusy, err := parseFocus(form.Focus) + // process the media and load it immediately + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID) if err != nil { - return nil, fmt.Errorf("couldn't parse attachment focus: %s", err) + return nil, err } - minAttachment := >smodel.MediaAttachment{ - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - AccountID: account.ID, - Description: text.SanitizeCaption(form.Description), - FileMeta: gtsmodel.FileMeta{ - Focus: gtsmodel.Focus{ - X: focusx, - Y: focusy, - }, - }, - } - - // allow the mediaManager to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using - attachment, err := p.mediaManager.ProcessAttachment(ctx, buf.Bytes(), minAttachment) + attachment, err := media.Load(ctx) if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) + return nil, err } // prepare the frontend representation now -- if there are any errors here at least we can bail without -- cgit v1.3 From f61c3ddcf72ff689b9d253546c58d499b6fe6ac8 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 8 Jan 2022 17:17:01 +0100 Subject: compiling now --- internal/api/client/admin/emojicreate.go | 6 -- internal/db/bundb/errors.go | 2 +- internal/federation/dereferencing/account.go | 48 ++++++--- internal/federation/dereferencing/attachment.go | 102 ------------------- .../federation/dereferencing/attachment_test.go | 106 ------------------- internal/federation/dereferencing/dereferencer.go | 29 +----- internal/federation/dereferencing/media.go | 55 ++++++++++ internal/federation/dereferencing/media_test.go | 102 +++++++++++++++++++ internal/federation/dereferencing/status.go | 10 +- internal/media/manager.go | 25 ++++- internal/media/manager_test.go | 4 + internal/media/media.go | 113 ++++++++++++++++++--- internal/media/media_test.go | 65 ++++++++++++ internal/processing/account/update.go | 47 ++++++--- internal/processing/admin/emoji.go | 13 +-- internal/processing/media/create.go | 11 +- internal/transport/derefmedia.go | 9 +- internal/transport/transport.go | 2 +- testrig/mediahandler.go | 6 +- 19 files changed, 437 insertions(+), 318 deletions(-) delete mode 100644 internal/federation/dereferencing/attachment.go delete mode 100644 internal/federation/dereferencing/attachment_test.go create mode 100644 internal/federation/dereferencing/media.go create mode 100644 internal/federation/dereferencing/media_test.go create mode 100644 internal/media/manager_test.go create mode 100644 internal/media/media_test.go diff --git a/internal/api/client/admin/emojicreate.go b/internal/api/client/admin/emojicreate.go index 617add413..882654ea9 100644 --- a/internal/api/client/admin/emojicreate.go +++ b/internal/api/client/admin/emojicreate.go @@ -27,7 +27,6 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/api" "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/validate" ) @@ -133,10 +132,5 @@ func validateCreateEmoji(form *model.EmojiCreateRequest) error { return errors.New("no emoji given") } - // a very superficial check to see if the media size limit is exceeded - if form.Image.Size > media.EmojiMaxBytes { - return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size) - } - return validate.EmojiShortcode(form.Shortcode) } diff --git a/internal/db/bundb/errors.go b/internal/db/bundb/errors.go index 7602d5e1d..7d0157373 100644 --- a/internal/db/bundb/errors.go +++ b/internal/db/bundb/errors.go @@ -35,7 +35,7 @@ func processSQLiteError(err error) db.Error { // Handle supplied error code: switch sqliteErr.Code() { - case sqlite3.SQLITE_CONSTRAINT_UNIQUE: + case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: return db.ErrAlreadyExists default: return err diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 19c98e203..5912ff29a 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -246,25 +246,49 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.AvatarRemoteURL, - Avatar: true, - }, targetAccount.ID) + avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) if err != nil { - return fmt.Errorf("error processing avatar for user: %s", err) + return err } - targetAccount.AvatarMediaAttachmentID = a.ID + + data, err := t.DereferenceMedia(ctx, avatarIRI) + if err != nil { + return err + } + + media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.AvatarRemoteURL) + if err != nil { + return err + } + + if err := media.SetAsAvatar(ctx); err != nil { + return err + } + + targetAccount.AvatarMediaAttachmentID = media.AttachmentID() } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.HeaderRemoteURL, - Header: true, - }, targetAccount.ID) + headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) if err != nil { - return fmt.Errorf("error processing header for user: %s", err) + return err } - targetAccount.HeaderMediaAttachmentID = a.ID + + data, err := t.DereferenceMedia(ctx, headerIRI) + if err != nil { + return err + } + + media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.HeaderRemoteURL) + if err != nil { + return err + } + + if err := media.SetAsHeader(ctx); err != nil { + return err + } + + targetAccount.HeaderMediaAttachmentID = media.AttachmentID() } return nil } diff --git a/internal/federation/dereferencing/attachment.go b/internal/federation/dereferencing/attachment.go deleted file mode 100644 index 30ab6da10..000000000 --- a/internal/federation/dereferencing/attachment.go +++ /dev/null @@ -1,102 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package dereferencing - -import ( - "context" - "fmt" - "net/url" - - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - if minAttachment.RemoteURL == "" { - return nil, fmt.Errorf("GetRemoteAttachment: minAttachment remote URL was empty") - } - remoteAttachmentURL := minAttachment.RemoteURL - - l := logrus.WithFields(logrus.Fields{ - "username": requestingUsername, - "remoteAttachmentURL": remoteAttachmentURL, - }) - - // return early if we already have the attachment somewhere - maybeAttachment := >smodel.MediaAttachment{} - where := []db.Where{ - { - Key: "remote_url", - Value: remoteAttachmentURL, - }, - } - - if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil { - // we already the attachment in the database - l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID) - return maybeAttachment, nil - } - - a, err := d.RefreshAttachment(ctx, requestingUsername, minAttachment) - if err != nil { - return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err) - } - - if err := d.db.Put(ctx, a); err != nil { - if err != db.ErrAlreadyExists { - return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err) - } - } - - return a, nil -} - -func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - // it just doesn't exist or we have to refresh - if minAttachment.AccountID == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") - } - - if minAttachment.File.ContentType == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment.file.contentType was empty") - } - - t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) - } - - derefURI, err := url.Parse(minAttachment.RemoteURL) - if err != nil { - return nil, err - } - - attachmentBytes, err := t.DereferenceMedia(ctx, derefURI, minAttachment.File.ContentType) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) - } - - a, err := d.mediaManager.ProcessAttachment(ctx, attachmentBytes, minAttachment) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) - } - - return a, nil -} diff --git a/internal/federation/dereferencing/attachment_test.go b/internal/federation/dereferencing/attachment_test.go deleted file mode 100644 index d07cf1c6a..000000000 --- a/internal/federation/dereferencing/attachment_test.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package dereferencing_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type AttachmentTestSuite struct { - DereferencerStandardTestSuite -} - -func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { - fetchingAccount := suite.testAccounts["local_account_1"] - - attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" - attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8" - attachmentContentType := "image/jpeg" - attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" - attachmentDescription := "It's a cute plushie." - - minAttachment := >smodel.MediaAttachment{ - RemoteURL: attachmentURL, - AccountID: attachmentOwner, - StatusID: attachmentStatus, - File: gtsmodel.File{ - ContentType: attachmentContentType, - }, - Description: attachmentDescription, - } - - attachment, err := suite.dereferencer.GetRemoteAttachment(context.Background(), fetchingAccount.Username, minAttachment) - suite.NoError(err) - suite.NotNil(attachment) - - suite.Equal(attachmentOwner, attachment.AccountID) - suite.Equal(attachmentStatus, attachment.StatusID) - suite.Equal(attachmentURL, attachment.RemoteURL) - suite.NotEmpty(attachment.URL) - suite.NotEmpty(attachment.Blurhash) - suite.NotEmpty(attachment.ID) - suite.NotEmpty(attachment.CreatedAt) - suite.NotEmpty(attachment.UpdatedAt) - suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) - suite.Equal(2071680, attachment.FileMeta.Original.Size) - suite.Equal(1245, attachment.FileMeta.Original.Height) - suite.Equal(1664, attachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) - suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) - suite.NotEmpty(attachment.File.Path) - suite.Equal(attachmentContentType, attachment.File.ContentType) - suite.Equal(attachmentDescription, attachment.Description) - - suite.NotEmpty(attachment.Thumbnail.Path) - suite.NotEmpty(attachment.Type) - - // attachment should also now be in the database - dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID) - suite.NoError(err) - suite.NotNil(dbAttachment) - - suite.Equal(attachmentOwner, dbAttachment.AccountID) - suite.Equal(attachmentStatus, dbAttachment.StatusID) - suite.Equal(attachmentURL, dbAttachment.RemoteURL) - suite.NotEmpty(dbAttachment.URL) - suite.NotEmpty(dbAttachment.Blurhash) - suite.NotEmpty(dbAttachment.ID) - suite.NotEmpty(dbAttachment.CreatedAt) - suite.NotEmpty(dbAttachment.UpdatedAt) - suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect) - suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) - suite.Equal(1245, dbAttachment.FileMeta.Original.Height) - suite.Equal(1664, dbAttachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) - suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) - suite.NotEmpty(dbAttachment.File.Path) - suite.Equal(attachmentContentType, dbAttachment.File.ContentType) - suite.Equal(attachmentDescription, dbAttachment.Description) - - suite.NotEmpty(dbAttachment.Thumbnail.Path) - suite.NotEmpty(dbAttachment.Type) -} - -func TestAttachmentTestSuite(t *testing.T) { - suite.Run(t, new(AttachmentTestSuite)) -} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 4f977b8c8..d4786f62d 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,34 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - // GetRemoteAttachment takes a minimal attachment struct and converts it into a fully fleshed out attachment, stored in the database and instance storage. - // - // The parameter minAttachment must have at least the following fields defined: - // * minAttachment.RemoteURL - // * minAttachment.AccountID - // * minAttachment.File.ContentType - // - // The returned attachment will have an ID generated for it, so no need to generate one beforehand. - // A blurhash will also be generated for the attachment. - // - // Most other fields will be preserved on the passed attachment, including: - // * minAttachment.StatusID - // * minAttachment.CreatedAt - // * minAttachment.UpdatedAt - // * minAttachment.FileMeta - // * minAttachment.AccountID - // * minAttachment.Description - // * minAttachment.ScheduledStatusID - // * minAttachment.Thumbnail.RemoteURL - // * minAttachment.Avatar - // * minAttachment.Header - // - // GetRemoteAttachment will return early if an attachment with the same value as minAttachment.RemoteURL - // is found in the database -- then that attachment will be returned and nothing else will be changed or stored. - GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) - // RefreshAttachment is like GetRemoteAttachment, but the attachment will always be dereferenced again, - // whether or not it was already stored in the database. - RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go new file mode 100644 index 000000000..4d62fe0a6 --- /dev/null +++ b/internal/federation/dereferencing/media.go @@ -0,0 +1,55 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/media" +) + +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) { + if accountID == "" { + return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") + } + + t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) + } + + derefURI, err := url.Parse(remoteURL) + if err != nil { + return nil, err + } + + data, err := t.DereferenceMedia(ctx, derefURI) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) + } + + m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, remoteURL) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) + } + + return m, nil +} diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go new file mode 100644 index 000000000..cc158c9a9 --- /dev/null +++ b/internal/federation/dereferencing/media_test.go @@ -0,0 +1,102 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type AttachmentTestSuite struct { + DereferencerStandardTestSuite +} + +func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { + ctx := context.Background() + + fetchingAccount := suite.testAccounts["local_account_1"] + + attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" + attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8" + attachmentContentType := "image/jpeg" + attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" + attachmentDescription := "It's a cute plushie." + + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL) + suite.NoError(err) + + attachment, err := media.LoadAttachment(ctx) + suite.NoError(err) + + suite.NotNil(attachment) + + suite.Equal(attachmentOwner, attachment.AccountID) + suite.Equal(attachmentStatus, attachment.StatusID) + suite.Equal(attachmentURL, attachment.RemoteURL) + suite.NotEmpty(attachment.URL) + suite.NotEmpty(attachment.Blurhash) + suite.NotEmpty(attachment.ID) + suite.NotEmpty(attachment.CreatedAt) + suite.NotEmpty(attachment.UpdatedAt) + suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) + suite.Equal(2071680, attachment.FileMeta.Original.Size) + suite.Equal(1245, attachment.FileMeta.Original.Height) + suite.Equal(1664, attachment.FileMeta.Original.Width) + suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) + suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) + suite.NotEmpty(attachment.File.Path) + suite.Equal(attachmentContentType, attachment.File.ContentType) + suite.Equal(attachmentDescription, attachment.Description) + + suite.NotEmpty(attachment.Thumbnail.Path) + suite.NotEmpty(attachment.Type) + + // attachment should also now be in the database + dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + suite.Equal(attachmentOwner, dbAttachment.AccountID) + suite.Equal(attachmentStatus, dbAttachment.StatusID) + suite.Equal(attachmentURL, dbAttachment.RemoteURL) + suite.NotEmpty(dbAttachment.URL) + suite.NotEmpty(dbAttachment.Blurhash) + suite.NotEmpty(dbAttachment.ID) + suite.NotEmpty(dbAttachment.CreatedAt) + suite.NotEmpty(dbAttachment.UpdatedAt) + suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect) + suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) + suite.Equal(1245, dbAttachment.FileMeta.Original.Height) + suite.Equal(1664, dbAttachment.FileMeta.Original.Width) + suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) + suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) + suite.NotEmpty(dbAttachment.File.Path) + suite.Equal(attachmentContentType, dbAttachment.File.ContentType) + suite.Equal(attachmentDescription, dbAttachment.Description) + + suite.NotEmpty(dbAttachment.Thumbnail.Path) + suite.NotEmpty(dbAttachment.Type) +} + +func TestAttachmentTestSuite(t *testing.T) { + suite.Run(t, new(AttachmentTestSuite)) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index d7de5936a..e184b585f 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -393,9 +393,15 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, a) + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL) if err != nil { - logrus.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err) + logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err) + continue + } + + attachment, err := media.LoadAttachment(ctx) + if err != nil { + logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue } diff --git a/internal/media/manager.go b/internal/media/manager.go index 8032ab42d..9ca450141 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -37,7 +37,17 @@ import ( // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { - ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error) + // ProcessMedia begins the process of decoding and storing the given data as a piece of media (aka an attachment). + // It will return a pointer to a Media struct upon which further actions can be performed, such as getting + // the finished media, thumbnail, decoded bytes, attachment, and setting additional fields. + // + // accountID should be the account that the media belongs to. + // + // RemoteURL is optional, and can be an empty string. Setting this to a non-empty string indicates that + // the piece of media originated on a remote instance and has been dereferenced to be cached locally. + ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) + + ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) } type manager struct { @@ -70,7 +80,7 @@ func New(database db.DB, storage *kv.KVStore) (Manager, error) { INTERFACE FUNCTIONS */ -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string) (*Media, error) { +func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { contentType, err := parseContentType(data) if err != nil { return nil, err @@ -85,7 +95,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin switch mainType { case mimeImage: - media, err := m.preProcessImage(ctx, data, contentType, accountID) + media, err := m.preProcessImage(ctx, data, contentType, accountID, remoteURL) if err != nil { return nil, err } @@ -97,7 +107,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin return default: // start preloading the media for the caller's convenience - media.PreLoad(innerCtx) + media.preLoad(innerCtx) } }) @@ -107,8 +117,12 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin } } +func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { + return nil, nil +} + // preProcessImage initializes processing -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, remoteURL string) (*Media, error) { if !supportedImage(contentType) { return nil, fmt.Errorf("image type %s not supported", contentType) } @@ -128,6 +142,7 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType ID: id, UpdatedAt: time.Now(), URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + RemoteURL: remoteURL, Type: gtsmodel.FileTypeImage, AccountID: accountID, Processing: 0, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go new file mode 100644 index 000000000..45428fbba --- /dev/null +++ b/internal/media/manager_test.go @@ -0,0 +1,4 @@ +package media_test + + + diff --git a/internal/media/media.go b/internal/media/media.go index 022de063e..e19997391 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -1,9 +1,28 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package media import ( "context" "fmt" "sync" + "time" "codeberg.org/gruf/go-store/kv" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -26,7 +45,8 @@ type Media struct { attachment will be updated incrementally as media goes through processing */ - attachment *gtsmodel.MediaAttachment + attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment + emoji *gtsmodel.Emoji // will only be set if the media is an emoji rawData []byte /* @@ -86,17 +106,10 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.attachment.Thumbnail.FileSize = thumb.size // put or update the attachment in the database - if err := m.database.Put(ctx, m.attachment); err != nil { - if err != db.ErrAlreadyExists { - m.err = fmt.Errorf("error putting attachment: %s", err) - m.thumbstate = errored - return nil, m.err - } - if err := m.database.UpdateByPrimaryKey(ctx, m.attachment); err != nil { - m.err = fmt.Errorf("error updating attachment: %s", err) - m.thumbstate = errored - return nil, m.err - } + if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { + m.err = err + m.thumbstate = errored + return nil, err } // set the thumbnail of this media @@ -148,6 +161,30 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { return nil, err } + // put the full size in storage + if err := m.storage.Put(m.attachment.File.Path, decoded.image); err != nil { + m.err = fmt.Errorf("error storing full size image: %s", err) + m.fullSizeState = errored + return nil, m.err + } + + // set appropriate fields on the attachment based on the image we derived + m.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + m.attachment.File.FileSize = decoded.size + m.attachment.File.UpdatedAt = time.Now() + + // put or update the attachment in the database + if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { + m.err = err + m.fullSizeState = errored + return nil, err + } + // set the fullsize of this media m.fullSize = decoded @@ -163,17 +200,46 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) } -// PreLoad begins the process of deriving the thumbnail and encoding the full-size image. +func (m *Media) SetAsAvatar(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.attachment.Avatar = true + return putOrUpdateAttachment(ctx, m.database, m.attachment) +} + +func (m *Media) SetAsHeader(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.attachment.Header = true + return putOrUpdateAttachment(ctx, m.database, m.attachment) +} + +func (m *Media) SetStatusID(ctx context.Context, statusID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.attachment.StatusID = statusID + return putOrUpdateAttachment(ctx, m.database, m.attachment) +} + +// AttachmentID returns the ID of the underlying media attachment without blocking processing. +func (m *Media) AttachmentID() string { + return m.attachment.ID +} + +// preLoad begins the process of deriving the thumbnail and encoding the full-size image. // It does this in a non-blocking way, so you can call it and then come back later and check // if it's finished. -func (m *Media) PreLoad(ctx context.Context) { +func (m *Media) preLoad(ctx context.Context) { go m.Thumb(ctx) go m.FullSize(ctx) } // Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image // have been processed, then it returns the full-size image. -func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { +func (m *Media) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { if _, err := m.Thumb(ctx); err != nil { return nil, err } @@ -184,3 +250,20 @@ func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { return m.attachment, nil } + +func (m *Media) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + return nil, nil +} + +func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { + if err := database.Put(ctx, attachment); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { + return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) + } + } + + return nil +} diff --git a/internal/media/media_test.go b/internal/media/media_test.go new file mode 100644 index 000000000..7e820c9ea --- /dev/null +++ b/internal/media/media_test.go @@ -0,0 +1,65 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "testing" + + "codeberg.org/gruf/go-store/kv" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type MediaStandardTestSuite struct { + suite.Suite + + db db.DB + storage *kv.KVStore + manager media.Manager +} + +func (suite *MediaStandardTestSuite) SetupSuite() { + testrig.InitTestLog() + testrig.InitTestConfig() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() +} + +func (suite *MediaStandardTestSuite) SetupTest() { + testrig.StandardStorageSetup(suite.storage, "../../testrig/media") + testrig.StandardDBSetup(suite.db, nil) + + m, err := media.New(suite.db, suite.storage) + if err != nil { + panic(err) + } + suite.manager = m +} + +func (suite *MediaStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func TestMediaStandardTestSuite(t *testing.T) { + suite.Run(t, &MediaStandardTestSuite{}) +} diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 8de6c83f0..6e74a0ccd 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -33,7 +33,6 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -140,31 +139,40 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(avatar.Size) > maxImageSize { - err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) + err = fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) return nil, err } f, err := avatar.Open() if err != nil { - return nil, fmt.Errorf("could not read provided avatar: %s", err) + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) } // extract the bytes buf := new(bytes.Buffer) size, err := io.Copy(buf, f) if err != nil { - return nil, fmt.Errorf("could not read provided avatar: %s", err) + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) } if size == 0 { - return nil, errors.New("could not read provided avatar: size 0 bytes") + return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") + } + + // we're done with the FileHeader now + if err := f.Close(); err != nil { + return nil, fmt.Errorf("UpdateAvatar: error closing multipart fileheader: %s", err) } // do the setting - avatarInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "") + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") if err != nil { - return nil, fmt.Errorf("error processing avatar: %s", err) + return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) + } + + if err := media.SetAsAvatar(ctx); err != nil { + return nil, fmt.Errorf("UpdateAvatar: error setting media as avatar: %s", err) } - return avatarInfo, f.Close() + return media.LoadAttachment(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, @@ -174,31 +182,40 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(header.Size) > maxImageSize { - err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) + err = fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) return nil, err } f, err := header.Open() if err != nil { - return nil, fmt.Errorf("could not read provided header: %s", err) + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) } // extract the bytes buf := new(bytes.Buffer) size, err := io.Copy(buf, f) if err != nil { - return nil, fmt.Errorf("could not read provided header: %s", err) + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) } if size == 0 { - return nil, errors.New("could not read provided header: size 0 bytes") + return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") + } + + // we're done with the FileHeader now + if err := f.Close(); err != nil { + return nil, fmt.Errorf("UpdateHeader: error closing multipart fileheader: %s", err) } // do the setting - headerInfo, err := p.mediaManager.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "") + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") if err != nil { - return nil, fmt.Errorf("error processing header: %s", err) + return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) + } + + if err := media.SetAsHeader(ctx); err != nil { + return nil, fmt.Errorf("UpdateHeader: error setting media as header: %s", err) } - return headerInfo, f.Close() + return media.LoadAttachment(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 5620374b8..6fb2ca8c5 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -27,7 +27,6 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" ) func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { @@ -49,26 +48,20 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, errors.New("could not read provided emoji: size 0 bytes") } - // allow the mediaManager to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using - emoji, err := p.mediaManager.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode) + media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID, "") if err != nil { - return nil, fmt.Errorf("error reading emoji: %s", err) + return nil, err } - emojiID, err := id.NewULID() + emoji, err := media.LoadEmoji(ctx) if err != nil { return nil, err } - emoji.ID = emojiID apiEmoji, err := p.tc.EmojiToAPIEmoji(ctx, emoji) if err != nil { return nil, fmt.Errorf("error converting emoji to apitype: %s", err) } - if err := p.db.Put(ctx, emoji); err != nil { - return nil, fmt.Errorf("database error while processing emoji: %s", err) - } - return &apiEmoji, nil } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 357278e64..d1840196a 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -44,13 +44,13 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, errors.New("could not read provided attachment: size 0 bytes") } - // process the media and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID) + // process the media attachment and load it immediately + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, "") if err != nil { return nil, err } - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) if err != nil { return nil, err } @@ -62,10 +62,5 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, fmt.Errorf("error parsing media attachment to frontend type: %s", err) } - // now we can confidently put the attachment in the database - if err := p.db.Put(ctx, attachment); err != nil { - return nil, fmt.Errorf("error storing media attachment in db: %s", err) - } - return &apiAttachment, nil } diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index 8a6aa4e24..3fa4a89e4 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -28,18 +28,15 @@ import ( "github.com/sirupsen/logrus" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) { l := logrus.WithField("func", "DereferenceMedia") l.Debugf("performing GET to %s", iri.String()) req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil) if err != nil { return nil, err } - if expectedContentType == "" { - req.Header.Add("Accept", "*/*") - } else { - req.Header.Add("Accept", expectedContentType) - } + + req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT") req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent)) req.Header.Set("Host", iri.Host) diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 73b015865..b470b289a 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -34,7 +34,7 @@ import ( type Transport interface { pub.Transport // DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType. - DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error) + DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. diff --git a/testrig/mediahandler.go b/testrig/mediahandler.go index ba2148655..046ddc5be 100644 --- a/testrig/mediahandler.go +++ b/testrig/mediahandler.go @@ -26,5 +26,9 @@ import ( // NewTestMediaManager returns a media handler with the default test config, and the given db and storage. func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager { - return media.New(db, storage) + m, err := media.New(db, storage) + if err != nil { + panic(err) + } + return m } -- cgit v1.3 From dccf21dd87638320a687a0556c973cced541c945 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 9 Jan 2022 18:41:22 +0100 Subject: tests are passing, but there's still much to be done --- cmd/gotosocial/action/server/server.go | 5 +- internal/ap/extract.go | 18 ++--- internal/ap/extractattachments_test.go | 2 +- internal/ap/interfaces.go | 8 ++- internal/federation/dereferencing/account.go | 25 +++---- internal/federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/media.go | 4 +- internal/federation/dereferencing/media_test.go | 17 +++-- internal/federation/dereferencing/status.go | 9 ++- internal/media/image.go | 32 +++++---- internal/media/manager.go | 80 +++++++++++++++++++---- internal/media/manager_test.go | 50 ++++++++++++++ internal/media/media.go | 51 ++++++--------- internal/media/media_test.go | 65 ------------------ internal/media/types.go | 26 ++++++++ internal/processing/account/update.go | 23 ++++--- internal/processing/admin/emoji.go | 2 +- internal/processing/media/create.go | 12 +++- 18 files changed, 260 insertions(+), 171 deletions(-) delete mode 100644 internal/media/media_test.go diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 05c2e8974..8a227edb7 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,7 +105,10 @@ var Start action.GTSAction = func(ctx context.Context) error { } // build backend handlers - mediaManager := media.New(dbService, storage) + mediaManager, err := media.New(dbService, storage) + if err != nil { + return fmt.Errorf("error creating media manager: %s", err) + } oauthServer := oauth.New(ctx, dbService) transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient) federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) diff --git a/internal/ap/extract.go b/internal/ap/extract.go index ed61faf1e..49dac7186 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -395,20 +395,20 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment.Description = name } + attachment.Blurhash = ExtractBlurhash(i) + attachment.Processing = gtsmodel.ProcessingStatusReceived return attachment, nil } -// func extractBlurhash(i withBlurhash) (string, error) { -// if i.GetTootBlurhashProperty() == nil { -// return "", errors.New("blurhash property was nil") -// } -// if i.GetTootBlurhashProperty().Get() == "" { -// return "", errors.New("empty blurhash string") -// } -// return i.GetTootBlurhashProperty().Get(), nil -// } +// ExtractBlurhash extracts the blurhash value (if present) from a WithBlurhash interface. +func ExtractBlurhash(i WithBlurhash) string { + if i.GetTootBlurhash() == nil { + return "" + } + return i.GetTootBlurhash().Get() +} // ExtractHashtags returns a slice of tags on the interface. func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { diff --git a/internal/ap/extractattachments_test.go b/internal/ap/extractattachments_test.go index 3cee98faa..b937911d2 100644 --- a/internal/ap/extractattachments_test.go +++ b/internal/ap/extractattachments_test.go @@ -42,7 +42,7 @@ func (suite *ExtractAttachmentsTestSuite) TestExtractAttachments() { suite.Equal("image/jpeg", attachment1.File.ContentType) suite.Equal("https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg", attachment1.RemoteURL) suite.Equal("It's a cute plushie.", attachment1.Description) - suite.Empty(attachment1.Blurhash) // atm we discard blurhashes and generate them ourselves during processing + suite.Equal("UxQ0EkRP_4tRxtRjWBt7%hozM_ayV@oLf6WB", attachment1.Blurhash) } func (suite *ExtractAttachmentsTestSuite) TestExtractNoAttachments() { diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index 582465ec3..6edaa42ba 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -70,6 +70,7 @@ type Attachmentable interface { WithMediaType WithURL WithName + WithBlurhash } // Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. @@ -284,9 +285,10 @@ type WithMediaType interface { GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty } -// type withBlurhash interface { -// GetTootBlurhashProperty() vocab.TootBlurhashProperty -// } +// WithBlurhash represents an activity with TootBlurhashProperty +type WithBlurhash interface { + GetTootBlurhash() vocab.TootBlurhashProperty +} // type withFocalPoint interface { // // TODO diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 5912ff29a..d83fc3bac 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -256,16 +257,16 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.AvatarRemoteURL) + avatar := true + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + RemoteURL: &targetAccount.AvatarRemoteURL, + Avatar: &avatar, + }) if err != nil { return err } - if err := media.SetAsAvatar(ctx); err != nil { - return err - } - - targetAccount.AvatarMediaAttachmentID = media.AttachmentID() + targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { @@ -279,16 +280,16 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.HeaderRemoteURL) + header := true + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + RemoteURL: &targetAccount.HeaderRemoteURL, + Header: &header, + }) if err != nil { return err } - if err := media.SetAsHeader(ctx); err != nil { - return err - } - - targetAccount.HeaderMediaAttachmentID = media.AttachmentID() + targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() } return nil } diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index d4786f62d..787a39739 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 4d62fe0a6..0ddab7ae0 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -26,7 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) { +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) { if accountID == "" { return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") } @@ -46,7 +46,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) } - m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, remoteURL) + m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) if err != nil { return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) } diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index cc158c9a9..8fb28d196 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" ) type AttachmentTestSuite struct { @@ -32,7 +33,7 @@ type AttachmentTestSuite struct { func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { ctx := context.Background() - + fetchingAccount := suite.testAccounts["local_account_1"] attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" @@ -40,8 +41,14 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { attachmentContentType := "image/jpeg" attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" attachmentDescription := "It's a cute plushie." - - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL) + attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" + + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + StatusID: &attachmentStatus, + RemoteURL: &attachmentURL, + Description: &attachmentDescription, + Blurhash: &attachmentBlurhash, + }) suite.NoError(err) attachment, err := media.LoadAttachment(ctx) @@ -61,7 +68,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { suite.Equal(2071680, attachment.FileMeta.Original.Size) suite.Equal(1245, attachment.FileMeta.Original.Height) suite.Equal(1664, attachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) + suite.Equal(attachmentBlurhash, attachment.Blurhash) suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) suite.NotEmpty(attachment.File.Path) suite.Equal(attachmentContentType, attachment.File.ContentType) @@ -87,7 +94,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) suite.Equal(1245, dbAttachment.FileMeta.Original.Height) suite.Equal(1664, dbAttachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) + suite.Equal(attachmentBlurhash, dbAttachment.Blurhash) suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) suite.NotEmpty(dbAttachment.File.Path) suite.Equal(attachmentContentType, dbAttachment.File.ContentType) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index e184b585f..47ce087a2 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/media" ) // EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form, @@ -393,7 +394,13 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL) + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalInfo{ + CreatedAt: &a.CreatedAt, + StatusID: &a.StatusID, + RemoteURL: &a.RemoteURL, + Description: &a.Description, + Blurhash: &a.Blurhash, + }) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err) continue diff --git a/internal/media/image.go b/internal/media/image.go index acc62a28b..4c0b28c02 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -108,7 +108,7 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { // // Note that the aspect ratio of the image will be retained, // so it will not necessarily be a square, even if x and y are set as the same value. -func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { +func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageMeta, error) { var i image.Image var err error @@ -138,10 +138,20 @@ func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, err + im := &ImageMeta{ + width: width, + height: height, + size: size, + aspect: aspect, + } + + if createBlurhash { + tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) + bh, err := blurhash.Encode(4, 3, tiny) + if err != nil { + return nil, err + } + im.blurhash = bh } out := &bytes.Buffer{} @@ -150,14 +160,10 @@ func deriveThumbnail(b []byte, contentType string) (*ImageMeta, error) { }); err != nil { return nil, err } - return &ImageMeta{ - image: out.Bytes(), - width: width, - height: height, - size: size, - aspect: aspect, - blurhash: bh, - }, nil + + im.image = out.Bytes() + + return im, nil } // deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. diff --git a/internal/media/manager.go b/internal/media/manager.go index 9ca450141..5e62b39b2 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -45,9 +45,9 @@ type Manager interface { // // RemoteURL is optional, and can be an empty string. Setting this to a non-empty string indicates that // the piece of media originated on a remote instance and has been dereferenced to be cached locally. - ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) + ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) - ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) + ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) } type manager struct { @@ -80,7 +80,7 @@ func New(database db.DB, storage *kv.KVStore) (Manager, error) { INTERFACE FUNCTIONS */ -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { +func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) { contentType, err := parseContentType(data) if err != nil { return nil, err @@ -95,7 +95,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin switch mainType { case mimeImage: - media, err := m.preProcessImage(ctx, data, contentType, accountID, remoteURL) + media, err := m.preProcessImage(ctx, data, contentType, accountID, ai) if err != nil { return nil, err } @@ -117,12 +117,12 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin } } -func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string, remoteURL string) (*Media, error) { +func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) { return nil, nil } // preProcessImage initializes processing -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, remoteURL string) (*Media, error) { +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Media, error) { if !supportedImage(contentType) { return nil, fmt.Errorf("image type %s not supported", contentType) } @@ -139,13 +139,24 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType extension := strings.Split(contentType, "/")[1] attachment := >smodel.MediaAttachment{ - ID: id, - UpdatedAt: time.Now(), - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - RemoteURL: remoteURL, - Type: gtsmodel.FileTypeImage, - AccountID: accountID, - Processing: 0, + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StatusID: "", + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + RemoteURL: "", + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: 0, File: gtsmodel.File{ Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), ContentType: contentType, @@ -161,6 +172,49 @@ func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType Header: false, } + // check if we have additional info to add to the attachment + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + media := &Media{ attachment: attachment, rawData: data, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 45428fbba..aad7f46d0 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -1,4 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package media_test +import ( + "codeberg.org/gruf/go-store/kv" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type MediaManagerStandardTestSuite struct { + suite.Suite + + db db.DB + storage *kv.KVStore + manager media.Manager +} + +func (suite *MediaManagerStandardTestSuite) SetupSuite() { + testrig.InitTestLog() + testrig.InitTestConfig() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() +} +func (suite *MediaManagerStandardTestSuite) SetupTest() { + testrig.StandardStorageSetup(suite.storage, "../../testrig/media") + testrig.StandardDBSetup(suite.db, nil) + suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) +} +func (suite *MediaManagerStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} diff --git a/internal/media/media.go b/internal/media/media.go index e19997391..e0cfe09b7 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -47,7 +47,8 @@ type Media struct { attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment emoji *gtsmodel.Emoji // will only be set if the media is an emoji - rawData []byte + + rawData []byte /* below fields represent the processing state of the media thumbnail @@ -81,7 +82,15 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { switch m.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now - thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) + + // check if we need to create a blurhash or if there's already one set + var createBlurhash bool + if m.attachment.Blurhash == "" { + // no blurhash created yet + createBlurhash = true + } + + thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType, createBlurhash) if err != nil { m.err = fmt.Errorf("error deriving thumbnail: %s", err) m.thumbstate = errored @@ -96,7 +105,10 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { } // set appropriate fields on the attachment based on the thumbnail we derived - m.attachment.Blurhash = thumb.blurhash + if createBlurhash { + m.attachment.Blurhash = thumb.blurhash + } + m.attachment.FileMeta.Small = gtsmodel.Small{ Width: thumb.width, Height: thumb.height, @@ -105,7 +117,6 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { } m.attachment.Thumbnail.FileSize = thumb.size - // put or update the attachment in the database if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { m.err = err m.thumbstate = errored @@ -177,8 +188,8 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { } m.attachment.File.FileSize = decoded.size m.attachment.File.UpdatedAt = time.Now() + m.attachment.Processing = gtsmodel.ProcessingStatusProcessed - // put or update the attachment in the database if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { m.err = err m.fullSizeState = errored @@ -200,30 +211,6 @@ func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) } -func (m *Media) SetAsAvatar(ctx context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.Avatar = true - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - -func (m *Media) SetAsHeader(ctx context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.Header = true - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - -func (m *Media) SetStatusID(ctx context.Context, statusID string) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.attachment.StatusID = statusID - return putOrUpdateAttachment(ctx, m.database, m.attachment) -} - // AttachmentID returns the ID of the underlying media attachment without blocking processing. func (m *Media) AttachmentID() string { return m.attachment.ID @@ -237,8 +224,8 @@ func (m *Media) preLoad(ctx context.Context) { go m.FullSize(ctx) } -// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image -// have been processed, then it returns the full-size image. +// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size +// image have been processed, then it returns the completed attachment. func (m *Media) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { if _, err := m.Thumb(ctx); err != nil { return nil, err @@ -255,6 +242,8 @@ func (m *Media) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { return nil, nil } +// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, +// and then if that doesn't work because the attachment already exists, updating it instead. func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { if err := database.Put(ctx, attachment); err != nil { if err != db.ErrAlreadyExists { diff --git a/internal/media/media_test.go b/internal/media/media_test.go deleted file mode 100644 index 7e820c9ea..000000000 --- a/internal/media/media_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media_test - -import ( - "testing" - - "codeberg.org/gruf/go-store/kv" - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -type MediaStandardTestSuite struct { - suite.Suite - - db db.DB - storage *kv.KVStore - manager media.Manager -} - -func (suite *MediaStandardTestSuite) SetupSuite() { - testrig.InitTestLog() - testrig.InitTestConfig() - - suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() -} - -func (suite *MediaStandardTestSuite) SetupTest() { - testrig.StandardStorageSetup(suite.storage, "../../testrig/media") - testrig.StandardDBSetup(suite.db, nil) - - m, err := media.New(suite.db, suite.storage) - if err != nil { - panic(err) - } - suite.manager = m -} - -func (suite *MediaStandardTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.db) - testrig.StandardStorageTeardown(suite.storage) -} - -func TestMediaStandardTestSuite(t *testing.T) { - suite.Run(t, &MediaStandardTestSuite{}) -} diff --git a/internal/media/types.go b/internal/media/types.go index d40f402d2..aaf423682 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -22,6 +22,7 @@ import ( "bytes" "errors" "fmt" + "time" "github.com/h2non/filetype" ) @@ -67,6 +68,31 @@ const ( TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests ) +// AdditionalInfo represents additional information that should be added to an attachment +// when processing a piece of media. +type AdditionalInfo struct { + // Time that this media was created; defaults to time.Now(). + CreatedAt *time.Time + // ID of the status to which this media is attached; defaults to "". + StatusID *string + // URL of the media on a remote instance; defaults to "". + RemoteURL *string + // Image description of this media; defaults to "". + Description *string + // Blurhash of this media; defaults to "". + Blurhash *string + // ID of the scheduled status to which this media is attached; defaults to "". + ScheduledStatusID *string + // Mark this media as in-use as an avatar; defaults to false. + Avatar *bool + // Mark this media as in-use as a header; defaults to false. + Header *bool + // X focus coordinate for this media; defaults to 0. + FocusX *float32 + // Y focus coordinate for this media; defaults to 0. + FocusY *float32 +} + // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. func parseContentType(content []byte) (string, error) { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 6e74a0ccd..e0dd493e1 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -33,6 +33,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -163,16 +164,15 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead } // do the setting - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") + isAvatar := true + processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + Avatar: &isAvatar, + }) if err != nil { return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } - if err := media.SetAsAvatar(ctx); err != nil { - return nil, fmt.Errorf("UpdateAvatar: error setting media as avatar: %s", err) - } - - return media.LoadAttachment(ctx) + return processingMedia.LoadAttachment(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, @@ -206,16 +206,15 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead } // do the setting - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, "") + isHeader := true + processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + Header: &isHeader, + }) if err != nil { return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } - if err := media.SetAsHeader(ctx); err != nil { - return nil, fmt.Errorf("UpdateHeader: error setting media as header: %s", err) - } - - return media.LoadAttachment(ctx) + return processingMedia.LoadAttachment(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 6fb2ca8c5..737a4ebe2 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -48,7 +48,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, errors.New("could not read provided emoji: size 0 bytes") } - media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID, "") + media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID) if err != nil { return nil, err } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index d1840196a..093a3d2be 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -27,6 +27,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { @@ -44,8 +45,17 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, errors.New("could not read provided attachment: size 0 bytes") } + focusX, focusY, err := parseFocus(form.Focus) + if err != nil { + return nil, fmt.Errorf("could not parse focus value %s: %s", form.Focus, err) + } + // process the media attachment and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, "") + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, &media.AdditionalInfo{ + Description: &form.Description, + FocusX: &focusX, + FocusY: &focusY, + }) if err != nil { return nil, err } -- cgit v1.3 From 0ef478584cd695c9c4899354119e396cc7535c04 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 9 Jan 2022 18:52:28 +0100 Subject: add async test --- internal/federation/dereferencing/media_test.go | 57 ++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index 8fb28d196..1f076f62c 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -21,6 +21,7 @@ package dereferencing_test import ( "context" "testing" + "time" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -31,7 +32,7 @@ type AttachmentTestSuite struct { DereferencerStandardTestSuite } -func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { +func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { ctx := context.Background() fetchingAccount := suite.testAccounts["local_account_1"] @@ -51,6 +52,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { }) suite.NoError(err) + // make a blocking call to load the attachment from the in-process media attachment, err := media.LoadAttachment(ctx) suite.NoError(err) @@ -104,6 +106,59 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { suite.NotEmpty(dbAttachment.Type) } +func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { + ctx := context.Background() + + fetchingAccount := suite.testAccounts["local_account_1"] + + attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" + attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8" + attachmentContentType := "image/jpeg" + attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" + attachmentDescription := "It's a cute plushie." + attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" + + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + StatusID: &attachmentStatus, + RemoteURL: &attachmentURL, + Description: &attachmentDescription, + Blurhash: &attachmentBlurhash, + }) + suite.NoError(err) + attachmentID := media.AttachmentID() + + // wait 5 seconds to let the image process in the background + // it probably won't really take this long but hey let's be sure + time.Sleep(5 * time.Second) + + // now get the attachment from the database + attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + + suite.NotNil(attachment) + + suite.Equal(attachmentOwner, attachment.AccountID) + suite.Equal(attachmentStatus, attachment.StatusID) + suite.Equal(attachmentURL, attachment.RemoteURL) + suite.NotEmpty(attachment.URL) + suite.NotEmpty(attachment.Blurhash) + suite.NotEmpty(attachment.ID) + suite.NotEmpty(attachment.CreatedAt) + suite.NotEmpty(attachment.UpdatedAt) + suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) + suite.Equal(2071680, attachment.FileMeta.Original.Size) + suite.Equal(1245, attachment.FileMeta.Original.Height) + suite.Equal(1664, attachment.FileMeta.Original.Width) + suite.Equal(attachmentBlurhash, attachment.Blurhash) + suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) + suite.NotEmpty(attachment.File.Path) + suite.Equal(attachmentContentType, attachment.File.ContentType) + suite.Equal(attachmentDescription, attachment.Description) + + suite.NotEmpty(attachment.Thumbnail.Path) + suite.NotEmpty(attachment.Type) +} + func TestAttachmentTestSuite(t *testing.T) { suite.Run(t, new(AttachmentTestSuite)) } -- cgit v1.3 From e0f9323b9aa98b55f3557086f7b0a17047943f39 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 10 Jan 2022 18:36:09 +0100 Subject: test the media manager a bit, add shutdown logic --- cmd/gotosocial/action/server/server.go | 4 +- cmd/gotosocial/action/testrig/testrig.go | 7 +- internal/api/client/account/account_test.go | 21 +- internal/api/client/fileserver/servefile_test.go | 4 +- .../api/client/followrequest/followrequest_test.go | 17 +- internal/api/client/media/mediacreate_test.go | 6 +- internal/api/client/status/status_test.go | 19 +- internal/api/client/user/user_test.go | 19 +- internal/api/s2s/user/inboxpost_test.go | 16 +- internal/api/s2s/user/outboxget_test.go | 12 +- internal/api/s2s/user/repliesget_test.go | 12 +- internal/api/s2s/user/user_test.go | 7 +- internal/api/s2s/user/userget_test.go | 4 +- internal/api/s2s/webfinger/webfinger_test.go | 7 +- internal/db/bundb/trace.go | 9 - internal/federation/dereferencing/account.go | 6 +- internal/federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/media.go | 6 +- internal/federation/dereferencing/media_test.go | 4 +- internal/federation/dereferencing/status.go | 2 +- internal/gotosocial/gotosocial.go | 26 ++- internal/media/image.go | 156 +++++++++++-- internal/media/manager.go | 207 ++++++----------- internal/media/manager_test.go | 242 +++++++++++++++++-- internal/media/media.go | 258 --------------------- internal/media/media_test.go | 54 +++++ internal/media/processing.go | 256 ++++++++++++++++++++ internal/media/test/test-corrupted.jpg | 1 - internal/media/test/test-jpeg-blurhash.jpg | Bin 8802 -> 0 bytes internal/media/test/test-with-exif.jpg | Bin 1227452 -> 0 bytes internal/media/test/test-without-exif.jpg | Bin 1227452 -> 0 bytes internal/processing/account/account_test.go | 2 +- internal/processing/account/update.go | 4 +- internal/processing/media/create.go | 2 +- internal/processing/processor_test.go | 6 +- testrig/federator.go | 5 +- testrig/mediahandler.go | 2 +- testrig/processor.go | 5 +- 38 files changed, 872 insertions(+), 538 deletions(-) delete mode 100644 internal/media/media.go create mode 100644 internal/media/media_test.go create mode 100644 internal/media/processing.go delete mode 100644 internal/media/test/test-corrupted.jpg delete mode 100644 internal/media/test/test-jpeg-blurhash.jpg delete mode 100644 internal/media/test/test-with-exif.jpg delete mode 100644 internal/media/test/test-without-exif.jpg diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 8a227edb7..5c1192b56 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,7 +105,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } // build backend handlers - mediaManager, err := media.New(dbService, storage) + mediaManager, err := media.NewManager(dbService, storage) if err != nil { return fmt.Errorf("error creating media manager: %s", err) } @@ -203,7 +203,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } } - gts, err := gotosocial.NewServer(dbService, router, federator) + gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager) if err != nil { return fmt.Errorf("error creating gotosocial service: %s", err) } diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 33f5217e4..d79ba3ea4 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -80,11 +80,12 @@ var Start action.GTSAction = func(ctx context.Context) error { Body: r, }, nil }), dbService) - federator := testrig.NewTestFederator(dbService, transportController, storageBackend) + mediaManager := testrig.NewTestMediaManager(dbService, storageBackend) + federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager) emailSender := testrig.NewEmailSender("./web/template/", nil) - processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender) + processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager) if err := processor.Start(ctx); err != nil { return fmt.Errorf("error starting processor: %s", err) } @@ -156,7 +157,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } } - gts, err := gotosocial.NewServer(dbService, router, federator) + gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager) if err != nil { return fmt.Errorf("error creating gotosocial service: %s", err) } diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go index b642dbcb4..01c8c9599 100644 --- a/internal/api/client/account/account_test.go +++ b/internal/api/client/account/account_test.go @@ -16,6 +16,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -25,13 +26,14 @@ import ( type AccountStandardTestSuite struct { // standard suite interfaces suite.Suite - db db.DB - tc typeutils.TypeConverter - storage *kv.KVStore - federator federation.Federator - processor processing.Processor - emailSender email.Sender - sentEmails map[string]string + db db.DB + tc typeutils.TypeConverter + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator + processor processing.Processor + emailSender email.Sender + sentEmails map[string]string // standard suite models testTokens map[string]*gtsmodel.Token @@ -61,10 +63,11 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() testrig.InitTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.accountModule = account.New(suite.processor).(*account.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index 109ed4eba..5cd4b529b 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -77,10 +77,10 @@ func (suite *ServeFileTestSuite) SetupSuite() { testrig.InitTestLog() suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage)) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage)) suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go index 2d327f461..36b4912e9 100644 --- a/internal/api/client/followrequest/followrequest_test.go +++ b/internal/api/client/followrequest/followrequest_test.go @@ -33,6 +33,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/testrig" @@ -40,11 +41,12 @@ import ( type FollowRequestStandardTestSuite struct { suite.Suite - db db.DB - storage *kv.KVStore - federator federation.Federator - processor processing.Processor - emailSender email.Sender + db db.DB + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator + processor processing.Processor + emailSender email.Sender // standard suite models testTokens map[string]*gtsmodel.Token @@ -74,9 +76,10 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { testrig.InitTestLog() suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 72a377c25..22e0e2188 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -51,9 +51,9 @@ type MediaCreateTestSuite struct { suite.Suite db db.DB storage *kv.KVStore + mediaManager media.Manager federator federation.Federator tc typeutils.TypeConverter - mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender processor processing.Processor @@ -83,9 +83,9 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) // setup module being tested suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module) diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go index dd037f6f4..c6e5b354f 100644 --- a/internal/api/client/status/status_test.go +++ b/internal/api/client/status/status_test.go @@ -26,6 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -34,12 +35,13 @@ import ( type StatusStandardTestSuite struct { // standard suite interfaces suite.Suite - db db.DB - tc typeutils.TypeConverter - federator federation.Federator - emailSender email.Sender - processor processing.Processor - storage *kv.KVStore + db db.DB + tc typeutils.TypeConverter + mediaManager media.Manager + federator federation.Federator + emailSender email.Sender + processor processing.Processor + storage *kv.KVStore // standard suite models testTokens map[string]*gtsmodel.Token @@ -70,9 +72,10 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.statusModule = status.New(suite.processor).(*status.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index d60bf2f1a..e84a78cde 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -26,6 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -33,12 +34,13 @@ import ( type UserStandardTestSuite struct { suite.Suite - db db.DB - tc typeutils.TypeConverter - federator federation.Federator - emailSender email.Sender - processor processing.Processor - storage *kv.KVStore + db db.DB + tc typeutils.TypeConverter + mediaManager media.Manager + federator federation.Federator + emailSender email.Sender + processor processing.Processor + storage *kv.KVStore testTokens map[string]*gtsmodel.Token testClients map[string]*gtsmodel.Client @@ -62,10 +64,11 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.userModule = user.New(suite.processor).(*user.Module) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go index 7cd9ce93a..2f43799c8 100644 --- a/internal/api/s2s/user/inboxpost_test.go +++ b/internal/api/s2s/user/inboxpost_test.go @@ -84,9 +84,9 @@ func (suite *InboxPostTestSuite) TestPostBlock() { body := bytes.NewReader(bodyJson) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -184,9 +184,9 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { body := bytes.NewReader(bodyJson) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -274,9 +274,9 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { body := bytes.NewReader(bodyJson) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -393,9 +393,9 @@ func (suite *InboxPostTestSuite) TestPostDelete() { body := bytes.NewReader(bodyJson) tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) err = processor.Start(context.Background()) suite.NoError(err) userModule := user.New(processor).(*user.Module) diff --git a/internal/api/s2s/user/outboxget_test.go b/internal/api/s2s/user/outboxget_test.go index 2591a80f1..3f5b54c07 100644 --- a/internal/api/s2s/user/outboxget_test.go +++ b/internal/api/s2s/user/outboxget_test.go @@ -45,9 +45,9 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { targetAccount := suite.testAccounts["local_account_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -100,9 +100,9 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { targetAccount := suite.testAccounts["local_account_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -155,9 +155,9 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { targetAccount := suite.testAccounts["local_account_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go index d128e30da..81249091c 100644 --- a/internal/api/s2s/user/repliesget_test.go +++ b/internal/api/s2s/user/repliesget_test.go @@ -48,9 +48,9 @@ func (suite *RepliesGetTestSuite) TestGetReplies() { targetStatus := suite.testStatuses["local_account_1_status_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -109,9 +109,9 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { targetStatus := suite.testStatuses["local_account_1_status_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request @@ -173,9 +173,9 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { targetStatus := suite.testStatuses["local_account_1_status_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 43b835165..b1be59073 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -27,6 +27,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -38,6 +39,7 @@ type UserStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter + mediaManager media.Manager federator federation.Federator emailSender email.Sender processor processing.Processor @@ -77,9 +79,10 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.userModule = user.New(suite.processor).(*user.Module) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module) diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go index 2f3109c92..a764f6993 100644 --- a/internal/api/s2s/user/userget_test.go +++ b/internal/api/s2s/user/userget_test.go @@ -46,9 +46,9 @@ func (suite *UserGetTestSuite) TestGetUser() { targetAccount := suite.testAccounts["local_account_1"] tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go index 7e57f7e45..d7b1647b0 100644 --- a/internal/api/s2s/webfinger/webfinger_test.go +++ b/internal/api/s2s/webfinger/webfinger_test.go @@ -32,6 +32,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -43,6 +44,7 @@ type WebfingerStandardTestSuite struct { suite.Suite db db.DB tc typeutils.TypeConverter + mediaManager media.Manager federator federation.Federator emailSender email.Sender processor processing.Processor @@ -80,9 +82,10 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module) diff --git a/internal/db/bundb/trace.go b/internal/db/bundb/trace.go index 3726506a9..9eaad6880 100644 --- a/internal/db/bundb/trace.go +++ b/internal/db/bundb/trace.go @@ -20,7 +20,6 @@ package bundb import ( "context" - "database/sql" "time" "github.com/sirupsen/logrus" @@ -48,13 +47,5 @@ func (q *debugQueryHook) AfterQuery(_ context.Context, event *bun.QueryEvent) { "operation": event.Operation(), }) - if event.Err != nil && event.Err != sql.ErrNoRows { - // if there's an error the it'll be handled in the application logic, - // but we can still debug log it here alongside the query - l = l.WithField("query", event.Query) - l.Debug(event.Err) - return - } - l.Tracef("[%s] %s", dur, event.Operation()) } diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index d83fc3bac..27d9f39ce 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -119,7 +119,7 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc } else { // take the id we already have and do an update gtsAccount.ID = maybeAccount.ID - +aaaaaaaaaaaaaaaaaa if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil { return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) } @@ -260,7 +260,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * avatar := true processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ RemoteURL: &targetAccount.AvatarRemoteURL, - Avatar: &avatar, + Avatar: &avatar, }) if err != nil { return err @@ -283,7 +283,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * header := true processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ RemoteURL: &targetAccount.HeaderRemoteURL, - Header: &header, + Header: &header, }) if err != nil { return err diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 787a39739..1e6f781b8 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 0ddab7ae0..f02303aa1 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -26,7 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Media, error) { +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) { if accountID == "" { return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") } @@ -46,10 +46,10 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) } - m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) if err != nil { return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) } - return m, nil + return processingMedia, nil } diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index 1f076f62c..61ee6edb6 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -52,8 +52,8 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { }) suite.NoError(err) - // make a blocking call to load the attachment from the in-process media - attachment, err := media.LoadAttachment(ctx) + // make a blocking call to load the attachment from the in-process media + attachment, err := media.Load(ctx) suite.NoError(err) suite.NotNil(attachment) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 47ce087a2..041cfa6b4 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -406,7 +406,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. continue } - attachment, err := media.LoadAttachment(ctx) + attachment, err := media.Load(ctx) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue diff --git a/internal/gotosocial/gotosocial.go b/internal/gotosocial/gotosocial.go index 01a77ce2a..7b2d16e5e 100644 --- a/internal/gotosocial/gotosocial.go +++ b/internal/gotosocial/gotosocial.go @@ -23,6 +23,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/router" ) @@ -41,19 +42,21 @@ type Server interface { // NewServer returns a new gotosocial server, initialized with the given configuration. // An error will be returned the caller if something goes wrong during initialization // eg., no db or storage connection, port for router already in use, etc. -func NewServer(db db.DB, apiRouter router.Router, federator federation.Federator) (Server, error) { +func NewServer(db db.DB, apiRouter router.Router, federator federation.Federator, mediaManager media.Manager) (Server, error) { return &gotosocial{ - db: db, - apiRouter: apiRouter, - federator: federator, + db: db, + apiRouter: apiRouter, + federator: federator, + mediaManager: mediaManager, }, nil } // gotosocial fulfils the gotosocial interface. type gotosocial struct { - db db.DB - apiRouter router.Router - federator federation.Federator + db db.DB + apiRouter router.Router + federator federation.Federator + mediaManager media.Manager } // Start starts up the gotosocial server. If something goes wrong @@ -63,13 +66,16 @@ func (gts *gotosocial) Start(ctx context.Context) error { return nil } -// Stop closes down the gotosocial server, first closing the router -// then the database. If something goes wrong while stopping, an -// error will be returned. +// Stop closes down the gotosocial server, first closing the router, +// then the media manager, then the database. +// If something goes wrong while stopping, an error will be returned. func (gts *gotosocial) Stop(ctx context.Context) error { if err := gts.apiRouter.Stop(ctx); err != nil { return err } + if err := gts.mediaManager.Stop(); err != nil { + return err + } if err := gts.db.Stop(ctx); err != nil { return err } diff --git a/internal/media/image.go b/internal/media/image.go index 4c0b28c02..074dd3839 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,16 +20,22 @@ package media import ( "bytes" + "context" "errors" "fmt" "image" "image/gif" "image/jpeg" "image/png" + "strings" + "time" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" "github.com/superseriousbusiness/exifremove/pkg/exifremove" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) const ( @@ -38,13 +44,119 @@ const ( ) type ImageMeta struct { - image []byte - contentType string - width int - height int - size int - aspect float64 - blurhash string + image []byte + width int + height int + size int + aspect float64 + blurhash string +} + +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Processing, error) { + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + extension := strings.Split(contentType, "/")[1] + + // populate initial fields on the media attachment -- some of these will be overwritten as we proceed + attachment := >smodel.MediaAttachment{ + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StatusID: "", + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + RemoteURL: "", + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: gtsmodel.ProcessingStatusReceived, + File: gtsmodel.File{ + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), + ContentType: contentType, + UpdatedAt: time.Now(), + }, + Thumbnail: gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + }, + Avatar: false, + Header: false, + } + + // check if we have additional info to add to the attachment, + // and overwrite some of the attachment fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + + media := &Processing{ + attachment: attachment, + rawData: data, + thumbstate: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return media, nil } func decodeGif(b []byte) (*ImageMeta, error) { @@ -106,8 +218,12 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { // deriveThumbnail returns a byte slice and metadata for a thumbnail // of a given jpeg, png, or gif, or an error if something goes wrong. // -// Note that the aspect ratio of the image will be retained, -// so it will not necessarily be a square, even if x and y are set as the same value. +// If createBlurhash is true, then a blurhash will also be generated from a tiny +// version of the image. This costs precious CPU cycles, so only use it if you +// really need a blurhash and don't have one already. +// +// If createBlurhash is false, then the blurhash field on the returned ImageAndMeta +// will be an empty string. func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageMeta, error) { var i image.Image var err error @@ -115,21 +231,20 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM switch contentType { case mimeImageJpeg: i, err = jpeg.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } case mimeImagePng: i, err = png.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } case mimeImageGif: i, err = gif.Decode(bytes.NewReader(b)) - if err != nil { - return nil, err - } default: - return nil, fmt.Errorf("content type %s not recognised", contentType) + err = fmt.Errorf("content type %s can't be thumbnailed", contentType) + } + + if err != nil { + return nil, err + } + + if i == nil { + return nil, errors.New("processed image was nil") } thumb := resize.Thumbnail(thumbnailMaxWidth, thumbnailMaxHeight, i, resize.NearestNeighbor) @@ -146,6 +261,8 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM } if createBlurhash { + // for generating blurhashes, it's more cost effective to lose detail rather than + // pass a big image into the blurhash algorithm, so make a teeny tiny version tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor) bh, err := blurhash.Encode(4, 3, tiny) if err != nil { @@ -156,6 +273,7 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM out := &bytes.Buffer{} if err := jpeg.Encode(out, thumb, &jpeg.Options{ + // Quality isn't extremely important for thumbnails, so 75 is "good enough" Quality: 75, }); err != nil { return nil, err diff --git a/internal/media/manager.go b/internal/media/manager.go index 5e62b39b2..c8642fcb4 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -24,63 +24,84 @@ import ( "fmt" "runtime" "strings" - "time" "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { // ProcessMedia begins the process of decoding and storing the given data as a piece of media (aka an attachment). // It will return a pointer to a Media struct upon which further actions can be performed, such as getting - // the finished media, thumbnail, decoded bytes, attachment, and setting additional fields. + // the finished media, thumbnail, attachment, etc. // // accountID should be the account that the media belongs to. // - // RemoteURL is optional, and can be an empty string. Setting this to a non-empty string indicates that - // the piece of media originated on a remote instance and has been dereferenced to be cached locally. - ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) - - ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) + // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. + ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) + ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) + // NumWorkers returns the total number of workers available to this manager. + NumWorkers() int + // QueueSize returns the total capacity of the queue. + QueueSize() int + // JobsQueued returns the number of jobs currently in the task queue. + JobsQueued() int + // ActiveWorkers returns the number of workers currently performing jobs. + ActiveWorkers() int + // Stop stops the underlying worker pool of the manager. It should be called + // when closing GoToSocial in order to cleanly finish any in-progress jobs. + // It will block until workers are finished processing. + Stop() error } type manager struct { - db db.DB - storage *kv.KVStore - pool runners.WorkerPool + db db.DB + storage *kv.KVStore + pool runners.WorkerPool + numWorkers int + queueSize int } -// New returns a media manager with the given db and underlying storage. -func New(database db.DB, storage *kv.KVStore) (Manager, error) { - workers := runtime.NumCPU() / 2 - queue := workers * 10 - pool := runners.NewWorkerPool(workers, queue) - - if start := pool.Start(); !start { - return nil, errors.New("could not start worker pool") +// NewManager returns a media manager with the given db and underlying storage. +// +// A worker pool will also be initialized for the manager, to ensure that only +// a limited number of media will be processed in parallel. +// +// The number of workers will be the number of CPUs available to the Go runtime, +// divided by 2 (rounding down, but always at least 1). +// +// The length of the queue will be the number of workers multiplied by 10. +// +// So for an 8 core machine, the media manager will get 4 workers, and a queue of length 40. +// For a 4 core machine, this will be 2 workers, and a queue length of 20. +// For a single or 2-core machine, the media manager will get 1 worker, and a queue of length 10. +func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { + numWorkers := runtime.NumCPU() / 2 + // make sure we always have at least 1 worker even on single-core machines + if numWorkers == 0 { + numWorkers = 1 } - logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", workers, queue) + queueSize := numWorkers * 10 m := &manager{ - db: database, - storage: storage, - pool: pool, + db: database, + storage: storage, + pool: runners.NewWorkerPool(numWorkers, queueSize), + numWorkers: numWorkers, + queueSize: queueSize, + } + + if start := m.pool.Start(); !start { + return nil, errors.New("could not start worker pool") } + logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", numWorkers, queueSize) return m, nil } -/* - INTERFACE FUNCTIONS -*/ - -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Media, error) { +func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) { contentType, err := parseContentType(data) if err != nil { return nil, err @@ -100,16 +121,20 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin return nil, err } + logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) m.pool.Enqueue(func(innerCtx context.Context) { select { case <-innerCtx.Done(): // if the inner context is done that means the worker pool is closing, so we should just return return default: - // start preloading the media for the caller's convenience - media.preLoad(innerCtx) + // start loading the media already for the caller's convenience + if _, err := media.Load(innerCtx); err != nil { + logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", media.AttachmentID(), err) + } } }) + logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) return media, nil default: @@ -117,112 +142,32 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin } } -func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Media, error) { +func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) { return nil, nil } -// preProcessImage initializes processing -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Media, error) { - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - extension := strings.Split(contentType, "/")[1] - - attachment := >smodel.MediaAttachment{ - ID: id, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - StatusID: "", - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - RemoteURL: "", - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Focus: gtsmodel.Focus{ - X: 0, - Y: 0, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: "", - Processing: 0, - File: gtsmodel.File{ - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), - ContentType: contentType, - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeJpeg, - UpdatedAt: time.Now(), - }, - Avatar: false, - Header: false, - } - - // check if we have additional info to add to the attachment - if ai != nil { - if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description - } - - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID - } - - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash - } +func (m *manager) NumWorkers() int { + return m.numWorkers +} - if ai.Avatar != nil { - attachment.Avatar = *ai.Avatar - } +func (m *manager) QueueSize() int { + return m.queueSize +} - if ai.Header != nil { - attachment.Header = *ai.Header - } +func (m *manager) JobsQueued() int { + return m.pool.Queue() +} - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX - } +func (m *manager) ActiveWorkers() int { + return m.pool.Workers() +} - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY - } - } +func (m *manager) Stop() error { + logrus.Info("stopping media manager worker pool") - media := &Media{ - attachment: attachment, - rawData: data, - thumbstate: received, - fullSizeState: received, - database: m.db, - storage: m.storage, + stopped := m.pool.Stop() + if !stopped { + return errors.New("could not stop media manager worker pool") } - - return media, nil + return nil } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index aad7f46d0..74d0c3008 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -19,36 +19,238 @@ package media_test import ( - "codeberg.org/gruf/go-store/kv" + "context" + "fmt" + "os" + "testing" + "time" + "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init" "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/testrig" ) -type MediaManagerStandardTestSuite struct { - suite.Suite +type ManagerTestSuite struct { + MediaStandardTestSuite +} + +func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { + ctx := context.Background() + + // load bytes from a test image + testBytes, err := os.ReadFile("./test/test-jpeg.jpg") + suite.NoError(err) + suite.NotEmpty(testBytes) + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + // process the media with no additional info provided + processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + suite.NoError(err) + // fetch the attachment id from the processing media + attachmentID := processingMedia.AttachmentID() + + // do a blocking call to fetch the attachment + attachment, err := processingMedia.Load(ctx) + suite.NoError(err) + suite.NotNil(attachment) + + // make sure it's got the stuff set on it that we expect + // the attachment ID and accountID we expect + suite.Equal(attachmentID, attachment.ID) + suite.Equal(accountID, attachment.AccountID) + + // file meta should be correctly derived from the image + suite.EqualValues(gtsmodel.Original{ + Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Original) + suite.EqualValues(gtsmodel.Small{ + Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Small) + suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) + + // now make sure the attachment is in the database + dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + // make sure the processed file is in storage + processedFullBytes, err := suite.storage.Get(attachment.File.Path) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) - db db.DB - storage *kv.KVStore - manager media.Manager + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytes) + + processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg") + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytesExpected) + + suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) } -func (suite *MediaManagerStandardTestSuite) SetupSuite() { - testrig.InitTestLog() - testrig.InitTestConfig() +func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { + ctx := context.Background() + + // load bytes from a test image + testBytes, err := os.ReadFile("./test/test-jpeg.jpg") + suite.NoError(err) + suite.NotEmpty(testBytes) + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + // process the media with no additional info provided + processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + suite.NoError(err) + // fetch the attachment id from the processing media + attachmentID := processingMedia.AttachmentID() + + // wait for the media to finish processing + for finished := processingMedia.Finished(); !finished; finished = processingMedia.Finished() { + time.Sleep(10 * time.Millisecond) + fmt.Printf("\n\nnot finished yet...\n\n") + } + fmt.Printf("\n\nfinished!\n\n") + + // fetch the attachment from the database + attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(attachment) + + // make sure it's got the stuff set on it that we expect + // the attachment ID and accountID we expect + suite.Equal(attachmentID, attachment.ID) + suite.Equal(accountID, attachment.AccountID) + + // file meta should be correctly derived from the image + suite.EqualValues(gtsmodel.Original{ + Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Original) + suite.EqualValues(gtsmodel.Small{ + Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Small) + suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) - suite.db = testrig.NewTestDB() - suite.storage = testrig.NewTestStorage() + // now make sure the attachment is in the database + dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + // make sure the processed file is in storage + processedFullBytes, err := suite.storage.Get(attachment.File.Path) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) + + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytes) + + processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg") + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytesExpected) + + suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) } -func (suite *MediaManagerStandardTestSuite) SetupTest() { - testrig.StandardStorageSetup(suite.storage, "../../testrig/media") - testrig.StandardDBSetup(suite.db, nil) - suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) +func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { + // in this test, we spam the manager queue with 50 new media requests, just to see how it holds up + + ctx := context.Background() + + // load bytes from a test image + testBytes, err := os.ReadFile("./test/test-jpeg.jpg") + suite.NoError(err) + suite.NotEmpty(testBytes) + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + spam := 50 + inProcess := []*media.Processing{} + for i := 0; i < spam; i++ { + // process the media with no additional info provided + processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + suite.NoError(err) + inProcess = append(inProcess, processingMedia) + } + + for _, processingMedia := range inProcess { + fmt.Printf("\n\n\nactive workers: %d, queue length: %d\n\n\n", suite.manager.ActiveWorkers(), suite.manager.JobsQueued()) + + // fetch the attachment id from the processing media + attachmentID := processingMedia.AttachmentID() + + // do a blocking call to fetch the attachment + attachment, err := processingMedia.Load(ctx) + suite.NoError(err) + suite.NotNil(attachment) + + // make sure it's got the stuff set on it that we expect + // the attachment ID and accountID we expect + suite.Equal(attachmentID, attachment.ID) + suite.Equal(accountID, attachment.AccountID) + + // file meta should be correctly derived from the image + suite.EqualValues(gtsmodel.Original{ + Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Original) + suite.EqualValues(gtsmodel.Small{ + Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Small) + suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) + + // now make sure the attachment is in the database + dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + // make sure the processed file is in storage + processedFullBytes, err := suite.storage.Get(attachment.File.Path) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) + + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytes) + + processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg") + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytesExpected) + + suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) + } } -func (suite *MediaManagerStandardTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.db) - testrig.StandardStorageTeardown(suite.storage) +func TestManagerTestSuite(t *testing.T) { + suite.Run(t, &ManagerTestSuite{}) } diff --git a/internal/media/media.go b/internal/media/media.go deleted file mode 100644 index e0cfe09b7..000000000 --- a/internal/media/media.go +++ /dev/null @@ -1,258 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "context" - "fmt" - "sync" - "time" - - "codeberg.org/gruf/go-store/kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type processState int - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - -type Media struct { - mu sync.Mutex - - /* - below fields should be set on newly created media; - attachment will be updated incrementally as media goes through processing - */ - - attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment - emoji *gtsmodel.Emoji // will only be set if the media is an emoji - - rawData []byte - - /* - below fields represent the processing state of the media thumbnail - */ - - thumbstate processState - thumb *ImageMeta - - /* - below fields represent the processing state of the full-sized media - */ - - fullSizeState processState - fullSize *ImageMeta - - /* - below pointers to database and storage are maintained so that - the media can store and update itself during processing steps - */ - - database db.DB - storage *kv.KVStore - - err error // error created during processing, if any -} - -func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { - m.mu.Lock() - defer m.mu.Unlock() - - switch m.thumbstate { - case received: - // we haven't processed a thumbnail for this media yet so do it now - - // check if we need to create a blurhash or if there's already one set - var createBlurhash bool - if m.attachment.Blurhash == "" { - // no blurhash created yet - createBlurhash = true - } - - thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType, createBlurhash) - if err != nil { - m.err = fmt.Errorf("error deriving thumbnail: %s", err) - m.thumbstate = errored - return nil, m.err - } - - // put the thumbnail in storage - if err := m.storage.Put(m.attachment.Thumbnail.Path, thumb.image); err != nil { - m.err = fmt.Errorf("error storing thumbnail: %s", err) - m.thumbstate = errored - return nil, m.err - } - - // set appropriate fields on the attachment based on the thumbnail we derived - if createBlurhash { - m.attachment.Blurhash = thumb.blurhash - } - - m.attachment.FileMeta.Small = gtsmodel.Small{ - Width: thumb.width, - Height: thumb.height, - Size: thumb.size, - Aspect: thumb.aspect, - } - m.attachment.Thumbnail.FileSize = thumb.size - - if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { - m.err = err - m.thumbstate = errored - return nil, err - } - - // set the thumbnail of this media - m.thumb = thumb - - // we're done processing the thumbnail! - m.thumbstate = complete - fallthrough - case complete: - return m.thumb, nil - case errored: - return nil, m.err - } - - return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbstate) -} - -func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { - m.mu.Lock() - defer m.mu.Unlock() - - switch m.fullSizeState { - case received: - var clean []byte - var err error - var decoded *ImageMeta - - ct := m.attachment.File.ContentType - switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(m.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = m.rawData - decoded, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", ct) - } - - if err != nil { - m.err = err - m.fullSizeState = errored - return nil, err - } - - // put the full size in storage - if err := m.storage.Put(m.attachment.File.Path, decoded.image); err != nil { - m.err = fmt.Errorf("error storing full size image: %s", err) - m.fullSizeState = errored - return nil, m.err - } - - // set appropriate fields on the attachment based on the image we derived - m.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, - } - m.attachment.File.FileSize = decoded.size - m.attachment.File.UpdatedAt = time.Now() - m.attachment.Processing = gtsmodel.ProcessingStatusProcessed - - if err := putOrUpdateAttachment(ctx, m.database, m.attachment); err != nil { - m.err = err - m.fullSizeState = errored - return nil, err - } - - // set the fullsize of this media - m.fullSize = decoded - - // we're done processing the full-size image - m.fullSizeState = complete - fallthrough - case complete: - return m.fullSize, nil - case errored: - return nil, m.err - } - - return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) -} - -// AttachmentID returns the ID of the underlying media attachment without blocking processing. -func (m *Media) AttachmentID() string { - return m.attachment.ID -} - -// preLoad begins the process of deriving the thumbnail and encoding the full-size image. -// It does this in a non-blocking way, so you can call it and then come back later and check -// if it's finished. -func (m *Media) preLoad(ctx context.Context) { - go m.Thumb(ctx) - go m.FullSize(ctx) -} - -// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size -// image have been processed, then it returns the completed attachment. -func (m *Media) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - if _, err := m.Thumb(ctx); err != nil { - return nil, err - } - - if _, err := m.FullSize(ctx); err != nil { - return nil, err - } - - return m.attachment, nil -} - -func (m *Media) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - return nil, nil -} - -// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, -// and then if that doesn't work because the attachment already exists, updating it instead. -func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { - if err := database.Put(ctx, attachment); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { - return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) - } - } - - return nil -} diff --git a/internal/media/media_test.go b/internal/media/media_test.go new file mode 100644 index 000000000..f3e73ed79 --- /dev/null +++ b/internal/media/media_test.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "codeberg.org/gruf/go-store/kv" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type MediaStandardTestSuite struct { + suite.Suite + + db db.DB + storage *kv.KVStore + manager media.Manager +} + +func (suite *MediaStandardTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() +} + +func (suite *MediaStandardTestSuite) SetupTest() { + testrig.StandardStorageSetup(suite.storage, "../../testrig/media") + testrig.StandardDBSetup(suite.db, nil) + suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) +} + +func (suite *MediaStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} diff --git a/internal/media/processing.go b/internal/media/processing.go new file mode 100644 index 000000000..3f9fc2bfc --- /dev/null +++ b/internal/media/processing.go @@ -0,0 +1,256 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + "sync" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + +// Processing represents a piece of media that is currently being processed. It exposes +// various functions for retrieving data from the process. +type Processing struct { + mu sync.Mutex + + /* + below fields should be set on newly created media; + attachment will be updated incrementally as media goes through processing + */ + + attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment + emoji *gtsmodel.Emoji // will only be set if the media is an emoji + + rawData []byte + + /* + below fields represent the processing state of the media thumbnail + */ + + thumbstate processState + thumb *ImageMeta + + /* + below fields represent the processing state of the full-sized media + */ + + fullSizeState processState + fullSize *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any +} + +func (p *Processing) Thumb(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.thumbstate { + case received: + // we haven't processed a thumbnail for this media yet so do it now + + // check if we need to create a blurhash or if there's already one set + var createBlurhash bool + if p.attachment.Blurhash == "" { + // no blurhash created yet + createBlurhash = true + } + + thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) + if err != nil { + p.err = fmt.Errorf("error deriving thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // put the thumbnail in storage + if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { + p.err = fmt.Errorf("error storing thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the thumbnail we derived + if createBlurhash { + p.attachment.Blurhash = thumb.blurhash + } + + p.attachment.FileMeta.Small = gtsmodel.Small{ + Width: thumb.width, + Height: thumb.height, + Size: thumb.size, + Aspect: thumb.aspect, + } + p.attachment.Thumbnail.FileSize = thumb.size + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.thumbstate = errored + return nil, err + } + + // set the thumbnail of this media + p.thumb = thumb + + // we're done processing the thumbnail! + p.thumbstate = complete + fallthrough + case complete: + return p.thumb, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) +} + +func (p *Processing) FullSize(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := p.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(p.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = p.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + + if err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // put the full size in storage + if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size image: %s", err) + p.fullSizeState = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the image we derived + p.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + p.attachment.File.FileSize = decoded.size + p.attachment.File.UpdatedAt = time.Now() + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + p.fullSize = decoded + + // we're done processing the full-size image + p.fullSizeState = complete + fallthrough + case complete: + return p.fullSize, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) +} + +// AttachmentID returns the ID of the underlying media attachment without blocking processing. +func (p *Processing) AttachmentID() string { + return p.attachment.ID +} + +// Load blocks until the thumbnail and fullsize content has been processed, and then +// returns the completed attachment. +func (p *Processing) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { + if _, err := p.Thumb(ctx); err != nil { + return nil, err + } + + if _, err := p.FullSize(ctx); err != nil { + return nil, err + } + + return p.attachment, nil +} + +func (p *Processing) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + return nil, nil +} + +func (p *Processing) Finished() bool { + return p.thumbstate == complete && p.fullSizeState == complete +} + +// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, +// and then if that doesn't work because the attachment already exists, updating it instead. +func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { + if err := database.Put(ctx, attachment); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { + return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) + } + } + + return nil +} diff --git a/internal/media/test/test-corrupted.jpg b/internal/media/test/test-corrupted.jpg deleted file mode 100644 index 86e4d75ce..000000000 --- a/internal/media/test/test-corrupted.jpg +++ /dev/null @@ -1 +0,0 @@ -3BHI03zDX4cEuh#Ak%Ag~GOs8Q#dXdu9zH*51jjoO%FYNf*wa&!G-*uN&iLPb0%^FtLLqcQX6x4CorRP82Q8AYsLi!uL!lyX&u^w6xUiwmX*vX^S#waa_A6&CnDO7rXO%3ICVdmGSaUcaVfD2oki4LQO9*b~YM$-@*i2^BnFVPdKw1Nyt9xb$QK3*um4CHpWi^!t#GL8C-VnAOU2Sr6hThro6HA9LBR6nM_oc~bfxR789@mrsG6hH5ZM%uO1IGmHJX4fq!HyL0iYLv*wvhUWqztsyxwXPpyDnZN~PA!uG#*wSxKoVwMITI5hKGEEGk5BK4W~z80WzBr2s~D%Gzks5SNG2hIT@5lQowr0&DWCi2BwDpN0r4P0F2mmds_teB$NoTllvH4vnJ&$C#M5DBXDJdUgI4iZ$G_ZQZLxYW%TzO~2_-vwQLP1J3Y-aXsXlPNCbNmoz1BvQkx*UUP~Nn6sLIrrzarKLtyo2Zb5B584~rA24AN8NV-6!XTS2DP00&$bwRVYOB79sdYwpHaw!Gu&W@vf~TPVTLA~jmals4#~OQuOH%uaL2boZ~_@_81FI3I_Jx_ida9Fzr7z6WxNNFeOEAMBP98b_f6wtmbcFxPen#PHzeDj*xPbdnP^iLMY~zT^YWYICGaR~A0hx_oVtT#V*1-IgW@9TtQDcIe#SiU8jpKEp5nwJ@c6heA2wmPdru!P8b#H2uOokHZd$sGgx&V6zO90S8&kNUjD^D~wsYhZ0p~B2bOa!OimhA^GV440EqQ6o6HWzEGp*zyQ48jXB$LZyzbUtLx7SZw$PmvtcK@4DtZOwyG5fp0Y_6YC~zGYWbLaCVIDy#dBfGMFtCXg-TxKXVk~PHm3@a3po-*6DY2L@PKQp8YZqgB%PgnlNUYHpSryaSvsh$G4C*UW%MkiyC_TScXLYdgyLV@#oG~8U$kA4$*rrbShcL#gi#dr@DBOXO$o0n^p0-nu$mBQI2qYdP%FympbjhK6MI&~U^KQpFcYjXp^*#Op4YaKP4-O3Ibs889Lgr8*NroQ8$2nr^rK3sgOlydjUdi%j_u@YPQzJJDYgxr*Z*qivABBMLLqXv6T^ZAllo3%d3iyW9uOlsrB~2rG$zKLUAnP98VjR!MV1^vm@L~eobU7~&fJw~o1@uj1sokB4KnVk&@_YVkYWu*2fcUY6fvNx~!y09*e@nU1oay0Ai&XGSKw6n2T4cL_Pgtf6XtGgXH~wiWkVL&6LKbO38P9poIX#01w17xVbKfZAoFwxD8$1l9VV-WwI@404Z-EJoDCE2!kGMcOfdrYJhBu7#vJ*aOVO4Lgxoh7ybUefdC0DHht51deeEgWxNq8npu0%pMvu2dPM&Cnx#knz2CO9GQL#A%wB6fXBx-dit$Wf@ii%#~ttUon&eYP&iFLC*475CWN8&cMg#4i~dS*O4mJVD@xELn3LzuKJk-W7Pkj~5NZlhVL0pr_RH5Oic7GMWK&rsB~FY-%PGqpS6E4Rmhr~*^n@I7EY&FvTHn*1gJ@m139p_hT3Gh6uteFxVhlD9r471l1&#sJ6b9aut5yFHhxsVAvMeI5i1A!P0$-BGsegn8YpjNf15Ce0Dz%Wb7NYp5eY2J!dRZS8Tld!e1_72ER8KUIW%&NKGbL09Z~!F7322O^wjCaV~49jxCu6c1Dqdg0ZO^iT0#mkg@BwAytt^c%H^bI45mkegz0btunQ6zJv0Ecypsf99sGsIomRB#rbluVc1mU*DjA&Y-WV-XhzI$^RCSdH2k8jTRZRv3w7%3kNLQ47Q*p$brF-Zqtv0pkMe^a!cmZ@jDx##^6893Uuv_iH^#sA1XeMvE50wsP#gD0w2LbEh0iq_NMe%e~x^AO3kK8UbVBd2y*zQ!-y-o-^0sZ5xe#*BDJk$aD8#8Or*&0Jr8A$-Z^0Mx1awGh55e%69Di^JAjr3FE9B_ZpoH*uha%SxjPJWRU%ElstN!7L4z#@1ReWdY8QvdfFVUe_zNnP_pKcI8iW&92Eh@IVVH~6EJT$tlUeiz1QLKFE&f5PLKwXNl#trj&@cmKfJ87mMNRb6e@zWeOXr9&%3KSpo9&_6kAoOgHh@f-%D@GOVTf7AWqIc8m9BNd0EE_@ApS44Y-g$*#s_bO3%B^G_er05bqNxoQ$H@GAlUnhXcgo@sY2SxR1xm@51CL^*sYupBf!OcqiC~fKytFZWl&mh0$F3w1n!CfrmD3htCLFx-X3eV5&oG^smNVB_*#aLQuhC#9@@k9MccBwshe5WpklDEGAx%_2Y4CBi3ne^V*BqRGlxLR^OnW#tD-6rrv6ivt_4NOvHOUyC~86TQ%bh4QqWJ-ali$Hf@OV59O67I#ZdBEp6vA0hFJm^RIk6J3!Xlpqdhik9mcTPznjXknpMW*N4&nVT~v5jVW5-XY^b-fzaUNh5Ej1QaeaR#d*nEzyUlXTCd7%im_HByP30gNr!aX69m6Tsexd*oTk&6COPu0TM9F*rWqUXTgR^S9jF3JcFwZ_BXZ@i4Q61*6rm^rQnKEFSGV4^f1^E0__!LqIXT^v^2%Wj*1HHhCEisrsg$sfoz1G1JcKhV!B^q!8mz1c8CWg~*cC!FRAZQTz@OiUyPf0dbduR#ydk0DA@d_YCB$$$dfprxW0vmEY@z56s22$BsF80HS*n@QpclLKJX8*OPr_Oj!tUpR4TV1^_DDYG_#HAFBo-vX*IEQmn58HV9AFjxxQ$9H1E2NsbsGzT50d379K6AQ&#RE-KTg&hwdnS&K*-qLcrdqX6@RN&-a-%D@VNh^7r2Eb^oKJ*kj2apNKnc^eJ-kUb*YtjMyK^#fyaDsMBcp*Qk@bd-Dm6Z*TgGJF2KPBuyQP!shfB2%qQk__b~!cw3$U&eIOZceV5wQ4u@JtTuB9W6RqR6ZqA3Ct4PNxw^z~aEB!wZ_ldow75#VWM407d@3m2Xntl@@j9NWRcR-KgqWK8TaVf8F_rbz1O4XYyy4Vg%QnPQcDEZ_$pBuaIv^vHVFKYVzKFoULJp!UyMKog2mnJQmA@IPis$F3oTL^RNm_45&nb3Y6Mx$OXZy&8c-7C2CLpv2m52LRb^JcaCJH3eoB0AyKhCQj%isfr~s*5GJ!jjd!lzxzW1qTS6sp2jn#s@UFYxoz#&@Pk_n&dwR&5^aZGADDfNRQV0_kL7_ECmaXsKDfV*J*pFBr@*tui4ndv!rv!KU5FsuT#nQJnOAejYweHIoq#Z^ML~LXh_iBEs5!4&ej-^DGPzpMyEtB9y_~Hp4%l&av0lnXeYFQtA^3UZ4cmw0UGo9QjOQyN*-JBOZdGPDWpmEuFp#F5W!4^OOdzmXw~9%Eg4WZkFJ6&n^8NVguUkm92eUP6X2BN#X0@A$0U@D^OZpq&BYL^iFm9u4nO01gyRQTJlaenYvZZEV!3%Fur#Y6CISyxq-DuxMQL8sw1qPY~^yBO@HN-@4JKJBKOIFi27JtZ%@p$hFseaWW*prM&9!5IBV7^xsv@BJVJ5YYS#5OEd_qtb8edjytgO8Sy4AX16W2JmoRr#3dcjmxmVrvvPHy*Ouo0WAmo559GcW3rgH4jFi-_UH6lJN!_sIASJYvEI1lSm!9gDylbqc#HJjPJm$4b_fl5aN5jkB0Zf^dB4vEFS0FatCHx!siu6XJEl3rkM!I&kqmQxTxQN3nuZklhifiaV^&3Cj8RI5CVi7S%$khEP_8F2_@*YJJXC5Ng6oFBF7@~Hk$hDG9dh7MirrW7%PMNQ~alVwn-Hj%OdtnlLKliK_t^yimq-DcV$ZgFVH7i%&jvcfD!!LuN8kF6h%OG5-_~1gmv06xovdUiqk!WFl^Kt&RkEVBo1WxtGQKWR&ulcWxVYmjn#sg%9v76#f3tNc!C~^zze&igwVzDwgbMlzV#skwi!kBn$nnfXV8Dm9%3D@0Dp_3G2!*kr7@tiJpM&P_j9F2FK2~YVrDHPHkjZp!efZif*0oE3NnQN5qCzsZ#&!hB4bkrt&#igE#yQKsANt6oG6$lllQ09UXmA2X48nV@RICX3I6AhmB9Q1XKrEUnaj^0SIs#0EqU_KCWWEAtYJg$a1da6laTWjrF@-m-%rl5F8H23pYRvusRkdgm~uD~td7RHVPg3zXXrc76RfdSS8aAsSBmjm_4oSd!^1Io&lkk&M9pfPmdteYzAqKw%cb*~FY9A%3&-1uM!oi%W1bUzoxBWpi$ljTXK4&Pr*cA2qUFNkZW1jUAlphk9WXF1c&!eYuTD$*_JV#jL%2yfRG7aZDFtVbersd%iFJg07XtW_J74irWzW#ft0sq-NW@v-DTg$#s~@c_xeXPFJfyu^8ai67iTcTpp#V~EUeKepk60Cs7RxKLz#mdhbdma~Hf8PEj1Y1Azwfl9HJoyvCUfENVYAWVZIALy2v618ZvTiQCO18*$96y$NP_LthuS~Z$1CG1%AGjpqXWqouhktfz!SPoVw-xr6rpRh0^oxwjaTup7No~H!o4*k-YsLlW2WQ0jTNh3gXm5NGrz*3M$fjjEfJMsvS7ARActzuzgAZm9_mi9n%-vf6zCP@NZV%1CY@bDVd@7u#6bK#1P$k5*XRB4Wj5L0yC#pKY*W&rv!jGUQ%OsDo~MbGWCPuYLIZaL6#sX$lT9p1c9g-I5~xCr%!@J7IbX#Y_fy_WBgZxjD21H213Y#*A!e#s&u1@Gx^R@_ngUUZg-iWDmS~$T8VA*sWNDv~F$G2xvFTZ%Z#DGrh4xLMWg~fSt--EOhi_qWk1xSnt!9AFAoZOBY-A5OIwJ9gfRUzaMR1Yk$0irw-2mJRzrHbP8d2CcI~CWkJ_g8hkbdQQsgheWe5oZHWN8S!T5q00Nw1lCqnx7MhzZq&Oi@0UBosF%_aIvQDFxQH!TmWimN2EZ%0#5@mJ_FL5J^0!ZahIbKfb&76dT4Qs*uvuNzwb^eIgcIn~AMhuB-rNfFhjQEm@YmAcCJzzgPBag#yNSeBswux&^g1WytSp-a6z1_0gr3_k~eYADTNCn6KeB6dVQ*im0stw-HtExVZ@0MYEUJWNqu8dC^AJ7t00IP1Fuf9go&*fNvrk0fK#KUBEOkTdHstq_QeOvtmE4bI@@odg1EUDJQLRY~cCGNiFBXVBHp_Z3J~sNXkH_~F~&6wXVLo-dM*ND_q!c0a98J&NkbbuNdfVq4T@8es3qLIPF4*RV@vYjR~NkiQcaBf1CIOY6lczE%fslmjMJKca_x3Hovtd4oPGhOOEwm63TFx@YJMn@21*MGk_60-8-jOi#@qytjxGrdrewet66!Mm74DuxtqQr!2@ku3iktxORbE7GKT~SCZWj0HL~$EQY1c5vYduZ_77Pqrh2Mfru93KcYyE$-QZfMzxBwrKLfJaHh!uXZYfSoTv_GFYYTENDGjZAZlWOc81g4%cPTg_fQ#-JzzS0NqO4nJ1jF95yJ42K8D@cd%%DCDZ~MaLJshEhnmEpSv_y@55M%OWGorW6PjOyC4r2!nLbRFSFdRdu2nUT~^o3OE7PIxt1#tZjjMT3uh9ZSzXxlUrmKO*&q@D-RNnhdNmrQk1T!uKQC^xX!~r2lh!~DhJv73j-8yPf5g8b&62A0tDhup0qCA7ITH3296H4ZvBLBKWwRZT6MgYbYdXASbx!A^9tCWPHHnBpP0SU#rht1gsC4xk@T9*Jd9BHNlquFfQ5NPU191lZ22pUu&G~nPmB$zW9K4vbOcYqV8cp$ShUOu92KW&K0fsRkckkSNgJNE3AJmBmMtr@TuADy-OK&^G7RNVe31nqD*bsvQI-~hcb*CnVuf!HMBFaxfEapVsAMTMf*qGg5PdzKUMDl#xSP*n9A@9%ip&eZmk-605hHeRHarPgUdQEqimpzsRZL0^eEoNCxx*5@zTXQo@*kbuh0c*mybPBEh1tKi&kJi76hRqqaZaP#tBsnO2rzYM!MHa$QZfs9Nd@VgOBo&7tvvUQtGhTVUr&ulSGG$d%*bXQjzyOfXmAtho-SH*2!1#9hY6dQ51eUtOMcKNA6-9OAI_gmPBRBuKRKJyW2A6Iyr0vbkp-87O7_lmst15jDf&0_M_smSAyvgL%n-q3K9AVpxLghgld9$iMfrB@d8N&33wmGDcXt@NXk~7^&LUuGeT15DauP&Bj0014LOg$vkm9U6P#PyL@m~Qtbr79ULoT^hSJBMZWWzjp$F-$%mY0EPghUAtYlAeq#^II4_fJNZmJGc%d-h1$Du^n4uGa@**0uMX_D@sb3Pcuzb3tQ0AhZcQNa0iMQc5DOV~yF6VJ8&qhnVXh$68pTO4qxrW&LNDbex%_LigONjXNPuPbsbbWUsUe1P^YbKjpy0JatODmHSNZ1IF~AA-uhc6q~n7UH@tATI%~qkdFO4ch6onp8F%&9-wjUAseD5xlG138m~5nMcTnE0qaHx25bOd~emMw3ZqSyRtxn~9~ACCd6sl4wOHuzQlALJZBo^yZzcx_lyKt0b^CDBME5nF81wwTN39Zk9qL*F%&&nBO5xq&uNomBszQsKcQJx*e2G99gK@IpzQk!1%Eoc!oV2iex@yfcl-z^_z!aS7aJDVm%r28o3LEDn-yQvv4i~r43_O4!LcNJIKvF63wNyhiM4EAOHmkjs#i^r3t1#7MjaU^@zLFhCMo1*5Y2d&jjZoqnSrEj^js#~TCWbKi#2xrZA4m&~u-!z~15aX4c8E7qhTle5-b-Yz@Q88i&Kn0Aaa$*sQg2jBbXTZg6XzD-2Pd__cg06AQ@zE#Obwi-CdtpvJW~g7SEaOz$88D0RQun4d~F7k5Yw*CstxYJseCm6d3r&NuzIh&awT2hWw^qPuCvIqHrx~l^RBkTGOVRPdrJtWqALQ@NIc%480&!02R92!vNEyOfm^M&4BoAGdpxo3Y^Pg0R6h2MX2_MHhnwBYvh20c72D$iU-UC5-$3S%yJQri6@OTXcz7HYRqas1XQrE3dlm%7MZY!pKXJ$TmDFzTlKoHS8JXdP*oFyG1Lze3P_jHgidsQpypp^w^*hf~EGXsD0I0@$IzFj3-_wKp%xRCy^O8oKs5kPJ~cT99zBBkl^Qm&PGp&YtI3Y5rP7#Y8qdqeD~3f45QVBAo-S!m-BIBfNCrTr~UJy_POWT!$sW~DZrH41aDfTlyhNyk2HI0Ks%vYs2ixpat%mjXcpaOd2O&WS1@kPE!8#eJDU5o4%VQUz@%f4ivTqk~zY4zBtUO2XRp_~RbbpqS@^yCetTy4_X8&#t1H~f406Z^*Nha-PTXl8cgXXxWpvkCf8V9$AiG%igkk2~WYF~$VLduZVGB-6F4eEjmuWc8JJnIx$k^dA-BFdzvhpc-FI26Cdr7ljwUZ*zhdNRwHqIlBLdWXYbocGAZxaFR#TeKi^1cB%1S*ayqup&6hKu_mBSqyWb7rLSPI3GQJWjVSr~UO*CH%vf@WKH~RBJBv_TRIFPz^yKA$DU^68*yoK4UusI3z9Ipq3zY7OSjSefJ4mMTzg~BKDEUhffP6nIFB%M!6^qfu5Wpp@coOGEcIuG#~tUf@VGQ6vPN8CY96ZHE6bnbJ@&!^Mm3Ouci3Yv*eA-MvqP9Mes$xGe-vjEc4^zUnM*iO$J5#Vdsj25*FyES#~AkVQszi9Lgd4OV2ztzSLdwLibs_TF@uQ!*KM7xS!&ty^6DCkUpSdtJP7bW!7mVPVYOTt^1~VI9d^&qetbkwkeIjNMZ4nzDl$4eIvDhE7vd@yNy7fgaLEfVuS1&_1P%LzuqRpr%t*oE*rCr1LQd*0Jpbh1f6-v48rHTe!tfcGFXGT9XQY^ZKH^qxEZ&uW38sgd-5R0KCjb2S*51-itFiWFM6ZyeQu$G2d3Q$j_~0fU8p8e-yMOzkofq!g&i-NPYgZu5#3DZLrb0~sXyY7sCEqaJ-IHfL!&hj~H5D4WYcHN%6rv$@Zl%3VuD5m!frKuZVEP~pL*&rZbtDD^YMp7q3Qo^WFzZRdT4QItNsHqe#rPTI7wJ0U8bj8YeqL5y&!pnKVnDUFmrP$3du8nZsT$M9YtVD^$pVdc##w^ksSYmSVd_Ff1$w&P4Bl3&t$4HZ682!oGc4Y&jVwkj$d%OlhotakyprCgCQebIp~$m&5k0Hu2lG~xf#t*n5~sGf@51_owF1c1PKunwMc%F%Qcjs01H1!mR*bEN-0O$UN3vznFkaMuoR5zF2#Ct~6aC*Xk*w9Tq&ngW_#airnTLt0WG8ReiMyA*s%*nD!7&u*4_CW3NbRoJVpIUCjh6&uV~1bxp4EjxmOZ_im~zwXvhCrjHVZ*paoaA6c686bgoUpz1$#uuEwxEQrRvjC@x^W$O%GE_P69--RR^ywwz01tTkeaHnp#zpY^McD-VzL78QLRr0Iy_770sa*i^e1f3x8$LsQo591foQ-!4lv~iVFE&@ee@8#oftHQsCxy7J&9g-9jZe!8xCTyM73p2LQnegsXlNcmgbLVY95WuE~y*sRLhVhR%VEheqEfdO^poi1Qhe0xK_1Q3L6B8p$@ew^9udr1eeVJ&edsOtUO*n2T!Fq9qsD3!sB@K8PxDj!$W~^JZpAm-5d2zt9ielNtuyv_V53OWWSypnr0z8hRuo$LKlpz~eMP~zLQ2sc!QxI-C_W-!&$&q#r6y8HvyLM0By0#*w&vfdRN_LsvmOXfpnRF3ipU@sWI5_-h9rP0mdRk#QIB6VeNOmRB&G31-1mUY-HYeMVCCuZovKf_FVi#z0dgKrVYnCHQc19VF@Fc%QYuQZegqVMwilt2cbYHtfg*11vttIC0Hr~Z_Z~#uNdkG#5#A^&&xMneHLH@MrU5H~v7Bh4qcvn@Gfw0wH&vWDD0r#BY@&4to5Bp4_rY0WL30!dXEIQU1zoz7k~psB-Ko8PADG4lY_gKXS%pDM!mR*9$$35rbS2Kgz2XoK_~_zzovOLa-V7eKN8B%4oxK278MI8MoES@_S$KHuJaMjmOGD2sXLMsmT2q1$C9MQh!qsilCaTSWzEURTOzf2LA&!Wr39k5Y11O!xjPcOjVUdf54I1vDS3cR0-*S$U~tLtBY_SwS#QcDGYIIGTODWUAuXwwzaFZkIcJ5~~&GEsx*fAwXqzTRNW~vlEj!VnUB5myRa9*mUOO^I871#J68E&dzp_HBeP-E@P9nzjC**j&wa6XCC8MvDBoXYDj~oqisy~RhpBDyU8PjV&Rdk4$uD-qD6J$jXK$rWgsqricjKATfBnhf5Td28UWQNF1IpFmu2GraHH5EnT$J#q&LH2xeiYLX6Hqw^W#*$@tk~fqOICUe7a_DraRdiUE$2TlXOmomz2QD_m0@WZ&zMrZg7!I38Eu5snvCcpa&&e8ZKJnnE*RY-_O#nIY#~4FI3-e5gn&@%jM7zW!eNx&&4g6U5v~jXpwgy^KS3UcgWyS5u0w#GC3cmPtQ9^QBgXMpHppwJ1zf0-mD@Qd8RPU^CXwG!&fBTCVS%zKa1-fDyG6*&s%tuqHz~ky2cC2S5H^Y#U9iY2COvvbDjWPo445bcSl6ku-R#yo7u3#Rx!Xl0^$nBcIEHW9NdP-_eqp%hHnFmAhLaR-P~Ox-nHd#DICdhSxuFi^LX*DtS4O2EJzgb6bi*6gq*LIJ4h^4FCpvI-RFUf779QsVZXNf%3QJO6aFK~9PX%%kkNKMP0Nc#Mf2^cu*46*ni!E*Okd8La%!JDpKs2$wx#0#kuFkEh#k1Cv5i*#6ww%KnH6oc4B$xa54Gr!!&kW$R_YVPajOjTuTQz2^Hg^MYE2NWEG4~AYs#JY4i1FJEL-kKXjg#zx0mdyX8KuBP^@#s!1#1-N*j_oB2TZEC*fTnA6pu&areSJpgOO~uQKAcyURV&$cp^SMe$*RGILWlth7S^It8Vyp3_xKq3yrbOKjfiyd$sQQFYv&4JJ1#kVCZ4Ihof9iZa@iG26eWkc!NrPwEjqO8*N1ZvriP%jGyX0hsSiT%xC1Vfy~3uyjDMFJiY*Vc9*gC-P^#Mcjm%2ohcvNhME6fx*4f9fThRijQb11J@MzqHpcXHYSibWJrEiBSouw$!98PoEfWfJs_xejA^TL7h%Yj$6GhLCvs30BvprqB18^OnH3J%w9IfY4HUR@6W_6EmEVNzGUy7@HeNFKIJy@6r*W6l!jslHoI*d8hpY4qpFQWWQFM09Ev6#rA7#VXfWiKFq!XfhSTYG~a~9Zjb6*4xbNIn9o&w9mM4zLouHO6a^1z*k7FyQ&EFguhgwLh7DC@6pAGVOw^bG#h4&aYqyzJX4XM%xPB8a3SdVZ-N-#d9doRj#bBAvUvydxcsDEeML8YRauL4q0p-JLT$n#t2oXT93Ge2b_52L67455~*bcH8aNG^7AqdGPO_adaGHgbP^3A8^5JGyiHf&PNQ^gbDNgvk1Z1JM8_e4jXGy~6Ja3PNIahqjbqreWUtb66GfLMr$HLwbfUAI@@boQsHQI9x4Z_Zal#^u-X-n_x-mjW~*LH-bs936MmqRvAuOcM~cXSNiZl4&8k9CN!gH_5hv*Ok6*ZQ28s2bq7*RE*zLw_TGJOBpKBqsDr!_YNHwRW8~K_XKitndhpo2P-hiMW9kftc2~WgJPyuoY&f6y*hjg-3aR&zA#eVlgE8#C4uDB7*oRx!o5$F@-AkAZeqENIW*QIu5LYSyhgOu&b30i7lERhtmJLRJgfQ90yiior7hIc_QYnyWNOX@Yd&2*wM6Tv3R%MfN^foK1QV4QOUWA_**rH6AxMAbrrB2$xgQPZl2X!x~7z4yOTsHm0uV~mwU3cOn@kxVNiaisdSAEYBl6JQ_DyXXpe%O!2ag#IxYOXWC*S#BURDR0JCHafpJ&4AWtpsxo4dGoaNZtp3J$Ch^m$~g#r2u$plnsU4ruc8HfF47V1mxllNnfyRqF3UX#~h_@mc$1DoVyPM9PH_7N9^Nz7n8bKrSrqpNyck@4Yv1I547vhzateMH!4W8%ugbqdN0&9A#nxwchqZN2qhYMEk#dD0WOug#yhvrc-f&c&4O7v8nmcvm6-3cOPl2P5PcDc6bQixVNpvp*4IALVWGaWk$rTRT8U2ZYWSkkL$BZD--&F%iIh5K0aROVIjCOhU@k0-@cXeYx&fMjw7Md#kBb6CYg#t9T@k3PQY7b7tJlse@qWlf~knivM3sx-II1T4r_VwF#_pJOYPYJqTav19hhg@pIIDxdMkZc3Ig0Fgr9B9VgpweaspyV5B5PfBb#p@8SGE$hP@g6TVjhh5!IIafDx$L0NzYOii$~ZRxxBfeS6uCnIq5vF!iiIgdVhkj-z-nXObVOa15P*S9$5yMKOUQK$H$zIENk30%m@n3svUY_NBnY!gtSCJ8E4IKGxlaFPdsxZ9Mhzzy%p!wz5nxr-zk7AaV@Mz#7rCRhTnsXCs5YFST7TWO~s_E1jVC^^%~G7xiN9UoH8@tCn#*dmbHXxiMuCZau_rFHehKN2Ke#!_V_khRSsxTdAK-!FcmPxa%pdjV~3U5H-nzR*CMzPBN#wxd03t9y%E~cx1~$5g!2JTv^Ahh$EraTDW&J92Oi~XKxKjuGEiPR^x8~sTo^yOqIy9&f#_m8cQrKQsSBaWSqN~!w#H5gipIx@QZqQJ_ALspwIlrhjaK%Y46iAgbze$J2x6M^HPuLJ^QztEalfY@uJz*o32WrJhX^A6A4i@SH!&XXxK7JIJE5NWaZOcPhzXxOFc!BsdAKJ32Yp!E1QQZLNeZNvAlOeN9Me!Fiq~-YnUUVjSYLn*J!DSDOO$t%cW#Rg4lzaGzK9ujasGv^pbaL%wyJmySxcyF1N^opMc%O!502A^eflQx5p~Oad*mQ9QbkSS0K60N4Hsfw&Qw7gpDKMpjW@Wz1ORA65ay_O9Msli&sQY9L#0$JTxD^Y~OX0axs&Wlf_xLTUy4t5%q0EG3Zj#q_NYV4wzh3^r!BK~XoClS~C18aYiHiPeY$-Y6YkF$lHz8sVp#so42T3YcP4tEI4-_FHVIwZ-g-x@p$YpCr6SwZU*A2J$2!$GHskFa3Z#Vh&m8RK1J_e96Q-Eq63vt7dGoZHKUT0Q!jq5Be_$UMK5Y$EDcaaGCXWaTH0EdZ-tj1nj_HhRTU$uH^hj_0GHaRE6yz9ftK%@@9@*Sn%*6z8G-yxjr8M1S5kwcK_HdZDzn-eMBJcG^h5qEGOo8DK8#H_7pUxvO_rUBoch0en15*NY!jHXM9wzLUVdqtmh!*L^KKR#8qrjJ%PxDgRB1BXyp$UDHofwmfMS!OzkfCt!bVfZ84d@nf#G&h7L!1S!@spMVXa^EuNv~f2H5Aisni6ZPJd-bb1*QHI_&Xxiy8X5ru74I2VeF*kSFE2j%!l7fDRNib4R_3q$sVHmlAjreB^4gp~kviNg5JB5Q5sAqA#JpsD#qom%QQr$FdR!CENmOojI&XaNd7rO76RL^dh3Eiq2CJiPc*3ACLYwh*KZMBbfN6VurMdOxTC#nfZPiTckaA8y68AszOd7YbPZpL7vPlx4rfs&mZjDcR88WV^DJUpHU71rp*Pw@SHbp$wG3K*m6nwnT8-ZO18qOY%nfL8&%E~2#EzYa$JEUhxLXN*aVbvcw920oO!VeTh2j1laX6v%XX8DLdRH_tHbpyh6xvHl76WseE#KB@P97o4&!uMkDdhK~raDnUOkmUG26#3@uZZV!jLcIHG^dASqH~mpThk&Y$PA6ZFuIa7S5YWrVCdDSb*kAo*BuyzEF&N&DD0F9B7z_P%npKtT7LQGdR4Dkl5MTo8Al37Qiv#VSfxuKop-GmTMz-ZMTPxf7-O8SSbUgyNkt8ZnBhx9oP3wNuqEB9cI-w%~3aZiUaVAvC#-v_N0u-V&FR6eaJMAV3PoZcTRK8CjMQiKhPDZ%XnY$3qR&C!OEK!JUebDgys80ZLZ5nWxRVgAXuK$f*I!#bYhXGzQ5kWLDa*j5%Fm_T8&Ux8n*msB^NhpPK@1avNLQKJa28PH%Xd39#*Y8v!s7IMAhFvkUFO3ly#2lmwNvIvFmRmF_@NVak6iMOmXs2kS$76BevXio&A8j@8tC&4UrYPOaHzn^XCAjct*_&VG-y^FqWp^5$s3A61gMkQ6!@-UqwESXe2utCOU~Ain%XdUQ^YWfcCgC^b#6Qs$IUn1gxIL72ckNbW5&yRRL9&%-GImO9wmrn8WrfVvZKiipmlfS0TlYTGbDwzJl7VFNn&4ytFjIt2S2v96q%Asey~dAkPD9nqt^UdlUCML#sNTNonjCjH$8BmcovVsT8Ag7vANkz-ww&SJY6~Hk5CexGxoIc!RoO^CV2rtBU~wS&PuKPX7iEs3Vepl*vuiST4u%y7ItR2o&y@%KZizVVdj7-o2U$peKyYZmM34S!lly#$bzSpzp%OC!tj0RTkR6UQq65u0#Mjo4VscxOPi0TCFeJnQhaM3tTIVB%@4Y6Xkb!9o6I1nuFpcWWbrO#0%B6Ov~GDHIosyv*q$Iy6ru6*fwj!ba8bCRgS%gCGTD%zdHS#Qp~zkfG*1wJiIi4u14i$Dd*si7!1VblHC5dJfRxMX_4&8f0&4WF3i1*orucNXS2WAQiKdL~3$U*FIm7Ky^XuIE^zpP$*OoGU2@q!yfHASsXTRQ2~^mQObTvNDJ*5TTMJB4R_YoMltN^x#-Fve85nG6a6-#zzGH@YADN1gU6zX8EglgznD2xRGv5gdxtIgNUWk89r~r!VBEA^MH2_N!~Q^p&hvWpsdkX$%mMQA9DZ*$uM5vzx89Pp^MjQalk*R-Bf!3uH4Nm#B@th~cuaVM&zOrsMuMgxt~v2Y#7oGhhoq7pKjJN@t@&5*V~rQCmV0DeY$mS1-1V$dtqXmVHMzHwct^eMODO^73B3NMJDJXGJ@%GQlEl~3f_P4l_$m401~w~wn4mPHJ%MKctx43vVlN2f@x3fFAPKBq0wSQ3MEp2~^#zUJjA%Msk-h3^CiGm%!^e5QklYqlwAVBtsWsI9sG%9jB30Ey$z-0H@13ngj!g$u6B66yx~Do7$v6g8N_fBIzZXh3$dvD9mt%brJso~AGiHtHgFS0JF*X&!9_vk5nJtOJaBQ-hHXiNAa9ooaBSX2EDOZP~3bIcUeVzk#Bi~9320M3_^4nRIPp45c*7Aqy@Jjws*!WGv8ha&Hw!tyJWmCWlH~DIg@HrJZnEqNnD%2Vu%!4mDULqpFBSotghLFqyZiLol7GmIs7qzj1jjakhgF^$MS%ia-FVp&~UJv9h_XhAIfsNslM_P4OVWqn9^o6VJdRZL@MYq~*cJovrVWPW0k0b4aCgWrGIT5Rn$ogfs*%OUi3&Ful_Rn#gh-U85ynsEe5OAVcVCiqWgL#SFuyR$xw&kl@yLagO-Ri2mp$~uG@mcLQ~wmQ2c5daujWV229Cyi-6Rq_&qL##FITesVeh18OqOHsDW!BHuclR%e0x2-%LYu8u8H8U833^jI2CKn*NJSAthYHffK@t&fEd~OQ&FSEgD9sGuf-#bkPJYqBXdGWR1vRX3Yhwssn_%qd~khls6ff#j@FDjtX2cBvd-UP8OcI52DG3~ZJ*a53HBT2AETacG5fODRpCEd#o5e_%10Yd^pu%gt&$9DG!k8SZSds8I*R!xr diff --git a/internal/media/test/test-jpeg-blurhash.jpg b/internal/media/test/test-jpeg-blurhash.jpg deleted file mode 100644 index 6b6ba472e..000000000 Binary files a/internal/media/test/test-jpeg-blurhash.jpg and /dev/null differ diff --git a/internal/media/test/test-with-exif.jpg b/internal/media/test/test-with-exif.jpg deleted file mode 100644 index de56cd654..000000000 Binary files a/internal/media/test/test-with-exif.jpg and /dev/null differ diff --git a/internal/media/test/test-without-exif.jpg b/internal/media/test/test-without-exif.jpg deleted file mode 100644 index 274188ee7..000000000 Binary files a/internal/media/test/test-without-exif.jpg and /dev/null differ diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 9c7f0fe67..5a9382ed6 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -85,7 +85,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) suite.httpClient = testrig.NewMockHTTPClient(nil) suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db) - suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index e0dd493e1..6d15b5afb 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -172,7 +172,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } - return processingMedia.LoadAttachment(ctx) + return processingMedia.Load(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, @@ -214,7 +214,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } - return processingMedia.LoadAttachment(ctx) + return processingMedia.Load(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 093a3d2be..9df5c7c1f 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -60,7 +60,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, err } - attachment, err := media.LoadAttachment(ctx) + attachment, err := media.Load(ctx) if err != nil { return nil, err } diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index 04793898f..851d3d5fb 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -47,11 +47,11 @@ type ProcessingStandardTestSuite struct { suite.Suite db db.DB storage *kv.KVStore + mediaManager media.Manager typeconverter typeutils.TypeConverter transportController transport.Controller federator federation.Federator oauthServer oauth.Server - mediaManager media.Manager timelineManager timeline.Manager emailSender email.Sender @@ -216,9 +216,9 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { }) suite.transportController = testrig.NewTestTransportController(httpClient, suite.db) - suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage) - suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager) + suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.timelineManager = testrig.NewTestTimelineManager(suite.db) suite.emailSender = testrig.NewEmailSender("../../web/template/", nil) diff --git a/testrig/federator.go b/testrig/federator.go index b9c742832..1b5e0fdc5 100644 --- a/testrig/federator.go +++ b/testrig/federator.go @@ -22,10 +22,11 @@ import ( "codeberg.org/gruf/go-store/kv" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" ) // NewTestFederator returns a federator with the given database and (mock!!) transport controller. -func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore) federation.Federator { - return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaManager(db, storage)) +func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager) federation.Federator { + return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), mediaManager) } diff --git a/testrig/mediahandler.go b/testrig/mediahandler.go index 046ddc5be..38190eca3 100644 --- a/testrig/mediahandler.go +++ b/testrig/mediahandler.go @@ -26,7 +26,7 @@ import ( // NewTestMediaManager returns a media handler with the default test config, and the given db and storage. func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager { - m, err := media.New(db, storage) + m, err := media.NewManager(db, storage) if err != nil { panic(err) } diff --git a/testrig/processor.go b/testrig/processor.go index d86dd8411..cc2c3605e 100644 --- a/testrig/processor.go +++ b/testrig/processor.go @@ -23,10 +23,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing" ) // NewTestProcessor returns a Processor suitable for testing purposes -func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor { - return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaManager(db, storage), storage, NewTestTimelineManager(db), db, emailSender) +func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager) processing.Processor { + return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, NewTestTimelineManager(db), db, emailSender) } -- cgit v1.3 From 113f9d9ab4797de6ae17819c96ae866992214021 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 11 Jan 2022 17:49:14 +0100 Subject: pass a function into the manager, start work on emoji --- internal/api/client/admin/admin.go | 2 +- internal/api/client/admin/admin_test.go | 123 +++++++ internal/api/client/admin/emojicreate.go | 4 +- internal/api/client/admin/emojicreate_test.go | 50 +++ internal/federation/dereferencing/account.go | 15 +- internal/federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/media.go | 17 +- internal/federation/dereferencing/media_test.go | 20 +- internal/federation/dereferencing/status.go | 4 +- internal/media/image.go | 108 ------ internal/media/manager.go | 81 +++-- internal/media/manager_test.go | 32 +- internal/media/processing.go | 256 -------------- internal/media/processingemoji.go | 382 ++++++++++++++++++++ internal/media/processingmedia.go | 411 ++++++++++++++++++++++ internal/media/types.go | 12 +- internal/processing/account/update.go | 95 ++--- internal/processing/admin/emoji.go | 34 +- internal/processing/media/create.go | 31 +- internal/transport/transport.go | 2 +- testrig/testmodels.go | 10 + 21 files changed, 1164 insertions(+), 527 deletions(-) create mode 100644 internal/api/client/admin/admin_test.go create mode 100644 internal/api/client/admin/emojicreate_test.go delete mode 100644 internal/media/processing.go create mode 100644 internal/media/processingemoji.go create mode 100644 internal/media/processingmedia.go diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index f8ea03cc6..f5256c996 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -58,7 +58,7 @@ func New(processor processing.Processor) api.ClientModule { // Route attaches all routes from this module to the given router func (m *Module) Route(r router.Router) error { - r.AttachHandler(http.MethodPost, EmojiPath, m.emojiCreatePOSTHandler) + r.AttachHandler(http.MethodPost, EmojiPath, m.EmojiCreatePOSTHandler) r.AttachHandler(http.MethodPost, DomainBlocksPath, m.DomainBlocksPOSTHandler) r.AttachHandler(http.MethodGet, DomainBlocksPath, m.DomainBlocksGETHandler) r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go new file mode 100644 index 000000000..da5b03949 --- /dev/null +++ b/internal/api/client/admin/admin_test.go @@ -0,0 +1,123 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin_test + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + + "codeberg.org/gruf/go-store/kv" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/email" + "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type AdminStandardTestSuite struct { + // standard suite interfaces + suite.Suite + db db.DB + tc typeutils.TypeConverter + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator + processor processing.Processor + emailSender email.Sender + sentEmails map[string]string + + // standard suite models + testTokens map[string]*gtsmodel.Token + testClients map[string]*gtsmodel.Client + testApplications map[string]*gtsmodel.Application + testUsers map[string]*gtsmodel.User + testAccounts map[string]*gtsmodel.Account + testAttachments map[string]*gtsmodel.MediaAttachment + testStatuses map[string]*gtsmodel.Status + + // module being tested + adminModule *admin.Module +} + +func (suite *AdminStandardTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *AdminStandardTestSuite) SetupTest() { + testrig.InitTestConfig() + testrig.InitTestLog() + + suite.db = testrig.NewTestDB() + suite.storage = testrig.NewTestStorage() + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.sentEmails = make(map[string]string) + suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.adminModule = admin.New(suite.processor).(*admin.Module) + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") +} + +func (suite *AdminStandardTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func (suite *AdminStandardTestSuite) newContext(recorder *httptest.ResponseRecorder, requestMethod string, requestBody []byte, requestPath string, bodyContentType string) *gin.Context { + ctx, _ := gin.CreateTestContext(recorder) + + ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["admin_account"]) + ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(suite.testTokens["admin_account"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["admin_account"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["admin_account"]) + + protocol := viper.GetString(config.Keys.Protocol) + host := viper.GetString(config.Keys.Host) + + baseURI := fmt.Sprintf("%s://%s", protocol, host) + requestURI := fmt.Sprintf("%s/%s", baseURI, requestPath) + + ctx.Request = httptest.NewRequest(http.MethodPatch, requestURI, bytes.NewReader(requestBody)) // the endpoint we're hitting + + if bodyContentType != "" { + ctx.Request.Header.Set("Content-Type", bodyContentType) + } + + ctx.Request.Header.Set("accept", "application/json") + + return ctx +} diff --git a/internal/api/client/admin/emojicreate.go b/internal/api/client/admin/emojicreate.go index 882654ea9..de7a2979a 100644 --- a/internal/api/client/admin/emojicreate.go +++ b/internal/api/client/admin/emojicreate.go @@ -31,7 +31,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/validate" ) -// emojiCreateRequest swagger:operation POST /api/v1/admin/custom_emojis emojiCreate +// EmojiCreatePOSTHandler swagger:operation POST /api/v1/admin/custom_emojis emojiCreate // // Upload and create a new instance emoji. // @@ -73,7 +73,7 @@ import ( // description: forbidden // '400': // description: bad request -func (m *Module) emojiCreatePOSTHandler(c *gin.Context) { +func (m *Module) EmojiCreatePOSTHandler(c *gin.Context) { l := logrus.WithFields(logrus.Fields{ "func": "emojiCreatePOSTHandler", "request_uri": c.Request.RequestURI, diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go new file mode 100644 index 000000000..290b478f7 --- /dev/null +++ b/internal/api/client/admin/emojicreate_test.go @@ -0,0 +1,50 @@ +package admin_test + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type EmojiCreateTestSuite struct { + AdminStandardTestSuite +} + +func (suite *EmojiCreateTestSuite) TestEmojiCreate() { + // set up the request + requestBody, w, err := testrig.CreateMultipartFormData( + "image", "../../../../testrig/media/rainbow-original.png", + map[string]string{ + "shortcode": "rainbow", + }) + if err != nil { + panic(err) + } + bodyBytes := requestBody.Bytes() + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPath, w.FormDataContentType()) + + // call the handler + suite.adminModule.EmojiCreatePOSTHandler(ctx) + + // 1. we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + // 2. we should have no error message in the result body + result := recorder.Result() + defer result.Body.Close() + + // check the response + b, err := ioutil.ReadAll(result.Body) + suite.NoError(err) + suite.NotEmpty(b) +} + +func TestEmojiCreateTestSuite(t *testing.T) { + suite.Run(t, &EmojiCreateTestSuite{}) +} diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 27d9f39ce..b9efbfa45 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -119,7 +119,6 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc } else { // take the id we already have and do an update gtsAccount.ID = maybeAccount.ID -aaaaaaaaaaaaaaaaaa if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil { return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) } @@ -252,13 +251,12 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data, err := t.DereferenceMedia(ctx, avatarIRI) - if err != nil { - return err + data := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, avatarIRI) } avatar := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ RemoteURL: &targetAccount.AvatarRemoteURL, Avatar: &avatar, }) @@ -275,13 +273,12 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data, err := t.DereferenceMedia(ctx, headerIRI) - if err != nil { - return err + data := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, headerIRI) } header := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalInfo{ + processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ RemoteURL: &targetAccount.HeaderRemoteURL, Header: &header, }) diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 1e6f781b8..800da6c04 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index f02303aa1..46cb4a5f4 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -26,29 +26,28 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalInfo) (*media.Processing, error) { +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) { if accountID == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") + return nil, fmt.Errorf("GetRemoteMedia: account ID was empty") } t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) + return nil, fmt.Errorf("GetRemoteMedia: error creating transport: %s", err) } derefURI, err := url.Parse(remoteURL) if err != nil { - return nil, err + return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - data, err := t.DereferenceMedia(ctx, derefURI) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) + dataFunc := func(innerCtx context.Context) ([]byte, error) { + return t.DereferenceMedia(innerCtx, derefURI) } - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, accountID, ai) + processingMedia, err := d.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) + return nil, fmt.Errorf("GetRemoteMedia: error processing attachment: %s", err) } return processingMedia, nil diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go index 61ee6edb6..26d5c0c49 100644 --- a/internal/federation/dereferencing/media_test.go +++ b/internal/federation/dereferencing/media_test.go @@ -20,6 +20,7 @@ package dereferencing_test import ( "context" + "fmt" "testing" "time" @@ -44,7 +45,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { attachmentDescription := "It's a cute plushie." attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, RemoteURL: &attachmentURL, Description: &attachmentDescription, @@ -53,7 +54,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { suite.NoError(err) // make a blocking call to load the attachment from the in-process media - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) @@ -118,18 +119,21 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { attachmentDescription := "It's a cute plushie." attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" - media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalInfo{ + processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, RemoteURL: &attachmentURL, Description: &attachmentDescription, Blurhash: &attachmentBlurhash, }) suite.NoError(err) - attachmentID := media.AttachmentID() - - // wait 5 seconds to let the image process in the background - // it probably won't really take this long but hey let's be sure - time.Sleep(5 * time.Second) + attachmentID := processingMedia.AttachmentID() + + // wait for the media to finish processing + for finished := processingMedia.Finished(); !finished; finished = processingMedia.Finished() { + time.Sleep(10 * time.Millisecond) + fmt.Printf("\n\nnot finished yet...\n\n") + } + fmt.Printf("\n\nfinished!\n\n") // now get the attachment from the database attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 041cfa6b4..004d648b5 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -394,7 +394,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalInfo{ + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{ CreatedAt: &a.CreatedAt, StatusID: &a.StatusID, RemoteURL: &a.RemoteURL, @@ -406,7 +406,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. continue } - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue diff --git a/internal/media/image.go b/internal/media/image.go index 074dd3839..a5a818206 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -27,7 +27,6 @@ import ( "image/gif" "image/jpeg" "image/png" - "strings" "time" "github.com/buckket/go-blurhash" @@ -52,113 +51,6 @@ type ImageMeta struct { blurhash string } -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string, ai *AdditionalInfo) (*Processing, error) { - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - extension := strings.Split(contentType, "/")[1] - - // populate initial fields on the media attachment -- some of these will be overwritten as we proceed - attachment := >smodel.MediaAttachment{ - ID: id, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - StatusID: "", - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - RemoteURL: "", - Type: gtsmodel.FileTypeImage, - FileMeta: gtsmodel.FileMeta{ - Focus: gtsmodel.Focus{ - X: 0, - Y: 0, - }, - }, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: "", - Processing: gtsmodel.ProcessingStatusReceived, - File: gtsmodel.File{ - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), - ContentType: contentType, - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeJpeg, - UpdatedAt: time.Now(), - }, - Avatar: false, - Header: false, - } - - // check if we have additional info to add to the attachment, - // and overwrite some of the attachment fields if so - if ai != nil { - if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description - } - - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID - } - - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash - } - - if ai.Avatar != nil { - attachment.Avatar = *ai.Avatar - } - - if ai.Header != nil { - attachment.Header = *ai.Header - } - - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX - } - - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY - } - } - - media := &Processing{ - attachment: attachment, - rawData: data, - thumbstate: received, - fullSizeState: received, - database: m.db, - storage: m.storage, - } - - return media, nil -} - func decodeGif(b []byte) (*ImageMeta, error) { gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { diff --git a/internal/media/manager.go b/internal/media/manager.go index c8642fcb4..e34471591 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -21,9 +21,7 @@ package media import ( "context" "errors" - "fmt" "runtime" - "strings" "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" @@ -33,15 +31,17 @@ import ( // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { - // ProcessMedia begins the process of decoding and storing the given data as a piece of media (aka an attachment). + // ProcessMedia begins the process of decoding and storing the given data as an attachment. // It will return a pointer to a Media struct upon which further actions can be performed, such as getting // the finished media, thumbnail, attachment, etc. // + // data should be a function that the media manager can call to return raw bytes of a piece of media. + // // accountID should be the account that the media belongs to. // // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. - ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) - ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) + ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) + ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) // NumWorkers returns the total number of workers available to this manager. NumWorkers() int // QueueSize returns the total capacity of the queue. @@ -101,49 +101,52 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { return m, nil } -func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID string, ai *AdditionalInfo) (*Processing, error) { - contentType, err := parseContentType(data) +func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { + processingMedia, err := m.preProcessMedia(ctx, data, accountID, ai) if err != nil { return nil, err } - split := strings.Split(contentType, "/") - if len(split) != 2 { - return nil, fmt.Errorf("content type %s malformed", contentType) - } - - mainType := split[0] - - switch mainType { - case mimeImage: - media, err := m.preProcessImage(ctx, data, contentType, accountID, ai) - if err != nil { - return nil, err + logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue()) + m.pool.Enqueue(func(innerCtx context.Context) { + select { + case <-innerCtx.Done(): + // if the inner context is done that means the worker pool is closing, so we should just return + return + default: + // start loading the media already for the caller's convenience + if _, err := processingMedia.LoadAttachment(innerCtx); err != nil { + logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", processingMedia.AttachmentID(), err) + } } + }) + logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue()) - logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) - m.pool.Enqueue(func(innerCtx context.Context) { - select { - case <-innerCtx.Done(): - // if the inner context is done that means the worker pool is closing, so we should just return - return - default: - // start loading the media already for the caller's convenience - if _, err := media.Load(innerCtx); err != nil { - logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", media.AttachmentID(), err) - } - } - }) - logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", media.AttachmentID(), m.pool.Queue()) + return processingMedia, nil +} - return media, nil - default: - return nil, fmt.Errorf("content type %s not (yet) supported", contentType) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, ai) + if err != nil { + return nil, err } -} -func (m *manager) ProcessEmoji(ctx context.Context, data []byte, accountID string) (*Processing, error) { - return nil, nil + logrus.Tracef("ProcessEmoji: about to enqueue emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue()) + m.pool.Enqueue(func(innerCtx context.Context) { + select { + case <-innerCtx.Done(): + // if the inner context is done that means the worker pool is closing, so we should just return + return + default: + // start loading the emoji already for the caller's convenience + if _, err := processingEmoji.LoadEmoji(innerCtx); err != nil { + logrus.Errorf("ProcessEmoji: error processing emoji with id %s: %s", processingEmoji.EmojiID(), err) + } + } + }) + logrus.Tracef("ProcessEmoji: succesfully queued emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue()) + + return processingEmoji, nil } func (m *manager) NumWorkers() int { diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 74d0c3008..0fadceb37 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -37,21 +37,21 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + // load bytes from a test image + return os.ReadFile("./test/test-jpeg.jpg") + } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() // do a blocking call to fetch the attachment - attachment, err := processingMedia.Load(ctx) + attachment, err := processingMedia.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) @@ -103,15 +103,15 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + // load bytes from a test image + return os.ReadFile("./test/test-jpeg.jpg") + } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() @@ -183,13 +183,17 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.NoError(err) suite.NotEmpty(testBytes) + data := func(_ context.Context) ([]byte, error) { + return testBytes, nil + } + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" spam := 50 - inProcess := []*media.Processing{} + inProcess := []*media.ProcessingMedia{} for i := 0; i < spam; i++ { // process the media with no additional info provided - processingMedia, err := suite.manager.ProcessMedia(ctx, testBytes, accountID, nil) + processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil) suite.NoError(err) inProcess = append(inProcess, processingMedia) } @@ -201,7 +205,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { attachmentID := processingMedia.AttachmentID() // do a blocking call to fetch the attachment - attachment, err := processingMedia.Load(ctx) + attachment, err := processingMedia.LoadAttachment(ctx) suite.NoError(err) suite.NotNil(attachment) diff --git a/internal/media/processing.go b/internal/media/processing.go deleted file mode 100644 index 3f9fc2bfc..000000000 --- a/internal/media/processing.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package media - -import ( - "context" - "fmt" - "sync" - "time" - - "codeberg.org/gruf/go-store/kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type processState int - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - -// Processing represents a piece of media that is currently being processed. It exposes -// various functions for retrieving data from the process. -type Processing struct { - mu sync.Mutex - - /* - below fields should be set on newly created media; - attachment will be updated incrementally as media goes through processing - */ - - attachment *gtsmodel.MediaAttachment // will only be set if the media is an attachment - emoji *gtsmodel.Emoji // will only be set if the media is an emoji - - rawData []byte - - /* - below fields represent the processing state of the media thumbnail - */ - - thumbstate processState - thumb *ImageMeta - - /* - below fields represent the processing state of the full-sized media - */ - - fullSizeState processState - fullSize *ImageMeta - - /* - below pointers to database and storage are maintained so that - the media can store and update itself during processing steps - */ - - database db.DB - storage *kv.KVStore - - err error // error created during processing, if any -} - -func (p *Processing) Thumb(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.thumbstate { - case received: - // we haven't processed a thumbnail for this media yet so do it now - - // check if we need to create a blurhash or if there's already one set - var createBlurhash bool - if p.attachment.Blurhash == "" { - // no blurhash created yet - createBlurhash = true - } - - thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) - if err != nil { - p.err = fmt.Errorf("error deriving thumbnail: %s", err) - p.thumbstate = errored - return nil, p.err - } - - // put the thumbnail in storage - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { - p.err = fmt.Errorf("error storing thumbnail: %s", err) - p.thumbstate = errored - return nil, p.err - } - - // set appropriate fields on the attachment based on the thumbnail we derived - if createBlurhash { - p.attachment.Blurhash = thumb.blurhash - } - - p.attachment.FileMeta.Small = gtsmodel.Small{ - Width: thumb.width, - Height: thumb.height, - Size: thumb.size, - Aspect: thumb.aspect, - } - p.attachment.Thumbnail.FileSize = thumb.size - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.thumbstate = errored - return nil, err - } - - // set the thumbnail of this media - p.thumb = thumb - - // we're done processing the thumbnail! - p.thumbstate = complete - fallthrough - case complete: - return p.thumb, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) -} - -func (p *Processing) FullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.fullSizeState { - case received: - var clean []byte - var err error - var decoded *ImageMeta - - ct := p.attachment.File.ContentType - switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", ct) - } - - if err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) - p.fullSizeState = errored - return nil, p.err - } - - // set appropriate fields on the attachment based on the image we derived - p.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, - } - p.attachment.File.FileSize = decoded.size - p.attachment.File.UpdatedAt = time.Now() - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - - // set the fullsize of this media - p.fullSize = decoded - - // we're done processing the full-size image - p.fullSizeState = complete - fallthrough - case complete: - return p.fullSize, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) -} - -// AttachmentID returns the ID of the underlying media attachment without blocking processing. -func (p *Processing) AttachmentID() string { - return p.attachment.ID -} - -// Load blocks until the thumbnail and fullsize content has been processed, and then -// returns the completed attachment. -func (p *Processing) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - if _, err := p.Thumb(ctx); err != nil { - return nil, err - } - - if _, err := p.FullSize(ctx); err != nil { - return nil, err - } - - return p.attachment, nil -} - -func (p *Processing) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - return nil, nil -} - -func (p *Processing) Finished() bool { - return p.thumbstate == complete && p.fullSizeState == complete -} - -// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, -// and then if that doesn't work because the attachment already exists, updating it instead. -func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { - if err := database.Put(ctx, attachment); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { - return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) - } - } - - return nil -} diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go new file mode 100644 index 000000000..7e2d4f31f --- /dev/null +++ b/internal/media/processingemoji.go @@ -0,0 +1,382 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +// ProcessingEmoji represents an emoji currently processing. It exposes +// various functions for retrieving data from the process. +type ProcessingEmoji struct { + mu sync.Mutex + + /* + below fields should be set on newly created media; + emoji will be updated incrementally as media goes through processing + */ + + emoji *gtsmodel.Emoji + data DataFunc + + rawData []byte // will be set once the fetchRawData function has been called + + /* + below fields represent the processing state of the static version of the emoji + */ + + staticState processState + static *ImageMeta + + /* + below fields represent the processing state of the emoji image + */ + + fullSizeState processState + fullSize *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any +} + +// EmojiID returns the ID of the underlying emoji without blocking processing. +func (p *ProcessingEmoji) EmojiID() string { + return p.emoji.ID +} + +// LoadEmoji blocks until the static and fullsize image +// has been processed, and then returns the completed emoji. +func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + if err := p.fetchRawData(ctx); err != nil { + return nil, err + } + + if _, err := p.loadStatic(ctx); err != nil { + return nil, err + } + + if _, err := p.loadFullSize(ctx); err != nil { + return nil, err + } + + return p.emoji, nil +} + +// Finished returns true if processing has finished for both the thumbnail +// and full fized version of this piece of media. +func (p *ProcessingEmoji) Finished() bool { + return p.staticState == complete && p.fullSizeState == complete +} + +func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.staticState { + case received: + // we haven't processed a static version of this emoji yet so do it now + static, err := deriveStaticEmoji(p.rawData, p.emoji.ImageContentType) + if err != nil { + p.err = fmt.Errorf("error deriving static: %s", err) + p.staticState = errored + return nil, p.err + } + + // put the static in storage + if err := p.storage.Put(p.emoji.ImageStaticPath, static.image); err != nil { + p.err = fmt.Errorf("error storing static: %s", err) + p.staticState = errored + return nil, p.err + } + + // set appropriate fields on the emoji based on the static version we derived + p.attachment.FileMeta.Small = gtsmodel.Small{ + Width: static.width, + Height: static.height, + Size: static.size, + Aspect: static.aspect, + } + p.attachment.Thumbnail.FileSize = static.size + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.thumbstate = errored + return nil, err + } + + // set the thumbnail of this media + p.thumb = static + + // we're done processing the thumbnail! + p.thumbstate = complete + fallthrough + case complete: + return p.thumb, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) +} + +func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := p.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(p.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = p.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + + if err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // put the full size in storage + if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size image: %s", err) + p.fullSizeState = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the image we derived + p.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + p.attachment.File.FileSize = decoded.size + p.attachment.File.UpdatedAt = time.Now() + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + p.fullSize = decoded + + // we're done processing the full-size image + p.fullSizeState = complete + fallthrough + case complete: + return p.fullSize, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) +} + +// fetchRawData calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. +// It should only be called from within a function that already has a lock on p! +func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { + // check if we've already done this and bail early if we have + if p.rawData != nil { + return nil + } + + // execute the data function and pin the raw bytes for further processing + b, err := p.data(ctx) + if err != nil { + return fmt.Errorf("fetchRawData: error executing data function: %s", err) + } + p.rawData = b + + // now we have the data we can work out the content type + contentType, err := parseContentType(p.rawData) + if err != nil { + return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + } + + if !supportedEmoji(contentType) { + return fmt.Errorf("fetchRawData: content type %s was not valid for an emoji", contentType) + } + + split := strings.Split(contentType, "/") + mainType := split[0] // something like 'image' + extension := split[1] // something like 'gif' + + // set some additional fields on the emoji now that + // we know more about what the underlying image actually is + p.emoji.ImageURL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) + p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) + p.attachment.File.ContentType = contentType + + switch mainType { + case mimeImage: + if extension == mimeGif { + p.attachment.Type = gtsmodel.FileTypeGif + } else { + p.attachment.Type = gtsmodel.FileTypeImage + } + default: + return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) + } + + return nil +} + +// putOrUpdateEmoji is just a convenience function for first trying to PUT the emoji in the database, +// and then if that doesn't work because the emoji already exists, updating it instead. +func putOrUpdateEmoji(ctx context.Context, database db.DB, emoji *gtsmodel.Emoji) error { + if err := database.Put(ctx, emoji); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateEmoji: proper error while putting emoji: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, emoji); err != nil { + return fmt.Errorf("putOrUpdateEmoji: error while updating emoji: %s", err) + } + } + + return nil +} + +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) + } + + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + // populate initial fields on the emoji -- some of these will be overwritten as we proceed + emoji := >smodel.Emoji{ + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Shortcode: shortcode, + Domain: "", // assume our own domain unless told otherwise + ImageRemoteURL: "", + ImageStaticRemoteURL: "", + ImageURL: "", // we don't know yet + ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), id, mimePng), // all static emojis are encoded as png + ImagePath: "", // we don't know yet + ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, id, mimePng), // all static emojis are encoded as png + ImageContentType: "", // we don't know yet + ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png + ImageFileSize: 0, + ImageStaticFileSize: 0, + ImageUpdatedAt: time.Now(), + Disabled: false, + URI: "", // we don't know yet + VisibleInPicker: true, + CategoryID: "", + } + + // check if we have additional info to add to the emoji, + // and overwrite some of the emoji fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + + processingEmoji := &ProcessingEmoji{ + emoji: emoji, + data: data, + staticState: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return processingEmoji, nil +} diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go new file mode 100644 index 000000000..a6e45034f --- /dev/null +++ b/internal/media/processingmedia.go @@ -0,0 +1,411 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "codeberg.org/gruf/go-store/kv" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" +) + +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + +// ProcessingMedia represents a piece of media that is currently being processed. It exposes +// various functions for retrieving data from the process. +type ProcessingMedia struct { + mu sync.Mutex + + /* + below fields should be set on newly created media; + attachment will be updated incrementally as media goes through processing + */ + + attachment *gtsmodel.MediaAttachment + data DataFunc + + rawData []byte // will be set once the fetchRawData function has been called + + /* + below fields represent the processing state of the media thumbnail + */ + + thumbstate processState + thumb *ImageMeta + + /* + below fields represent the processing state of the full-sized media + */ + + fullSizeState processState + fullSize *ImageMeta + + /* + below pointers to database and storage are maintained so that + the media can store and update itself during processing steps + */ + + database db.DB + storage *kv.KVStore + + err error // error created during processing, if any +} + +// AttachmentID returns the ID of the underlying media attachment without blocking processing. +func (p *ProcessingMedia) AttachmentID() string { + return p.attachment.ID +} + +// LoadAttachment blocks until the thumbnail and fullsize content +// has been processed, and then returns the completed attachment. +func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { + if err := p.fetchRawData(ctx); err != nil { + return nil, err + } + + if _, err := p.loadThumb(ctx); err != nil { + return nil, err + } + + if _, err := p.loadFullSize(ctx); err != nil { + return nil, err + } + + return p.attachment, nil +} + +func (p *ProcessingMedia) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { + return nil, nil +} + +// Finished returns true if processing has finished for both the thumbnail +// and full fized version of this piece of media. +func (p *ProcessingMedia) Finished() bool { + return p.thumbstate == complete && p.fullSizeState == complete +} + +func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.thumbstate { + case received: + // we haven't processed a thumbnail for this media yet so do it now + + // check if we need to create a blurhash or if there's already one set + var createBlurhash bool + if p.attachment.Blurhash == "" { + // no blurhash created yet + createBlurhash = true + } + + thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) + if err != nil { + p.err = fmt.Errorf("error deriving thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // put the thumbnail in storage + if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { + p.err = fmt.Errorf("error storing thumbnail: %s", err) + p.thumbstate = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the thumbnail we derived + if createBlurhash { + p.attachment.Blurhash = thumb.blurhash + } + + p.attachment.FileMeta.Small = gtsmodel.Small{ + Width: thumb.width, + Height: thumb.height, + Size: thumb.size, + Aspect: thumb.aspect, + } + p.attachment.Thumbnail.FileSize = thumb.size + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.thumbstate = errored + return nil, err + } + + // set the thumbnail of this media + p.thumb = thumb + + // we're done processing the thumbnail! + p.thumbstate = complete + fallthrough + case complete: + return p.thumb, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) +} + +func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) { + p.mu.Lock() + defer p.mu.Unlock() + + switch p.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := p.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(p.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = p.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) + } + + if err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // put the full size in storage + if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size image: %s", err) + p.fullSizeState = errored + return nil, p.err + } + + // set appropriate fields on the attachment based on the image we derived + p.attachment.FileMeta.Original = gtsmodel.Original{ + Width: decoded.width, + Height: decoded.height, + Size: decoded.size, + Aspect: decoded.aspect, + } + p.attachment.File.FileSize = decoded.size + p.attachment.File.UpdatedAt = time.Now() + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + p.err = err + p.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + p.fullSize = decoded + + // we're done processing the full-size image + p.fullSizeState = complete + fallthrough + case complete: + return p.fullSize, nil + case errored: + return nil, p.err + } + + return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) +} + +// fetchRawData calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. +// It should only be called from within a function that already has a lock on p! +func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { + // check if we've already done this and bail early if we have + if p.rawData != nil { + return nil + } + + // execute the data function and pin the raw bytes for further processing + b, err := p.data(ctx) + if err != nil { + return fmt.Errorf("fetchRawData: error executing data function: %s", err) + } + p.rawData = b + + // now we have the data we can work out the content type + contentType, err := parseContentType(p.rawData) + if err != nil { + return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + } + + split := strings.Split(contentType, "/") + if len(split) != 2 { + return fmt.Errorf("fetchRawData: content type %s was not valid", contentType) + } + + mainType := split[0] // something like 'image' + extension := split[1] // something like 'jpeg' + + // set some additional fields on the attachment now that + // we know more about what the underlying media actually is + p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) + p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) + p.attachment.File.ContentType = contentType + + switch mainType { + case mimeImage: + if extension == mimeGif { + p.attachment.Type = gtsmodel.FileTypeGif + } else { + p.attachment.Type = gtsmodel.FileTypeImage + } + default: + return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) + } + + return nil +} + +// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, +// and then if that doesn't work because the attachment already exists, updating it instead. +func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { + if err := database.Put(ctx, attachment); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { + return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) + } + } + + return nil +} + +func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + file := gtsmodel.File{ + Path: "", // we don't know yet because it depends on the uncalled DataFunc + ContentType: "", // we don't know yet because it depends on the uncalled DataFunc + UpdatedAt: time.Now(), + } + + thumbnail := gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + } + + // populate initial fields on the media attachment -- some of these will be overwritten as we proceed + attachment := >smodel.MediaAttachment{ + ID: id, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StatusID: "", + URL: "", // we don't know yet because it depends on the uncalled DataFunc + RemoteURL: "", + Type: gtsmodel.FileTypeUnknown, // we don't know yet because it depends on the uncalled DataFunc + FileMeta: gtsmodel.FileMeta{}, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: gtsmodel.ProcessingStatusReceived, + File: file, + Thumbnail: thumbnail, + Avatar: false, + Header: false, + } + + // check if we have additional info to add to the attachment, + // and overwrite some of the attachment fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt + } + + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID + } + + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = *ai.Avatar + } + + if ai.Header != nil { + attachment.Header = *ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } + } + + processingMedia := &ProcessingMedia{ + attachment: attachment, + data: data, + thumbstate: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return processingMedia, nil +} diff --git a/internal/media/types.go b/internal/media/types.go index aaf423682..6426223d1 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,6 +20,7 @@ package media import ( "bytes" + "context" "errors" "fmt" "time" @@ -68,9 +69,9 @@ const ( TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests ) -// AdditionalInfo represents additional information that should be added to an attachment +// AdditionalMediaInfo represents additional information that should be added to an attachment // when processing a piece of media. -type AdditionalInfo struct { +type AdditionalMediaInfo struct { // Time that this media was created; defaults to time.Now(). CreatedAt *time.Time // ID of the status to which this media is attached; defaults to "". @@ -93,6 +94,13 @@ type AdditionalInfo struct { FocusY *float32 } +type AdditionalEmojiInfo struct { + +} + +// DataFunc represents a function used to retrieve the raw bytes of a piece of media. +type DataFunc func(ctx context.Context) ([]byte, error) + // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. func parseContentType(content []byte) (string, error) { diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 6d15b5afb..7b305dc95 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -137,84 +137,87 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form // parsing and checking the image, and doing the necessary updates in the database for this to become // the account's new avatar image. func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { - var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(avatar.Size) > maxImageSize { - err = fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) - return nil, err - } - f, err := avatar.Open() - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") - } + dataFunc := func(ctx context.Context) ([]byte, error) { + // pop open the fileheader + f, err := avatar.Open() + if err != nil { + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + } - // we're done with the FileHeader now - if err := f.Close(); err != nil { - return nil, fmt.Errorf("UpdateAvatar: error closing multipart fileheader: %s", err) + // extract the bytes + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) + } + if size == 0 { + return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") + } + + return buf.Bytes(), f.Close() } - // do the setting isAvatar := true - processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + ai := &media.AdditionalMediaInfo{ Avatar: &isAvatar, - }) + } + + processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) if err != nil { return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } - return processingMedia.Load(ctx) + return processingMedia.LoadAttachment(ctx) } // UpdateHeader does the dirty work of checking the header part of an account update form, // parsing and checking the image, and doing the necessary updates in the database for this to become // the account's new header image. func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { - var err error maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize) if int(header.Size) > maxImageSize { - err = fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) - return nil, err - } - f, err := header.Open() - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") - } + dataFunc := func(ctx context.Context) ([]byte, error) { + // pop open the fileheader + f, err := header.Open() + if err != nil { + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + } + + // extract the bytes + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) + } + if size == 0 { + return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") + } - // we're done with the FileHeader now - if err := f.Close(); err != nil { - return nil, fmt.Errorf("UpdateHeader: error closing multipart fileheader: %s", err) + return buf.Bytes(), f.Close() } - // do the setting isHeader := true - processingMedia, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), accountID, &media.AdditionalInfo{ + ai := &media.AdditionalMediaInfo{ Header: &isHeader, - }) + } + + processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai) + if err != nil { + return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) + } if err != nil { return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } - return processingMedia.Load(ctx) + return processingMedia.LoadAttachment(ctx) } func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 737a4ebe2..8858dbd02 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -30,30 +30,34 @@ import ( ) func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { - if user.Admin { + if !user.Admin { return nil, fmt.Errorf("user %s not an admin", user.ID) } - // open the emoji and extract the bytes from it - f, err := form.Image.Open() - if err != nil { - return nil, fmt.Errorf("error opening emoji: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading emoji: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided emoji: size 0 bytes") + data := func(innerCtx context.Context) ([]byte, error) { + // open the emoji and extract the bytes from it + f, err := form.Image.Open() + if err != nil { + return nil, fmt.Errorf("error opening emoji: %s", err) + } + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("error reading emoji: %s", err) + } + if size == 0 { + return nil, errors.New("could not read provided emoji: size 0 bytes") + } + + return buf.Bytes(), f.Close() } - media, err := p.mediaManager.ProcessEmoji(ctx, buf.Bytes(), account.ID) + processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, nil) if err != nil { return nil, err } - emoji, err := media.LoadEmoji(ctx) + emoji, err := processingEmoji.LoadEmoji(ctx) if err != nil { return nil, err } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 9df5c7c1f..0896315b1 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -31,18 +31,21 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - // open the attachment and extract the bytes from it - f, err := form.File.Open() - if err != nil { - return nil, fmt.Errorf("error opening attachment: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided attachment: size 0 bytes") + data := func(innerCtx context.Context) ([]byte, error) { + // open the attachment and extract the bytes from it + f, err := form.File.Open() + if err != nil { + return nil, fmt.Errorf("error opening attachment: %s", err) + } + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + return nil, fmt.Errorf("error reading attachment: %s", err) + } + if size == 0 { + return nil, errors.New("could not read provided attachment: size 0 bytes") + } + return buf.Bytes(), f.Close() } focusX, focusY, err := parseFocus(form.Focus) @@ -51,7 +54,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form } // process the media attachment and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID, &media.AdditionalInfo{ + media, err := p.mediaManager.ProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{ Description: &form.Description, FocusX: &focusX, FocusY: &focusY, @@ -60,7 +63,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, err } - attachment, err := media.Load(ctx) + attachment, err := media.LoadAttachment(ctx) if err != nil { return nil, err } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index b470b289a..c43515a42 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -33,7 +33,7 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType. + // DereferenceMedia fetches the bytes of the given media attachment IRI. DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 203aaef96..9a9ea5d2f 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -66,6 +66,16 @@ func NewTestTokens() map[string]*gtsmodel.Token { AccessCreateAt: time.Now(), AccessExpiresAt: time.Now().Add(72 * time.Hour), }, + "admin_account": { + ID: "01FS4TP8ANA5VE92EAPA9E0M7Q", + ClientID: "01F8MGWSJCND9BWBD4WGJXBM93", + UserID: "01F8MGWYWKVKS3VS8DV1AMYPGE", + RedirectURI: "http://localhost:8080", + Scope: "read write follow push admin", + Access: "AININALKNENFNF98717NAMG4LWE4NJITMWUXM2M4MTRHZDEX", + AccessCreateAt: time.Now(), + AccessExpiresAt: time.Now().Add(72 * time.Hour), + }, } return tokens } -- cgit v1.3 From 33ca5513ada0e21c41e6fe13866b4f5ae961f670 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 11 Jan 2022 17:51:45 +0100 Subject: pin instanceAccountID to in-process emoji --- internal/media/processingemoji.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 7e2d4f31f..41754830f 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -37,6 +37,9 @@ import ( type ProcessingEmoji struct { mu sync.Mutex + // id of this instance's account -- pinned for convenience here so we only need to fetch it once + instanceAccountID string + /* below fields should be set on newly created media; emoji will be updated incrementally as media goes through processing @@ -370,12 +373,13 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode } processingEmoji := &ProcessingEmoji{ - emoji: emoji, - data: data, - staticState: received, - fullSizeState: received, - database: m.db, - storage: m.storage, + instanceAccountID: instanceAccount.ID, + emoji: emoji, + data: data, + staticState: received, + fullSizeState: received, + database: m.db, + storage: m.storage, } return processingEmoji, nil -- cgit v1.3 From c4a533db72505ca5303d8da637f54fae12b137a2 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 15 Jan 2022 14:33:58 +0100 Subject: start fixing up emoji processing code --- internal/api/client/admin/emojicreate_test.go | 40 +++++++ internal/media/image.go | 5 - internal/media/manager.go | 6 +- internal/media/processingemoji.go | 151 +++++++------------------- internal/media/processingmedia.go | 35 +----- internal/media/types.go | 124 ++++++--------------- internal/media/util.go | 123 +++++++++++++++++++++ internal/processing/admin/emoji.go | 11 +- 8 files changed, 249 insertions(+), 246 deletions(-) create mode 100644 internal/media/util.go diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 290b478f7..14b83b534 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -1,6 +1,8 @@ package admin_test import ( + "context" + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -8,6 +10,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -43,6 +48,41 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { b, err := ioutil.ReadAll(result.Body) suite.NoError(err) suite.NotEmpty(b) + + // response should be an api model emoji + apiEmoji := &apimodel.Emoji{} + err = json.Unmarshal(b, apiEmoji) + suite.NoError(err) + + // appropriate fields should be set + suite.Equal("rainbow", apiEmoji.Shortcode) + suite.NotEmpty(apiEmoji.URL) + suite.NotEmpty(apiEmoji.StaticURL) + suite.True(apiEmoji.VisibleInPicker) + + // emoji should be in the db + dbEmoji := >smodel.Emoji{} + err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "rainbow"}}, dbEmoji) + suite.NoError(err) + + // check fields on the emoji + suite.NotEmpty(dbEmoji.ID) + suite.Equal("rainbow", dbEmoji.Shortcode) + suite.Empty(dbEmoji.Domain) + suite.Empty(dbEmoji.ImageRemoteURL) + suite.Empty(dbEmoji.ImageStaticRemoteURL) + suite.Equal(apiEmoji.URL, dbEmoji.ImageURL) + suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageURL) + suite.NotEmpty(dbEmoji.ImagePath) + suite.NotEmpty(dbEmoji.ImageStaticPath) + suite.Equal("image/png", dbEmoji.ImageContentType) + suite.Equal("image/png", dbEmoji.ImageStaticContentType) + suite.Equal(36702, dbEmoji.ImageFileSize) + suite.Equal(10413, dbEmoji.ImageStaticFileSize) + suite.False(dbEmoji.Disabled) + suite.NotEmpty(dbEmoji.URI) + suite.True(dbEmoji.VisibleInPicker) + suite.Empty(dbEmoji.CategoryID)aaaaaaaaa } func TestEmojiCreateTestSuite(t *testing.T) { diff --git a/internal/media/image.go b/internal/media/image.go index a5a818206..de4b71210 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,21 +20,16 @@ package media import ( "bytes" - "context" "errors" "fmt" "image" "image/gif" "image/jpeg" "image/png" - "time" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" "github.com/superseriousbusiness/exifremove/pkg/exifremove" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) const ( diff --git a/internal/media/manager.go b/internal/media/manager.go index e34471591..7f626271a 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -41,7 +41,7 @@ type Manager interface { // // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) - ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) + ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) // NumWorkers returns the total number of workers available to this manager. NumWorkers() int // QueueSize returns the total capacity of the queue. @@ -125,8 +125,8 @@ func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, accountID str return processingMedia, nil } -func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { - processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, ai) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { + processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, id, uri, ai) if err != nil { return nil, err } diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 41754830f..eeccdb281 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -28,7 +28,6 @@ import ( "codeberg.org/gruf/go-store/kv" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -126,33 +125,28 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { } // set appropriate fields on the emoji based on the static version we derived - p.attachment.FileMeta.Small = gtsmodel.Small{ - Width: static.width, - Height: static.height, - Size: static.size, - Aspect: static.aspect, - } - p.attachment.Thumbnail.FileSize = static.size + p.emoji.ImageStaticFileSize = len(static.image) - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + // update the emoji in the db + if err := putOrUpdate(ctx, p.database, p.emoji); err != nil { p.err = err - p.thumbstate = errored + p.staticState = errored return nil, err } - // set the thumbnail of this media - p.thumb = static + // set the static on the processing emoji + p.static = static - // we're done processing the thumbnail! - p.thumbstate = complete + // we're done processing the static version of the emoji! + p.staticState = complete fallthrough case complete: - return p.thumb, nil + return p.static, nil case errored: return nil, p.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) + return nil, fmt.Errorf("static processing status %d unknown", p.staticState) } func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { @@ -161,26 +155,17 @@ func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) switch p.fullSizeState { case received: - var clean []byte var err error var decoded *ImageMeta - ct := p.attachment.File.ContentType + ct := p.emoji.ImageContentType switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) + case mimeImagePng: + decoded, err = decodeImage(p.rawData, ct) case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) + decoded, err = decodeGif(p.rawData) default: - err = fmt.Errorf("content type %s not a processible image type", ct) + err = fmt.Errorf("content type %s not a processible emoji type", ct) } if err != nil { @@ -189,34 +174,17 @@ func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) return nil, err } - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) + // put the full size emoji in storage + if err := p.storage.Put(p.emoji.ImagePath, decoded.image); err != nil { + p.err = fmt.Errorf("error storing full size emoji: %s", err) p.fullSizeState = errored return nil, p.err } - // set appropriate fields on the attachment based on the image we derived - p.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, - } - p.attachment.File.FileSize = decoded.size - p.attachment.File.UpdatedAt = time.Now() - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - // set the fullsize of this media p.fullSize = decoded - // we're done processing the full-size image + // we're done processing the full-size emoji p.fullSizeState = complete fallthrough case complete: @@ -255,55 +223,24 @@ func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { } split := strings.Split(contentType, "/") - mainType := split[0] // something like 'image' extension := split[1] // something like 'gif' // set some additional fields on the emoji now that // we know more about what the underlying image actually is - p.emoji.ImageURL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) - p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) - p.attachment.File.ContentType = contentType - - switch mainType { - case mimeImage: - if extension == mimeGif { - p.attachment.Type = gtsmodel.FileTypeGif - } else { - p.attachment.Type = gtsmodel.FileTypeImage - } - default: - return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) - } - - return nil -} - -// putOrUpdateEmoji is just a convenience function for first trying to PUT the emoji in the database, -// and then if that doesn't work because the emoji already exists, updating it instead. -func putOrUpdateEmoji(ctx context.Context, database db.DB, emoji *gtsmodel.Emoji) error { - if err := database.Put(ctx, emoji); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateEmoji: proper error while putting emoji: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, emoji); err != nil { - return fmt.Errorf("putOrUpdateEmoji: error while updating emoji: %s", err) - } - } + p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) + p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) + p.emoji.ImageContentType = contentType + p.emoji.ImageFileSize = len(p.rawData) return nil } -func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { instanceAccount, err := m.db.GetInstanceAccount(ctx, "") if err != nil { return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) } - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - // populate initial fields on the emoji -- some of these will be overwritten as we proceed emoji := >smodel.Emoji{ ID: id, @@ -323,7 +260,7 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode ImageStaticFileSize: 0, ImageUpdatedAt: time.Now(), Disabled: false, - URI: "", // we don't know yet + URI: uri, VisibleInPicker: true, CategoryID: "", } @@ -332,43 +269,31 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode // and overwrite some of the emoji fields if so if ai != nil { if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description + emoji.CreatedAt = *ai.CreatedAt } - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID + if ai.Domain != nil { + emoji.Domain = *ai.Domain } - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash + if ai.ImageRemoteURL != nil { + emoji.ImageRemoteURL = *ai.ImageRemoteURL } - if ai.Avatar != nil { - attachment.Avatar = *ai.Avatar + if ai.ImageStaticRemoteURL != nil { + emoji.ImageStaticRemoteURL = *ai.ImageStaticRemoteURL } - if ai.Header != nil { - attachment.Header = *ai.Header + if ai.Disabled != nil { + emoji.Disabled = *ai.Disabled } - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX + if ai.VisibleInPicker != nil { + emoji.VisibleInPicker = *ai.VisibleInPicker } - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY + if ai.CategoryID != nil { + emoji.CategoryID = *ai.CategoryID } } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index a6e45034f..1bfd7b629 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -32,14 +32,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/uris" ) -type processState int - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - // ProcessingMedia represents a piece of media that is currently being processed. It exposes // various functions for retrieving data from the process. type ProcessingMedia struct { @@ -103,10 +95,6 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt return p.attachment, nil } -func (p *ProcessingMedia) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - return nil, nil -} - // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingMedia) Finished() bool { @@ -153,9 +141,9 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { Size: thumb.size, Aspect: thumb.aspect, } - p.attachment.Thumbnail.FileSize = thumb.size + p.attachment.Thumbnail.FileSize = len(thumb.image) - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { p.err = err p.thumbstate = errored return nil, err @@ -224,11 +212,11 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) Size: decoded.size, Aspect: decoded.aspect, } - p.attachment.File.FileSize = decoded.size + p.attachment.File.FileSize = len(decoded.image) p.attachment.File.UpdatedAt = time.Now() p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - if err := putOrUpdateAttachment(ctx, p.database, p.attachment); err != nil { + if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { p.err = err p.fullSizeState = errored return nil, err @@ -299,21 +287,6 @@ func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { return nil } -// putOrUpdateAttachment is just a convenience function for first trying to PUT the attachment in the database, -// and then if that doesn't work because the attachment already exists, updating it instead. -func putOrUpdateAttachment(ctx context.Context, database db.DB, attachment *gtsmodel.MediaAttachment) error { - if err := database.Put(ctx, attachment); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdateAttachment: proper error while putting attachment: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, attachment); err != nil { - return fmt.Errorf("putOrUpdateAttachment: error while updating attachment: %s", err) - } - } - - return nil -} - func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { id, err := id.NewRandomULID() if err != nil { diff --git a/internal/media/types.go b/internal/media/types.go index 6426223d1..5b3fe4a41 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -19,15 +19,17 @@ package media import ( - "bytes" "context" - "errors" - "fmt" "time" - - "github.com/h2non/filetype" ) +// maxFileHeaderBytes represents the maximum amount of bytes we want +// to examine from the beginning of a file to determine its type. +// +// See: https://en.wikipedia.org/wiki/File_format#File_header +// and https://github.com/h2non/filetype +const maxFileHeaderBytes = 262 + // mime consts const ( mimeImage = "image" @@ -42,16 +44,17 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) +type processState int + +const ( + received processState = iota // processing order has been received but not done yet + complete // processing order has been completed successfully + errored // processing order has been completed with an error +) + // EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) // const EmojiMaxBytes = 51200 -// maxFileHeaderBytes represents the maximum amount of bytes we want -// to examine from the beginning of a file to determine its type. -// -// See: https://en.wikipedia.org/wiki/File_format#File_header -// and https://github.com/h2non/filetype -const maxFileHeaderBytes = 262 - type Size string const ( @@ -94,89 +97,24 @@ type AdditionalMediaInfo struct { FocusY *float32 } +// AdditionalMediaInfo represents additional information +// that should be added to an emoji when processing it. type AdditionalEmojiInfo struct { - + // Time that this emoji was created; defaults to time.Now(). + CreatedAt *time.Time + // Domain the emoji originated from. Blank for this instance's domain. Defaults to "". + Domain *string + // URL of this emoji on a remote instance; defaults to "". + ImageRemoteURL *string + // URL of the static version of this emoji on a remote instance; defaults to "". + ImageStaticRemoteURL *string + // Whether this emoji should be disabled (not shown) on this instance; defaults to false. + Disabled *bool + // Whether this emoji should be visible in the instance's emoji picker; defaults to true. + VisibleInPicker *bool + // ID of the category this emoji should be placed in; defaults to "". + CategoryID *string } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. type DataFunc func(ctx context.Context) ([]byte, error) - -// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). -// Returns an error if the content type is not something we can process. -func parseContentType(content []byte) (string, error) { - - // read in the first bytes of the file - fileHeader := make([]byte, maxFileHeaderBytes) - if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { - return "", fmt.Errorf("could not read first magic bytes of file: %s", err) - } - - kind, err := filetype.Match(fileHeader) - if err != nil { - return "", err - } - - if kind == filetype.Unknown { - return "", errors.New("filetype unknown") - } - - return kind.MIME.Value, nil -} - -// supportedImage checks mime type of an image against a slice of accepted types, -// and returns True if the mime type is accepted. -func supportedImage(mimeType string) bool { - acceptedImageTypes := []string{ - mimeImageJpeg, - mimeImageGif, - mimeImagePng, - } - for _, accepted := range acceptedImageTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. -func supportedEmoji(mimeType string) bool { - acceptedEmojiTypes := []string{ - mimeImageGif, - mimeImagePng, - } - for _, accepted := range acceptedEmojiTypes { - if mimeType == accepted { - return true - } - } - return false -} - -// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized -func ParseMediaType(s string) (Type, error) { - switch s { - case string(TypeAttachment): - return TypeAttachment, nil - case string(TypeHeader): - return TypeHeader, nil - case string(TypeAvatar): - return TypeAvatar, nil - case string(TypeEmoji): - return TypeEmoji, nil - } - return "", fmt.Errorf("%s not a recognized MediaType", s) -} - -// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized -func ParseMediaSize(s string) (Size, error) { - switch s { - case string(SizeSmall): - return SizeSmall, nil - case string(SizeOriginal): - return SizeOriginal, nil - case string(SizeStatic): - return SizeStatic, nil - } - return "", fmt.Errorf("%s not a recognized MediaSize", s) -} diff --git a/internal/media/util.go b/internal/media/util.go new file mode 100644 index 000000000..16e874a99 --- /dev/null +++ b/internal/media/util.go @@ -0,0 +1,123 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/h2non/filetype" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). +// Returns an error if the content type is not something we can process. +func parseContentType(content []byte) (string, error) { + // read in the first bytes of the file + fileHeader := make([]byte, maxFileHeaderBytes) + if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { + return "", fmt.Errorf("could not read first magic bytes of file: %s", err) + } + + kind, err := filetype.Match(fileHeader) + if err != nil { + return "", err + } + + if kind == filetype.Unknown { + return "", errors.New("filetype unknown") + } + + return kind.MIME.Value, nil +} + +// supportedImage checks mime type of an image against a slice of accepted types, +// and returns True if the mime type is accepted. +func supportedImage(mimeType string) bool { + acceptedImageTypes := []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedImageTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. +func supportedEmoji(mimeType string) bool { + acceptedEmojiTypes := []string{ + mimeImageGif, + mimeImagePng, + } + for _, accepted := range acceptedEmojiTypes { + if mimeType == accepted { + return true + } + } + return false +} + +// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized +func ParseMediaType(s string) (Type, error) { + switch s { + case string(TypeAttachment): + return TypeAttachment, nil + case string(TypeHeader): + return TypeHeader, nil + case string(TypeAvatar): + return TypeAvatar, nil + case string(TypeEmoji): + return TypeEmoji, nil + } + return "", fmt.Errorf("%s not a recognized MediaType", s) +} + +// ParseMediaSize converts s to a recognized MediaSize, or returns an error if unrecognized +func ParseMediaSize(s string) (Size, error) { + switch s { + case string(SizeSmall): + return SizeSmall, nil + case string(SizeOriginal): + return SizeOriginal, nil + case string(SizeStatic): + return SizeStatic, nil + } + return "", fmt.Errorf("%s not a recognized MediaSize", s) +} + +// putOrUpdate is just a convenience function for first trying to PUT the attachment or emoji in the database, +// and then if that doesn't work because the attachment/emoji already exists, updating it instead. +func putOrUpdate(ctx context.Context, database db.DB, i interface{}) error { + if err := database.Put(ctx, i); err != nil { + if err != db.ErrAlreadyExists { + return fmt.Errorf("putOrUpdate: proper error while putting: %s", err) + } + if err := database.UpdateByPrimaryKey(ctx, i); err != nil { + return fmt.Errorf("putOrUpdate: error while updating: %s", err) + } + } + + return nil +} diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 8858dbd02..77fa5102b 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -27,6 +27,8 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { @@ -52,7 +54,14 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return buf.Bytes(), f.Close() } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, nil) + emojiID, err := id.NewRandomULID() + if err != nil { + return nil, fmt.Errorf("error creating id for new emoji: %s", err) + } + + emojiURI := uris.GenerateURIForEmoji(emojiID) + + processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, emojiID, emojiURI, nil) if err != nil { return nil, err } -- cgit v1.3 From 6bf39d0fc1286bdf2f4760adab52c6eff234d01d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 15 Jan 2022 17:36:15 +0100 Subject: emoji code passing muster --- internal/api/client/admin/emojicreate.go | 10 +++--- internal/api/client/admin/emojicreate_test.go | 50 +++++++++++++++++++++++---- internal/gtserror/withcode.go | 13 +++++++ internal/media/processingemoji.go | 22 +++++++----- internal/media/processingmedia.go | 25 +++++++------- internal/processing/admin.go | 2 +- internal/processing/admin/admin.go | 2 +- internal/processing/admin/emoji.go | 17 +++++---- internal/processing/processor.go | 2 +- 9 files changed, 104 insertions(+), 39 deletions(-) diff --git a/internal/api/client/admin/emojicreate.go b/internal/api/client/admin/emojicreate.go index de7a2979a..ef42d0a13 100644 --- a/internal/api/client/admin/emojicreate.go +++ b/internal/api/client/admin/emojicreate.go @@ -73,6 +73,8 @@ import ( // description: forbidden // '400': // description: bad request +// '409': +// description: conflict -- domain/shortcode combo for emoji already exists func (m *Module) EmojiCreatePOSTHandler(c *gin.Context) { l := logrus.WithFields(logrus.Fields{ "func": "emojiCreatePOSTHandler", @@ -116,10 +118,10 @@ func (m *Module) EmojiCreatePOSTHandler(c *gin.Context) { return } - apiEmoji, err := m.processor.AdminEmojiCreate(c.Request.Context(), authed, form) - if err != nil { - l.Debugf("error creating emoji: %s", err) - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + apiEmoji, errWithCode := m.processor.AdminEmojiCreate(c.Request.Context(), authed, form) + if errWithCode != nil { + l.Debugf("error creating emoji: %s", errWithCode.Error()) + c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()}) return } diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 14b83b534..2b7476da1 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -25,7 +25,7 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { requestBody, w, err := testrig.CreateMultipartFormData( "image", "../../../../testrig/media/rainbow-original.png", map[string]string{ - "shortcode": "rainbow", + "shortcode": "new_emoji", }) if err != nil { panic(err) @@ -55,24 +55,24 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { suite.NoError(err) // appropriate fields should be set - suite.Equal("rainbow", apiEmoji.Shortcode) + suite.Equal("new_emoji", apiEmoji.Shortcode) suite.NotEmpty(apiEmoji.URL) suite.NotEmpty(apiEmoji.StaticURL) suite.True(apiEmoji.VisibleInPicker) // emoji should be in the db dbEmoji := >smodel.Emoji{} - err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "rainbow"}}, dbEmoji) + err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "new_emoji"}}, dbEmoji) suite.NoError(err) // check fields on the emoji suite.NotEmpty(dbEmoji.ID) - suite.Equal("rainbow", dbEmoji.Shortcode) + suite.Equal("new_emoji", dbEmoji.Shortcode) suite.Empty(dbEmoji.Domain) suite.Empty(dbEmoji.ImageRemoteURL) suite.Empty(dbEmoji.ImageStaticRemoteURL) suite.Equal(apiEmoji.URL, dbEmoji.ImageURL) - suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageURL) + suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageStaticURL) suite.NotEmpty(dbEmoji.ImagePath) suite.NotEmpty(dbEmoji.ImageStaticPath) suite.Equal("image/png", dbEmoji.ImageContentType) @@ -82,7 +82,45 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { suite.False(dbEmoji.Disabled) suite.NotEmpty(dbEmoji.URI) suite.True(dbEmoji.VisibleInPicker) - suite.Empty(dbEmoji.CategoryID)aaaaaaaaa + suite.Empty(dbEmoji.CategoryID) + + // emoji should be in storage + emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath) + suite.NoError(err) + suite.Len(emojiBytes, dbEmoji.ImageFileSize) + emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath) + suite.NoError(err) + suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize) +} + +func (suite *EmojiCreateTestSuite) TestEmojiCreateAlreadyExists() { + // set up the request -- use a shortcode that already exists for an emoji in the database + requestBody, w, err := testrig.CreateMultipartFormData( + "image", "../../../../testrig/media/rainbow-original.png", + map[string]string{ + "shortcode": "rainbow", + }) + if err != nil { + panic(err) + } + bodyBytes := requestBody.Bytes() + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPath, w.FormDataContentType()) + + // call the handler + suite.adminModule.EmojiCreatePOSTHandler(ctx) + + suite.Equal(http.StatusConflict, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + + // check the response + b, err := ioutil.ReadAll(result.Body) + suite.NoError(err) + suite.NotEmpty(b) + + suite.Equal(`{"error":"conflict: emoji with shortcode rainbow already exists"}`, string(b)) } func TestEmojiCreateTestSuite(t *testing.T) { diff --git a/internal/gtserror/withcode.go b/internal/gtserror/withcode.go index a00cc8503..34889b961 100644 --- a/internal/gtserror/withcode.go +++ b/internal/gtserror/withcode.go @@ -122,3 +122,16 @@ func NewErrorInternalError(original error, helpText ...string) WithCode { code: http.StatusInternalServerError, } } + +// NewErrorConflict returns an ErrorWithCode 409 with the given original error and optional help text. +func NewErrorConflict(original error, helpText ...string) WithCode { + safe := "conflict" + if helpText != nil { + safe = safe + ": " + strings.Join(helpText, ": ") + } + return withCode{ + original: original, + safe: errors.New(safe), + code: http.StatusConflict, + } +} diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index eeccdb281..467b500fc 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -72,6 +72,9 @@ type ProcessingEmoji struct { storage *kv.KVStore err error // error created during processing, if any + + // track whether this emoji has already been put in the databse + insertedInDB bool } // EmojiID returns the ID of the underlying emoji without blocking processing. @@ -94,6 +97,16 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error return nil, err } + // store the result in the database before returning it + p.mu.Lock() + defer p.mu.Unlock() + if !p.insertedInDB { + if err := p.database.Put(ctx, p.emoji); err != nil { + return nil, err + } + p.insertedInDB = true + } + return p.emoji, nil } @@ -127,13 +140,6 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { // set appropriate fields on the emoji based on the static version we derived p.emoji.ImageStaticFileSize = len(static.image) - // update the emoji in the db - if err := putOrUpdate(ctx, p.database, p.emoji); err != nil { - p.err = err - p.staticState = errored - return nil, err - } - // set the static on the processing emoji p.static = static @@ -197,7 +203,7 @@ func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) } // fetchRawData calls the data function attached to p if it hasn't been called yet, -// and updates the underlying attachment fields as necessary. +// and updates the underlying emoji fields as necessary. // It should only be called from within a function that already has a lock on p! func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { // check if we've already done this and bail early if we have diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 1bfd7b629..fade64c24 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -70,6 +70,9 @@ type ProcessingMedia struct { storage *kv.KVStore err error // error created during processing, if any + + // track whether this media has already been put in the databse + insertedInDB bool } // AttachmentID returns the ID of the underlying media attachment without blocking processing. @@ -92,6 +95,16 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt return nil, err } + // store the result in the database before returning it + p.mu.Lock() + defer p.mu.Unlock() + if !p.insertedInDB { + if err := p.database.Put(ctx, p.attachment); err != nil { + return nil, err + } + p.insertedInDB = true + } + return p.attachment, nil } @@ -143,12 +156,6 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { } p.attachment.Thumbnail.FileSize = len(thumb.image) - if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { - p.err = err - p.thumbstate = errored - return nil, err - } - // set the thumbnail of this media p.thumb = thumb @@ -216,12 +223,6 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) p.attachment.File.UpdatedAt = time.Now() p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - if err := putOrUpdate(ctx, p.database, p.attachment); err != nil { - p.err = err - p.fullSizeState = errored - return nil, err - } - // set the fullsize of this media p.fullSize = decoded diff --git a/internal/processing/admin.go b/internal/processing/admin.go index c70bd79d0..764e6d302 100644 --- a/internal/processing/admin.go +++ b/internal/processing/admin.go @@ -26,7 +26,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" ) -func (p *processor) AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { +func (p *processor) AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) { return p.adminProcessor.EmojiCreate(ctx, authed.Account, authed.User, form) } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 27a7da47a..bdb586588 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -38,7 +38,7 @@ type Processor interface { DomainBlocksGet(ctx context.Context, account *gtsmodel.Account, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode) DomainBlockGet(ctx context.Context, account *gtsmodel.Account, id string, export bool) (*apimodel.DomainBlock, gtserror.WithCode) DomainBlockDelete(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode) - EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) + EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) } type processor struct { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 77fa5102b..fcc17c4be 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -26,14 +26,16 @@ import ( "io" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/uris" ) -func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) { +func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) { if !user.Admin { - return nil, fmt.Errorf("user %s not an admin", user.ID) + return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } data := func(innerCtx context.Context) ([]byte, error) { @@ -56,24 +58,27 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, emojiID, err := id.NewRandomULID() if err != nil { - return nil, fmt.Errorf("error creating id for new emoji: %s", err) + return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new emoji: %s", err), "error creating emoji ID") } emojiURI := uris.GenerateURIForEmoji(emojiID) processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, emojiID, emojiURI, nil) if err != nil { - return nil, err + return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji") } emoji, err := processingEmoji.LoadEmoji(ctx) if err != nil { - return nil, err + if err == db.ErrAlreadyExists { + return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode)) + } + return nil, gtserror.NewErrorInternalError(fmt.Errorf("error loading emoji: %s", err), "error loading emoji") } apiEmoji, err := p.tc.EmojiToAPIEmoji(ctx, emoji) if err != nil { - return nil, fmt.Errorf("error converting emoji to apitype: %s", err) + return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting emoji: %s", err), "error converting emoji to api representation") } return &apiEmoji, nil diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 2626c1fea..2406681ea 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -96,7 +96,7 @@ type Processor interface { AccountBlockRemove(ctx context.Context, authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode) // AdminEmojiCreate handles the creation of a new instance emoji by an admin, using the given form. - AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) + AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) // AdminDomainBlockCreate handles the creation of a new domain block by an admin, using the given form. AdminDomainBlockCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode) // AdminDomainBlocksImport handles the import of multiple domain blocks by an admin, using the given form. -- cgit v1.3 From 723bfe8944f80fd1ef935ad6878fc555fd42b8e7 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 15 Jan 2022 17:41:18 +0100 Subject: lint, fmt --- internal/media/processingmedia.go | 21 +++++++++------------ internal/media/util.go | 19 +------------------ 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index fade64c24..082c58607 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -260,31 +260,28 @@ func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { return fmt.Errorf("fetchRawData: error parsing content type: %s", err) } + if !supportedImage(contentType) { + return fmt.Errorf("fetchRawData: media type %s not (yet) supported", contentType) + } + split := strings.Split(contentType, "/") if len(split) != 2 { return fmt.Errorf("fetchRawData: content type %s was not valid", contentType) } - mainType := split[0] // something like 'image' extension := split[1] // something like 'jpeg' // set some additional fields on the attachment now that // we know more about what the underlying media actually is + if extension == mimeGif { + p.attachment.Type = gtsmodel.FileTypeGif + } else { + p.attachment.Type = gtsmodel.FileTypeImage + } p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) p.attachment.File.ContentType = contentType - switch mainType { - case mimeImage: - if extension == mimeGif { - p.attachment.Type = gtsmodel.FileTypeGif - } else { - p.attachment.Type = gtsmodel.FileTypeImage - } - default: - return fmt.Errorf("fetchRawData: cannot process mime type %s (yet)", mainType) - } - return nil } diff --git a/internal/media/util.go b/internal/media/util.go index 16e874a99..7a3d81c0f 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -20,12 +20,10 @@ package media import ( "bytes" - "context" "errors" "fmt" "github.com/h2non/filetype" - "github.com/superseriousbusiness/gotosocial/internal/db" ) // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). @@ -65,7 +63,7 @@ func supportedImage(mimeType string) bool { return false } -// supportedEmoji checks that the content type is image/png -- the only type supported for emoji. +// supportedEmoji checks that the content type is image/png or image/gif -- the only types supported for emoji. func supportedEmoji(mimeType string) bool { acceptedEmojiTypes := []string{ mimeImageGif, @@ -106,18 +104,3 @@ func ParseMediaSize(s string) (Size, error) { } return "", fmt.Errorf("%s not a recognized MediaSize", s) } - -// putOrUpdate is just a convenience function for first trying to PUT the attachment or emoji in the database, -// and then if that doesn't work because the attachment/emoji already exists, updating it instead. -func putOrUpdate(ctx context.Context, database db.DB, i interface{}) error { - if err := database.Put(ctx, i); err != nil { - if err != db.ErrAlreadyExists { - return fmt.Errorf("putOrUpdate: proper error while putting: %s", err) - } - if err := database.UpdateByPrimaryKey(ctx, i); err != nil { - return fmt.Errorf("putOrUpdate: error while updating: %s", err) - } - } - - return nil -} -- cgit v1.3 From 18047de666c2028b6dd8f90bf3f300b5ed932247 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 15 Jan 2022 17:47:52 +0100 Subject: add go-runners to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ae50e9f0d..1d5e5fb19 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude - [google/uuid](https://github.com/google/uuid); UUID generation. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html) - [go-playground/validator](https://github.com/go-playground/validator); struct validation. [MIT License](https://spdx.org/licenses/MIT.html) - [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html). +- [gruf/go-runners](https://codeberg.org/gruf/go-runners); worker pool library. [MIT License](https://spdx.org/licenses/MIT.html). - [gruf/go-store](https://codeberg.org/gruf/go-store); cacheing library. [MIT License](https://spdx.org/licenses/MIT.html). - [h2non/filetype](https://github.com/h2non/filetype); filetype checking. [MIT License](https://spdx.org/licenses/MIT.html). - [jackc/pgx](https://github.com/jackc/pgx); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html). -- cgit v1.3 From 6f5ccf435585e43a00e3cc50f4bcefac36ada818 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 16 Jan 2022 18:52:30 +0100 Subject: update dependencies --- go.mod | 6 +- go.sum | 11 + vendor/codeberg.org/gruf/go-errors/data.go | 27 +- vendor/codeberg.org/gruf/go-format/LICENSE | 9 + vendor/codeberg.org/gruf/go-format/README.md | 16 + vendor/codeberg.org/gruf/go-format/buffer.go | 81 + vendor/codeberg.org/gruf/go-format/format.go | 565 +++++ vendor/codeberg.org/gruf/go-format/formatter.go | 352 +++ vendor/codeberg.org/gruf/go-format/print.go | 88 + vendor/codeberg.org/gruf/go-format/util.go | 13 + vendor/codeberg.org/gruf/go-logger/LICENSE | 9 - vendor/codeberg.org/gruf/go-logger/README.md | 13 - vendor/codeberg.org/gruf/go-logger/clock.go | 21 - vendor/codeberg.org/gruf/go-logger/default.go | 107 - vendor/codeberg.org/gruf/go-logger/entry.go | 385 --- vendor/codeberg.org/gruf/go-logger/format.go | 87 - vendor/codeberg.org/gruf/go-logger/format_text.go | 914 ------- vendor/codeberg.org/gruf/go-logger/hook.go | 13 - vendor/codeberg.org/gruf/go-logger/level.go | 38 - vendor/codeberg.org/gruf/go-logger/logger.go | 187 -- vendor/codeberg.org/gruf/go-logger/writer.go | 29 - vendor/codeberg.org/gruf/go-store/kv/iterator.go | 2 +- vendor/codeberg.org/gruf/go-store/kv/state.go | 18 +- vendor/codeberg.org/gruf/go-store/kv/store.go | 82 +- vendor/codeberg.org/gruf/go-store/storage/block.go | 57 +- vendor/codeberg.org/gruf/go-store/storage/disk.go | 29 +- vendor/codeberg.org/gruf/go-store/storage/fs.go | 2 +- vendor/codeberg.org/gruf/go-store/storage/lock.go | 16 +- .../codeberg.org/gruf/go-store/storage/memory.go | 53 +- .../codeberg.org/gruf/go-store/storage/storage.go | 9 +- vendor/github.com/zeebo/blake3/.gitignore | 6 + vendor/github.com/zeebo/blake3/LICENSE | 125 + vendor/github.com/zeebo/blake3/Makefile | 11 + vendor/github.com/zeebo/blake3/README.md | 77 + vendor/github.com/zeebo/blake3/api.go | 166 ++ vendor/github.com/zeebo/blake3/blake3.go | 285 +++ vendor/github.com/zeebo/blake3/digest.go | 100 + vendor/github.com/zeebo/blake3/internal/alg/alg.go | 18 + .../zeebo/blake3/internal/alg/compress/compress.go | 15 + .../alg/compress/compress_pure/compress.go | 135 ++ .../alg/compress/compress_sse41/impl_amd64.s | 560 +++++ .../alg/compress/compress_sse41/impl_other.go | 9 + .../internal/alg/compress/compress_sse41/stubs.go | 6 + .../zeebo/blake3/internal/alg/hash/hash.go | 23 + .../internal/alg/hash/hash_avx2/impl_amd64.s | 2561 ++++++++++++++++++++ .../internal/alg/hash/hash_avx2/impl_other.go | 13 + .../blake3/internal/alg/hash/hash_avx2/stubs.go | 9 + .../blake3/internal/alg/hash/hash_pure/hashf.go | 56 + .../blake3/internal/alg/hash/hash_pure/hashp.go | 38 + .../zeebo/blake3/internal/consts/consts.go | 29 + .../github.com/zeebo/blake3/internal/consts/cpu.go | 17 + .../zeebo/blake3/internal/consts/cpu_big.go | 5 + .../zeebo/blake3/internal/consts/cpu_little.go | 5 + .../zeebo/blake3/internal/consts/cpu_other.go | 7 + .../zeebo/blake3/internal/utils/utils.go | 60 + vendor/modules.txt | 20 +- 56 files changed, 5682 insertions(+), 1913 deletions(-) create mode 100644 vendor/codeberg.org/gruf/go-format/LICENSE create mode 100644 vendor/codeberg.org/gruf/go-format/README.md create mode 100644 vendor/codeberg.org/gruf/go-format/buffer.go create mode 100644 vendor/codeberg.org/gruf/go-format/format.go create mode 100644 vendor/codeberg.org/gruf/go-format/formatter.go create mode 100644 vendor/codeberg.org/gruf/go-format/print.go create mode 100644 vendor/codeberg.org/gruf/go-format/util.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/LICENSE delete mode 100644 vendor/codeberg.org/gruf/go-logger/README.md delete mode 100644 vendor/codeberg.org/gruf/go-logger/clock.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/default.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/entry.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/format.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/format_text.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/hook.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/level.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/logger.go delete mode 100644 vendor/codeberg.org/gruf/go-logger/writer.go create mode 100644 vendor/github.com/zeebo/blake3/.gitignore create mode 100644 vendor/github.com/zeebo/blake3/LICENSE create mode 100644 vendor/github.com/zeebo/blake3/Makefile create mode 100644 vendor/github.com/zeebo/blake3/README.md create mode 100644 vendor/github.com/zeebo/blake3/api.go create mode 100644 vendor/github.com/zeebo/blake3/blake3.go create mode 100644 vendor/github.com/zeebo/blake3/digest.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/alg.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go create mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go create mode 100644 vendor/github.com/zeebo/blake3/internal/consts/consts.go create mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu.go create mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go create mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go create mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go create mode 100644 vendor/github.com/zeebo/blake3/internal/utils/utils.go diff --git a/go.mod b/go.mod index 71509ee6a..75956c7d6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( codeberg.org/gruf/go-runners v1.2.0 - codeberg.org/gruf/go-store v1.1.5 + codeberg.org/gruf/go-store v1.2.2 github.com/ReneKroon/ttlcache v1.7.0 github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.1.0 @@ -46,8 +46,9 @@ require ( require ( codeberg.org/gruf/go-bytes v1.0.2 // indirect - codeberg.org/gruf/go-errors v1.0.4 // indirect + codeberg.org/gruf/go-errors v1.0.5 // indirect codeberg.org/gruf/go-fastpath v1.0.2 // indirect + codeberg.org/gruf/go-format v1.0.3 // indirect codeberg.org/gruf/go-hashenc v1.0.1 // indirect codeberg.org/gruf/go-logger v1.3.2 // indirect codeberg.org/gruf/go-mutexes v1.0.1 // indirect @@ -107,6 +108,7 @@ require ( github.com/ugorji/go/codec v1.2.6 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/zeebo/blake3 v0.2.1 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect diff --git a/go.sum b/go.sum index 05d7597e1..082158f63 100644 --- a/go.sum +++ b/go.sum @@ -53,9 +53,13 @@ codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9 codeberg.org/gruf/go-cache v1.1.2/go.mod h1:/Dbc+xU72Op3hMn6x2PXF3NE9uIDFeS+sXPF00hN/7o= codeberg.org/gruf/go-errors v1.0.4 h1:jOJCn/GMb6ELLRVlnmpimGRC2CbTreH5/CBZNWh9GZA= codeberg.org/gruf/go-errors v1.0.4/go.mod h1:rJ08LdIE79Jg8vZ2TGylz/I+tZ1UuMJkGK5mNambIfQ= +codeberg.org/gruf/go-errors v1.0.5 h1:rxV70oQkfasUdggLHxOX2QAoJOMFM7XWxHQR45Zx/Fg= +codeberg.org/gruf/go-errors v1.0.5/go.mod h1:n03EpmvcmfzU3/xJKC0XXtleXXJUNFpT2fgISODvZ1Y= codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI= codeberg.org/gruf/go-fastpath v1.0.2 h1:O3nuYPMXnN89dsgAwVFU5iCGINtPJdITWmbRe2an/iQ= codeberg.org/gruf/go-fastpath v1.0.2/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI= +codeberg.org/gruf/go-format v1.0.3 h1:WoUGzTwZe6SIhILNvtr0qNIA7BOOCgdBlk5bUrfeiio= +codeberg.org/gruf/go-format v1.0.3/go.mod h1:k3TLXp1dqAXdDqxlon0yEM+3FFHdNn0D6BVJTwTy5As= codeberg.org/gruf/go-hashenc v1.0.1 h1:EBvNe2wW8IPMUqT1XihB6/IM6KMJDLMFBxIUvmsy1f8= codeberg.org/gruf/go-hashenc v1.0.1/go.mod h1:IfHhPCVScOiYmJLqdCQT9bYVS1nxNTV4ewMUvFWDPtc= codeberg.org/gruf/go-logger v1.3.1/go.mod h1:tBduUc+Yb9vqGRxY9/FB0ZlYznSteLy/KmIANo7zFjA= @@ -74,6 +78,8 @@ codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBs codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-store v1.1.5 h1:fp28vzGD15OsAF51CCwi7woH+Y3vb0aMl4OFh9JSjA0= codeberg.org/gruf/go-store v1.1.5/go.mod h1:Q6ev500ddKghDQ8KS4IstL/W9fptDKa2T9oeHP+tXsI= +codeberg.org/gruf/go-store v1.2.2 h1:YJPzJpZv/D3t9hQC00/u76eQDScQw4++OWjfobnjHAA= +codeberg.org/gruf/go-store v1.2.2/go.mod h1:Xjw1U098th0yXF2CCx6jThQ+9FIPWAX9OGjYslO+UtE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -734,6 +740,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.1 h1:O+N0Y8Re2XAYjp0adlZDA2juyRguhMfPCgh8YIf7vyE= +github.com/zeebo/blake3 v0.2.1/go.mod h1:TSQ0KjMH+pht+bRyvVooJ1rBpvvngSGaPISafq9MxJk= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -949,6 +959,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/codeberg.org/gruf/go-errors/data.go b/vendor/codeberg.org/gruf/go-errors/data.go index 3b242f03c..b5226172c 100644 --- a/vendor/codeberg.org/gruf/go-errors/data.go +++ b/vendor/codeberg.org/gruf/go-errors/data.go @@ -4,17 +4,9 @@ import ( "fmt" "sync" - "codeberg.org/gruf/go-bytes" - "codeberg.org/gruf/go-logger" + "codeberg.org/gruf/go-format" ) -// global logfmt data formatter. -var logfmt = logger.TextFormat{ - Strict: false, - Verbose: true, - MaxDepth: 5, -} - // KV is a structure for setting key-value pairs in ErrorData. type KV struct { Key string @@ -31,7 +23,7 @@ type ErrorData interface { Append(...KV) // Implement byte slice representation formatter. - logger.Formattable + format.Formattable // Implement string representation formatter. fmt.Stringer @@ -89,13 +81,22 @@ func (d *errorData) Append(kvs ...KV) { } func (d *errorData) AppendFormat(b []byte) []byte { - buf := bytes.Buffer{B: b} + buf := format.Buffer{B: b} d.mu.Lock() buf.B = append(buf.B, '{') + + // Append data as kv pairs for i := range d.data { - logfmt.AppendKey(&buf, d.data[i].Key) - logfmt.AppendValue(&buf, d.data[i].Value) + key := d.data[i].Key + val := d.data[i].Value + format.Appendf(&buf, "{:k}={:v} ", key, val) } + + // Drop trailing space + if len(d.data) > 0 { + buf.Truncate(1) + } + buf.B = append(buf.B, '}') d.mu.Unlock() return buf.B diff --git a/vendor/codeberg.org/gruf/go-format/LICENSE b/vendor/codeberg.org/gruf/go-format/LICENSE new file mode 100644 index 000000000..b7c4417ac --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2021 gruf + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/codeberg.org/gruf/go-format/README.md b/vendor/codeberg.org/gruf/go-format/README.md new file mode 100644 index 000000000..7126e215e --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/README.md @@ -0,0 +1,16 @@ +# go-format + +String formatting package using Rust-style formatting directives. + +Output is generally more visually-friendly than `"fmt"`, while performance is neck-and-neck. + +README is WIP. + +## todos + +- improved verbose printing of number types + +- more test cases + +- improved verbose printing of string ptr types + diff --git a/vendor/codeberg.org/gruf/go-format/buffer.go b/vendor/codeberg.org/gruf/go-format/buffer.go new file mode 100644 index 000000000..393f2fcd3 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/buffer.go @@ -0,0 +1,81 @@ +package format + +import ( + "io" + "unicode/utf8" + "unsafe" +) + +// ensure we conform to io.Writer. +var _ io.Writer = (*Buffer)(nil) + +// Buffer is a simple wrapper around a byte slice. +type Buffer struct { + B []byte +} + +// Write will append given byte slice to buffer, fulfilling io.Writer. +func (buf *Buffer) Write(b []byte) (int, error) { + buf.B = append(buf.B, b...) + return len(b), nil +} + +// AppendByte appends given byte to the buffer. +func (buf *Buffer) AppendByte(b byte) { + buf.B = append(buf.B, b) +} + +// AppendRune appends given rune to the buffer. +func (buf *Buffer) AppendRune(r rune) { + if r < utf8.RuneSelf { + buf.B = append(buf.B, byte(r)) + return + } + + l := buf.Len() + for i := 0; i < utf8.UTFMax; i++ { + buf.B = append(buf.B, 0) + } + n := utf8.EncodeRune(buf.B[l:buf.Len()], r) + buf.B = buf.B[:l+n] +} + +// Append will append given byte slice to the buffer. +func (buf *Buffer) Append(b []byte) { + buf.B = append(buf.B, b...) +} + +// AppendString appends given string to the buffer. +func (buf *Buffer) AppendString(s string) { + buf.B = append(buf.B, s...) +} + +// Len returns the length of the buffer's underlying byte slice. +func (buf *Buffer) Len() int { + return len(buf.B) +} + +// Cap returns the capacity of the buffer's underlying byte slice. +func (buf *Buffer) Cap() int { + return cap(buf.B) +} + +// Truncate will reduce the length of the buffer by 'n'. +func (buf *Buffer) Truncate(n int) { + if n > len(buf.B) { + n = len(buf.B) + } + buf.B = buf.B[:buf.Len()-n] +} + +// Reset will reset the buffer length to 0 (retains capacity). +func (buf *Buffer) Reset() { + buf.B = buf.B[:0] +} + +// String returns the underlying byte slice as a string. Please note +// this value is tied directly to the underlying byte slice, if you +// write to the buffer then returned string values will also change. +func (buf *Buffer) String() string { + return *(*string)(unsafe.Pointer(&buf.B)) +} diff --git a/vendor/codeberg.org/gruf/go-format/format.go b/vendor/codeberg.org/gruf/go-format/format.go new file mode 100644 index 000000000..856fe890e --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/format.go @@ -0,0 +1,565 @@ +package format + +import ( + "reflect" + "strconv" + "unsafe" +) + +// Formattable defines a type capable of being formatted and appended to a byte buffer. +type Formattable interface { + AppendFormat([]byte) []byte +} + +// format is the object passed among the append___ formatting functions. +type format struct { + flags uint8 // 'isKey' and 'verbose' flags + drefs uint8 // current value deref count + curd uint8 // current depth + maxd uint8 // maximum depth + buf *Buffer // out buffer +} + +const ( + // flag bit constants. + isKeyBit = uint8(1) << 0 + isValBit = uint8(1) << 1 + vboseBit = uint8(1) << 2 + panicBit = uint8(1) << 3 +) + +// AtMaxDepth returns whether format is currently at max depth. +func (f format) AtMaxDepth() bool { + return f.curd > f.maxd +} + +// Derefs returns no. times current value has been dereferenced. +func (f format) Derefs() uint8 { + return f.drefs +} + +// IsKey returns whether the isKey flag is set. +func (f format) IsKey() bool { + return (f.flags & isKeyBit) != 0 +} + +// IsValue returns whether the isVal flag is set. +func (f format) IsValue() bool { + return (f.flags & isValBit) != 0 +} + +// Verbose returns whether the verbose flag is set. +func (f format) Verbose() bool { + return (f.flags & vboseBit) != 0 +} + +// Panic returns whether the panic flag is set. +func (f format) Panic() bool { + return (f.flags & panicBit) != 0 +} + +// SetIsKey returns format instance with the isKey bit set to value. +func (f format) SetIsKey() format { + return format{ + flags: f.flags & ^isValBit | isKeyBit, + curd: f.curd, + maxd: f.maxd, + buf: f.buf, + } +} + +// SetIsValue returns format instance with the isVal bit set to value. +func (f format) SetIsValue() format { + return format{ + flags: f.flags & ^isKeyBit | isValBit, + curd: f.curd, + maxd: f.maxd, + buf: f.buf, + } +} + +// SetPanic returns format instance with the panic bit set to value. +func (f format) SetPanic() format { + return format{ + flags: f.flags | panicBit /* handle panic as value */ | isValBit & ^isKeyBit, + curd: f.curd, + maxd: f.maxd, + buf: f.buf, + } +} + +// IncrDepth returns format instance with depth incremented. +func (f format) IncrDepth() format { + return format{ + flags: f.flags, + curd: f.curd + 1, + maxd: f.maxd, + buf: f.buf, + } +} + +// IncrDerefs returns format instance with dereference count incremented. +func (f format) IncrDerefs() format { + return format{ + flags: f.flags, + drefs: f.drefs + 1, + curd: f.curd, + maxd: f.maxd, + buf: f.buf, + } +} + +// appendType appends a type using supplied type str. +func appendType(fmt format, t string) { + for i := uint8(0); i < fmt.Derefs(); i++ { + fmt.buf.AppendByte('*') + } + fmt.buf.AppendString(t) +} + +// appendNilType Appends nil to buf, type included if verbose. +func appendNilType(fmt format, t string) { + if fmt.Verbose() { + fmt.buf.AppendByte('(') + appendType(fmt, t) + fmt.buf.AppendString(`)(nil)`) + } else { + fmt.buf.AppendString(`nil`) + } +} + +// appendByte Appends a single byte to buf. +func appendByte(fmt format, b byte) { + if fmt.IsValue() || fmt.Verbose() { + fmt.buf.AppendString(`'` + string(b) + `'`) + } else { + fmt.buf.AppendByte(b) + } +} + +// appendBytes Appends a quoted byte slice to buf. +func appendBytes(fmt format, b []byte) { + if b == nil { + // Bytes CAN be nil formatted + appendNilType(fmt, `[]byte`) + } else { + // Append bytes as slice + fmt.buf.AppendByte('[') + for _, b := range b { + fmt.buf.AppendByte(b) + fmt.buf.AppendByte(',') + } + if len(b) > 0 { + fmt.buf.Truncate(1) + } + fmt.buf.AppendByte(']') + } +} + +// appendString Appends an escaped, double-quoted string to buf. +func appendString(fmt format, s string) { + switch { + // Key in a key-value pair + case fmt.IsKey(): + if !strconv.CanBackquote(s) { + // Requires quoting AND escaping + fmt.buf.B = strconv.AppendQuote(fmt.buf.B, s) + } else if containsSpaceOrTab(s) { + // Contains space, needs quotes + fmt.buf.AppendString(`"` + s + `"`) + } else { + // All else write as-is + fmt.buf.AppendString(s) + } + + // Value in a key-value pair (always escape+quote) + case fmt.IsValue(): + fmt.buf.B = strconv.AppendQuote(fmt.buf.B, s) + + // Verbose but neither key nor value (always quote) + case fmt.Verbose(): + fmt.buf.AppendString(`"` + s + `"`) + + // All else + default: + fmt.buf.AppendString(s) + } +} + +// appendBool Appends a formatted bool to buf. +func appendBool(fmt format, b bool) { + fmt.buf.B = strconv.AppendBool(fmt.buf.B, b) +} + +// appendInt Appends a formatted int to buf. +func appendInt(fmt format, i int64) { + fmt.buf.B = strconv.AppendInt(fmt.buf.B, i, 10) +} + +// appendUint Appends a formatted uint to buf. +func appendUint(fmt format, u uint64) { + fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 10) +} + +// appendFloat Appends a formatted float to buf. +func appendFloat(fmt format, f float64) { + fmt.buf.B = strconv.AppendFloat(fmt.buf.B, f, 'G', -1, 64) +} + +// appendComplex Appends a formatted complex128 to buf. +func appendComplex(fmt format, c complex128) { + appendFloat(fmt, real(c)) + fmt.buf.AppendByte('+') + appendFloat(fmt, imag(c)) + fmt.buf.AppendByte('i') +} + +// isNil will safely check if 'v' is nil without dealing with weird Go interface nil bullshit. +func isNil(i interface{}) bool { + e := *(*struct { + _ unsafe.Pointer // type + v unsafe.Pointer // value + })(unsafe.Pointer(&i)) + return (e.v == nil) +} + +// appendIfaceOrReflectValue will attempt to append as interface, falling back to reflection. +func appendIfaceOrRValue(fmt format, i interface{}) { + if !appendIface(fmt, i) { + appendRValue(fmt, reflect.ValueOf(i)) + } +} + +// appendValueNext checks for interface methods before performing appendRValue, checking + incr depth. +func appendRValueOrIfaceNext(fmt format, v reflect.Value) { + // Check we haven't hit max + if fmt.AtMaxDepth() { + fmt.buf.AppendString("...") + return + } + + // Incr the depth + fmt = fmt.IncrDepth() + + // Make actual call + if !v.CanInterface() || !appendIface(fmt, v.Interface()) { + appendRValue(fmt, v) + } +} + +// appendIface parses and Appends a formatted interface value to buf. +func appendIface(fmt format, i interface{}) (ok bool) { + ok = true // default + catchPanic := func() { + if r := recover(); r != nil { + // DON'T recurse catchPanic() + if fmt.Panic() { + panic(r) + } + + // Attempt to decode panic into buf + fmt.buf.AppendString(`!{PANIC=`) + appendIfaceOrRValue(fmt.SetPanic(), r) + fmt.buf.AppendByte('}') + + // Ensure return + ok = true + } + } + + switch i := i.(type) { + // Nil type + case nil: + fmt.buf.AppendString(`nil`) + + // Reflect types + case reflect.Type: + if isNil(i) /* safer nil check */ { + appendNilType(fmt, `reflect.Type`) + } else { + appendType(fmt, `reflect.Type`) + fmt.buf.AppendString(`(` + i.String() + `)`) + } + case reflect.Value: + appendType(fmt, `reflect.Value`) + fmt.buf.AppendByte('(') + fmt.flags |= vboseBit + appendRValue(fmt, i) + fmt.buf.AppendByte(')') + + // Bytes and string types + case byte: + appendByte(fmt, i) + case []byte: + appendBytes(fmt, i) + case string: + appendString(fmt, i) + + // Int types + case int: + appendInt(fmt, int64(i)) + case int8: + appendInt(fmt, int64(i)) + case int16: + appendInt(fmt, int64(i)) + case int32: + appendInt(fmt, int64(i)) + case int64: + appendInt(fmt, i) + + // Uint types + case uint: + appendUint(fmt, uint64(i)) + // case uint8 :: this is 'byte' + case uint16: + appendUint(fmt, uint64(i)) + case uint32: + appendUint(fmt, uint64(i)) + case uint64: + appendUint(fmt, i) + + // Float types + case float32: + appendFloat(fmt, float64(i)) + case float64: + appendFloat(fmt, i) + + // Bool type + case bool: + appendBool(fmt, i) + + // Complex types + case complex64: + appendComplex(fmt, complex128(i)) + case complex128: + appendComplex(fmt, i) + + // Method types + case error: + switch { + case fmt.Verbose(): + ok = false + case isNil(i) /* use safer nil check */ : + appendNilType(fmt, reflect.TypeOf(i).String()) + default: + defer catchPanic() + appendString(fmt, i.Error()) + } + case Formattable: + switch { + case fmt.Verbose(): + ok = false + case isNil(i) /* use safer nil check */ : + appendNilType(fmt, reflect.TypeOf(i).String()) + default: + defer catchPanic() + fmt.buf.B = i.AppendFormat(fmt.buf.B) + } + case interface{ String() string }: + switch { + case fmt.Verbose(): + ok = false + case isNil(i) /* use safer nil check */ : + appendNilType(fmt, reflect.TypeOf(i).String()) + default: + defer catchPanic() + appendString(fmt, i.String()) + } + + // No quick handler + default: + ok = false + } + + return ok +} + +// appendReflectValue will safely append a reflected value. +func appendRValue(fmt format, v reflect.Value) { + switch v.Kind() { + // String and byte types + case reflect.Uint8: + appendByte(fmt, byte(v.Uint())) + case reflect.String: + appendString(fmt, v.String()) + + // Float tpyes + case reflect.Float32, reflect.Float64: + appendFloat(fmt, v.Float()) + + // Int types + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + appendInt(fmt, v.Int()) + + // Uint types + case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64: + appendUint(fmt, v.Uint()) + + // Complex types + case reflect.Complex64, reflect.Complex128: + appendComplex(fmt, v.Complex()) + + // Bool type + case reflect.Bool: + appendBool(fmt, v.Bool()) + + // Slice and array types + case reflect.Array: + appendArrayType(fmt, v) + case reflect.Slice: + if v.IsNil() { + appendNilType(fmt, v.Type().String()) + } else { + appendArrayType(fmt, v) + } + + // Map types + case reflect.Map: + if v.IsNil() { + appendNilType(fmt, v.Type().String()) + } else { + appendMapType(fmt, v) + } + + // Struct types + case reflect.Struct: + appendStructType(fmt, v) + + // Deref'able ptr types + case reflect.Ptr, reflect.Interface: + if v.IsNil() { + appendNilType(fmt, v.Type().String()) + } else { + appendRValue(fmt.IncrDerefs(), v.Elem()) + } + + // 'raw' pointer types + case reflect.UnsafePointer: + appendType(fmt, `unsafe.Pointer`) + fmt.buf.AppendByte('(') + if u := v.Pointer(); u != 0 { + fmt.buf.AppendString("0x") + fmt.buf.B = strconv.AppendUint(fmt.buf.B, uint64(u), 16) + } else { + fmt.buf.AppendString(`nil`) + } + fmt.buf.AppendByte(')') + case reflect.Uintptr: + appendType(fmt, `uintptr`) + fmt.buf.AppendByte('(') + if u := v.Uint(); u != 0 { + fmt.buf.AppendString("0x") + fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 16) + } else { + fmt.buf.AppendString(`nil`) + } + fmt.buf.AppendByte(')') + + // Generic types we don't *exactly* handle + case reflect.Func, reflect.Chan: + if v.IsNil() { + appendNilType(fmt, v.Type().String()) + } else { + fmt.buf.AppendString(v.String()) + } + + // Unhandled kind + default: + fmt.buf.AppendString(v.String()) + } +} + +// appendArrayType Appends an array of unknown type (parsed by reflection) to buf, unlike appendSliceType does NOT catch nil slice. +func appendArrayType(fmt format, v reflect.Value) { + // get no. elements + n := v.Len() + + fmt.buf.AppendByte('[') + + // Append values + for i := 0; i < n; i++ { + appendRValueOrIfaceNext(fmt.SetIsValue(), v.Index(i)) + fmt.buf.AppendByte(',') + } + + // Drop last comma + if n > 0 { + fmt.buf.Truncate(1) + } + + fmt.buf.AppendByte(']') +} + +// appendMapType Appends a map of unknown types (parsed by reflection) to buf. +func appendMapType(fmt format, v reflect.Value) { + // Prepend type if verbose + if fmt.Verbose() { + appendType(fmt, v.Type().String()) + } + + // Get a map iterator + r := v.MapRange() + n := v.Len() + + fmt.buf.AppendByte('{') + + // Iterate pairs + for r.Next() { + appendRValueOrIfaceNext(fmt.SetIsKey(), r.Key()) + fmt.buf.AppendByte('=') + appendRValueOrIfaceNext(fmt.SetIsValue(), r.Value()) + fmt.buf.AppendByte(' ') + } + + // Drop last space + if n > 0 { + fmt.buf.Truncate(1) + } + + fmt.buf.AppendByte('}') +} + +// appendStructType Appends a struct (as a set of key-value fields) to buf. +func appendStructType(fmt format, v reflect.Value) { + // Get value type & no. fields + t := v.Type() + n := v.NumField() + + // Prepend type if verbose + if fmt.Verbose() { + appendType(fmt, v.Type().String()) + } + + fmt.buf.AppendByte('{') + + // Iterate fields + for i := 0; i < n; i++ { + vfield := v.Field(i) + tfield := t.Field(i) + + // Append field name + fmt.buf.AppendString(tfield.Name) + fmt.buf.AppendByte('=') + appendRValueOrIfaceNext(fmt.SetIsValue(), vfield) + + // Iter written count + fmt.buf.AppendByte(' ') + } + + // Drop last space + if n > 0 { + fmt.buf.Truncate(1) + } + + fmt.buf.AppendByte('}') +} + +// containsSpaceOrTab checks if "s" contains space or tabs. +func containsSpaceOrTab(s string) bool { + for _, r := range s { + if r == ' ' || r == '\t' { + return true + } + } + return false +} diff --git a/vendor/codeberg.org/gruf/go-format/formatter.go b/vendor/codeberg.org/gruf/go-format/formatter.go new file mode 100644 index 000000000..640fa3f04 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/formatter.go @@ -0,0 +1,352 @@ +package format + +import ( + "strings" +) + +// Formatter allows configuring value and string formatting. +type Formatter struct { + // MaxDepth specifies the max depth of fields the formatter will iterate. + // Once max depth is reached, value will simply be formatted as "...". + // e.g. + // + // MaxDepth=1 + // type A struct{ + // Nested B + // } + // type B struct{ + // Nested C + // } + // type C struct{ + // Field string + // } + // + // Append(&buf, A{}) => {Nested={Nested={Field=...}}} + MaxDepth uint8 +} + +// Append will append formatted form of supplied values into 'buf'. +func (f Formatter) Append(buf *Buffer, v ...interface{}) { + for _, v := range v { + appendIfaceOrRValue(format{maxd: f.MaxDepth, buf: buf}, v) + buf.AppendByte(' ') + } + if len(v) > 0 { + buf.Truncate(1) + } +} + +// Appendf will append the formatted string with supplied values into 'buf'. +// Supported format directives: +// - '{}' => format supplied arg, in place +// - '{0}' => format arg at index 0 of supplied, in place +// - '{:?}' => format supplied arg verbosely, in place +// - '{:k}' => format supplied arg as key, in place +// - '{:v}' => format supplied arg as value, in place +// +// To escape either of '{}' simply append an additional brace e.g. +// - '{{' => '{' +// - '}}' => '}' +// - '{{}}' => '{}' +// - '{{:?}}' => '{:?}' +// +// More formatting directives might be included in the future. +func (f Formatter) Appendf(buf *Buffer, s string, a ...interface{}) { + const ( + // ground state + modeNone = uint8(0) + + // prev reached '{' + modeOpen = uint8(1) + + // prev reached '}' + modeClose = uint8(2) + + // parsing directive index + modeIdx = uint8(3) + + // parsing directive operands + modeOp = uint8(4) + ) + + var ( + // mode is current parsing mode + mode uint8 + + // arg is the current arg index + arg int + + // carg is current directive-set arg index + carg int + + // last is the trailing cursor to see slice windows + last int + + // idx is the current index in 's' + idx int + + // fmt is the base argument formatter + fmt = format{ + maxd: f.MaxDepth, + buf: buf, + } + + // NOTE: these functions are defined here as function + // locals as it turned out to be better for performance + // doing it this way, than encapsulating their logic in + // some kind of parsing structure. Maybe if the parser + // was pooled along with the buffers it might work out + // better, but then it makes more internal functions i.e. + // .Append() .Appendf() less accessible outside package. + // + // Currently, passing '-gcflags "-l=4"' causes a not + // insignificant decrease in ns/op, which is likely due + // to more aggressive function inlining, which this + // function can obviously stand to benefit from :) + + // Str returns current string window slice, and updates + // the trailing cursor 'last' to current 'idx' + Str = func() string { + str := s[last:idx] + last = idx + return str + } + + // MoveUp moves the trailing cursor 'last' just past 'idx' + MoveUp = func() { + last = idx + 1 + } + + // MoveUpTo moves the trailing cursor 'last' either up to + // closest '}', or current 'idx', whichever is furthest + MoveUpTo = func() { + if i := strings.IndexByte(s[idx:], '}'); i >= 0 { + idx += i + } + MoveUp() + } + + // ParseIndex parses an integer from the current string + // window, updating 'last' to 'idx'. The string window + // is ASSUMED to contain only valid ASCII numbers. This + // only returns false if number exceeds platform int size + ParseIndex = func() bool { + // Get current window + str := Str() + if len(str) < 1 { + return true + } + + // Index HAS to fit within platform int + if !can32bitInt(str) && !can64bitInt(str) { + return false + } + + // Build integer from string + carg = 0 + for _, c := range []byte(str) { + carg = carg*10 + int(c-'0') + } + + return true + } + + // ParseOp parses operands from the current string + // window, updating 'last' to 'idx'. The string window + // is ASSUMED to contain only valid operand ASCII. This + // returns success on parsing of operand logic + ParseOp = func() bool { + // Get current window + str := Str() + if len(str) < 1 { + return true + } + + // (for now) only + // accept length = 1 + if len(str) > 1 { + return false + } + + switch str[0] { + case 'k': + fmt.flags |= isKeyBit + case 'v': + fmt.flags |= isValBit + case '?': + fmt.flags |= vboseBit + } + + return true + } + + // AppendArg will take either the directive-set, or + // iterated arg index, check within bounds of 'a' and + // append the that argument formatted to the buffer. + // On failure, it will append an error string + AppendArg = func() { + // Look for idx + if carg < 0 { + carg = arg + } + + // Incr idx + arg++ + + if carg < len(a) { + // Append formatted argument value + appendIfaceOrRValue(fmt, a[carg]) + } else { + // No argument found for index + buf.AppendString(`!{MISSING_ARG}`) + } + } + + // Reset will reset the mode to ground, the flags + // to empty and parsed 'carg' to empty + Reset = func() { + mode = modeNone + fmt.flags = 0 + carg = -1 + } + ) + + for idx = 0; idx < len(s); idx++ { + // Get next char + c := s[idx] + + switch mode { + // Ground mode + case modeNone: + switch c { + case '{': + // Enter open mode + buf.AppendString(Str()) + mode = modeOpen + MoveUp() + case '}': + // Enter close mode + buf.AppendString(Str()) + mode = modeClose + MoveUp() + } + + // Encountered open '{' + case modeOpen: + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + // Starting index + mode = modeIdx + MoveUp() + case '{': + // Escaped bracket + buf.AppendByte('{') + mode = modeNone + MoveUp() + case '}': + // Format arg + AppendArg() + Reset() + MoveUp() + case ':': + // Starting operands + mode = modeOp + MoveUp() + default: + // Bad char, missing a close + buf.AppendString(`!{MISSING_CLOSE}`) + mode = modeNone + MoveUpTo() + } + + // Encountered close '}' + case modeClose: + switch c { + case '}': + // Escaped close bracket + buf.AppendByte('}') + mode = modeNone + MoveUp() + default: + // Missing an open bracket + buf.AppendString(`!{MISSING_OPEN}`) + mode = modeNone + MoveUp() + } + + // Preparing index + case modeIdx: + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + case ':': + if !ParseIndex() { + // Unable to parse an integer + buf.AppendString(`!{BAD_INDEX}`) + mode = modeNone + MoveUpTo() + } else { + // Starting operands + mode = modeOp + MoveUp() + } + case '}': + if !ParseIndex() { + // Unable to parse an integer + buf.AppendString(`!{BAD_INDEX}`) + } else { + // Format arg + AppendArg() + } + Reset() + MoveUp() + default: + // Not a valid index character + buf.AppendString(`!{BAD_INDEX}`) + mode = modeNone + MoveUpTo() + } + + // Preparing operands + case modeOp: + switch c { + case 'k', 'v', '?': + // TODO: set flags as received + case '}': + if !ParseOp() { + // Unable to parse operands + buf.AppendString(`!{BAD_OPERAND}`) + } else { + // Format arg + AppendArg() + } + Reset() + MoveUp() + default: + // Not a valid operand char + buf.AppendString(`!{BAD_OPERAND}`) + Reset() + MoveUpTo() + } + } + } + + // Append any remaining + buf.AppendString(s[last:]) +} + +// formatter is the default formatter instance. +var formatter = Formatter{ + MaxDepth: 10, +} + +// Append will append formatted form of supplied values into 'buf' using default formatter. +// See Formatter.Append() for more documentation. +func Append(buf *Buffer, v ...interface{}) { + formatter.Append(buf, v...) +} + +// Appendf will append the formatted string with supplied values into 'buf' using default formatter. +// See Formatter.Appendf() for more documentation. +func Appendf(buf *Buffer, s string, a ...interface{}) { + formatter.Appendf(buf, s, a...) +} diff --git a/vendor/codeberg.org/gruf/go-format/print.go b/vendor/codeberg.org/gruf/go-format/print.go new file mode 100644 index 000000000..288e6af10 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/print.go @@ -0,0 +1,88 @@ +package format + +import ( + "io" + "os" + "sync" +) + +// pool is the global printer buffer pool. +var pool = sync.Pool{ + New: func() interface{} { + return &Buffer{} + }, +} + +// getBuf fetches a buffer from pool. +func getBuf() *Buffer { + return pool.Get().(*Buffer) +} + +// putBuf places a Buffer back in pool. +func putBuf(buf *Buffer) { + if buf.Cap() > 64<<10 { + return // drop large + } + buf.Reset() + pool.Put(buf) +} + +// Sprint will format supplied values, returning this string. +func Sprint(v ...interface{}) string { + buf := Buffer{} + Append(&buf, v...) + return buf.String() +} + +// Sprintf will format supplied format string and args, returning this string. +// See Formatter.Appendf() for more documentation. +func Sprintf(s string, a ...interface{}) string { + buf := Buffer{} + Appendf(&buf, s, a...) + return buf.String() +} + +// Print will format supplied values, print this to os.Stdout. +func Print(v ...interface{}) { + Fprint(os.Stdout, v...) //nolint +} + +// Printf will format supplied format string and args, printing this to os.Stdout. +// See Formatter.Appendf() for more documentation. +func Printf(s string, a ...interface{}) { + Fprintf(os.Stdout, s, a...) //nolint +} + +// Println will format supplied values, append a trailing newline and print this to os.Stdout. +func Println(v ...interface{}) { + Fprintln(os.Stdout, v...) //nolint +} + +// Fprint will format supplied values, writing this to an io.Writer. +func Fprint(w io.Writer, v ...interface{}) (int, error) { + buf := getBuf() + Append(buf, v...) + n, err := w.Write(buf.B) + putBuf(buf) + return n, err +} + +// Fprintf will format supplied format string and args, writing this to an io.Writer. +// See Formatter.Appendf() for more documentation. +func Fprintf(w io.Writer, s string, a ...interface{}) (int, error) { + buf := getBuf() + Appendf(buf, s, a...) + n, err := w.Write(buf.B) + putBuf(buf) + return n, err +} + +// Println will format supplied values, append a trailing newline and writer this to an io.Writer. +func Fprintln(w io.Writer, v ...interface{}) (int, error) { + buf := getBuf() + Append(buf, v...) + buf.AppendByte('\n') + n, err := w.Write(buf.B) + putBuf(buf) + return n, err +} diff --git a/vendor/codeberg.org/gruf/go-format/util.go b/vendor/codeberg.org/gruf/go-format/util.go new file mode 100644 index 000000000..68a9e2de3 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-format/util.go @@ -0,0 +1,13 @@ +package format + +import "strconv" + +// can32bitInt returns whether it's possible for 's' to contain an int on 32bit platforms. +func can32bitInt(s string) bool { + return strconv.IntSize == 32 && (0 < len(s) && len(s) < 10) +} + +// can64bitInt returns whether it's possible for 's' to contain an int on 64bit platforms. +func can64bitInt(s string) bool { + return strconv.IntSize == 64 && (0 < len(s) && len(s) < 19) +} diff --git a/vendor/codeberg.org/gruf/go-logger/LICENSE b/vendor/codeberg.org/gruf/go-logger/LICENSE deleted file mode 100644 index b7c4417ac..000000000 --- a/vendor/codeberg.org/gruf/go-logger/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2021 gruf - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/codeberg.org/gruf/go-logger/README.md b/vendor/codeberg.org/gruf/go-logger/README.md deleted file mode 100644 index 57410ea87..000000000 --- a/vendor/codeberg.org/gruf/go-logger/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Fast levelled logging package with customizable formatting. - -Supports logging in 2 modes: -- no locks, fastest possible logging, no guarantees for io.Writer thread safety -- mutex locks during writes, still far faster than standard library logger - -Running without locks isn't likely to cause you any issues*, but if it does, you can wrap your `io.Writer` using `AddSafety()` when instantiating your new Logger. Even when running the benchmarks, this library has no printing issues without locks, so in most cases you'll be fine, but the safety is there if you need it. - -*most logging libraries advertising high speeds are likely not performing mutex locks, which is why with this library you have the option to opt-in/out of them. - -Note there are 2 uses of the unsafe package: -- safer interface nil value checks, uses similar logic to reflect package to check if the value in the internal fat pointer is nil -- casting a byte slice to string to allow sharing of similar byte and string methods, performs same logic as `strings.Builder{}.String()` \ No newline at end of file diff --git a/vendor/codeberg.org/gruf/go-logger/clock.go b/vendor/codeberg.org/gruf/go-logger/clock.go deleted file mode 100644 index cc7d7ed0c..000000000 --- a/vendor/codeberg.org/gruf/go-logger/clock.go +++ /dev/null @@ -1,21 +0,0 @@ -package logger - -import ( - "sync" - "time" - - "codeberg.org/gruf/go-nowish" -) - -var ( - clock = nowish.Clock{} - clockOnce = sync.Once{} -) - -// startClock starts the global nowish clock. -func startClock() { - clockOnce.Do(func() { - clock.Start(time.Millisecond * 100) - clock.SetFormat("2006-01-02 15:04:05") - }) -} diff --git a/vendor/codeberg.org/gruf/go-logger/default.go b/vendor/codeberg.org/gruf/go-logger/default.go deleted file mode 100644 index 3fd65c6b1..000000000 --- a/vendor/codeberg.org/gruf/go-logger/default.go +++ /dev/null @@ -1,107 +0,0 @@ -package logger - -import ( - "os" - "sync" -) - -var ( - instance *Logger - instanceOnce = sync.Once{} -) - -// Default returns the default Logger instance. -func Default() *Logger { - instanceOnce.Do(func() { instance = New(os.Stdout) }) - return instance -} - -// Debug prints the provided arguments with the debug prefix to the global Logger instance. -func Debug(a ...interface{}) { - Default().Debug(a...) -} - -// Debugf prints the provided format string and arguments with the debug prefix to the global Logger instance. -func Debugf(s string, a ...interface{}) { - Default().Debugf(s, a...) -} - -// Info prints the provided arguments with the info prefix to the global Logger instance. -func Info(a ...interface{}) { - Default().Info(a...) -} - -// Infof prints the provided format string and arguments with the info prefix to the global Logger instance. -func Infof(s string, a ...interface{}) { - Default().Infof(s, a...) -} - -// Warn prints the provided arguments with the warn prefix to the global Logger instance. -func Warn(a ...interface{}) { - Default().Warn(a...) -} - -// Warnf prints the provided format string and arguments with the warn prefix to the global Logger instance. -func Warnf(s string, a ...interface{}) { - Default().Warnf(s, a...) -} - -// Error prints the provided arguments with the error prefix to the global Logger instance. -func Error(a ...interface{}) { - Default().Error(a...) -} - -// Errorf prints the provided format string and arguments with the error prefix to the global Logger instance. -func Errorf(s string, a ...interface{}) { - Default().Errorf(s, a...) -} - -// Fatal prints the provided arguments with the fatal prefix to the global Logger instance before exiting the program with os.Exit(1). -func Fatal(a ...interface{}) { - Default().Fatal(a...) -} - -// Fatalf prints the provided format string and arguments with the fatal prefix to the global Logger instance before exiting the program with os.Exit(1). -func Fatalf(s string, a ...interface{}) { - Default().Fatalf(s, a...) -} - -// Log prints the provided arguments with the supplied log level to the global Logger instance. -func Log(lvl LEVEL, a ...interface{}) { - Default().Log(lvl, a...) -} - -// Logf prints the provided format string and arguments with the supplied log level to the global Logger instance. -func Logf(lvl LEVEL, s string, a ...interface{}) { - Default().Logf(lvl, s, a...) -} - -// LogFields prints the provided fields formatted as key-value pairs at the supplied log level to the global Logger instance. -func LogFields(lvl LEVEL, fields map[string]interface{}) { - Default().LogFields(lvl, fields) -} - -// LogValues prints the provided values formatted as-so at the supplied log level to the global Logger instance. -func LogValues(lvl LEVEL, a ...interface{}) { - Default().LogValues(lvl, a...) -} - -// Print simply prints provided arguments to the global Logger instance. -func Print(a ...interface{}) { - Default().Print(a...) -} - -// Printf simply prints provided the provided format string and arguments to the global Logger instance. -func Printf(s string, a ...interface{}) { - Default().Printf(s, a...) -} - -// PrintFields prints the provided fields formatted as key-value pairs to the global Logger instance. -func PrintFields(fields map[string]interface{}) { - Default().PrintFields(fields) -} - -// PrintValues prints the provided values formatted as-so to the global Logger instance. -func PrintValues(a ...interface{}) { - Default().PrintValues(a...) -} diff --git a/vendor/codeberg.org/gruf/go-logger/entry.go b/vendor/codeberg.org/gruf/go-logger/entry.go deleted file mode 100644 index 11e383086..000000000 --- a/vendor/codeberg.org/gruf/go-logger/entry.go +++ /dev/null @@ -1,385 +0,0 @@ -package logger - -import ( - "context" - "fmt" - "time" - - "codeberg.org/gruf/go-bytes" -) - -// Entry defines an entry in the log, it is NOT safe for concurrent use -type Entry struct { - ctx context.Context - lvl LEVEL - buf *bytes.Buffer - log *Logger -} - -// Context returns the current set Entry context.Context -func (e *Entry) Context() context.Context { - return e.ctx -} - -// WithContext updates Entry context value to the supplied -func (e *Entry) WithContext(ctx context.Context) *Entry { - e.ctx = ctx - return e -} - -// Level appends the supplied level to the log entry, and sets the entry level. -// Please note this CAN be called and append log levels multiple times -func (e *Entry) Level(lvl LEVEL) *Entry { - e.log.Format.AppendLevel(e.buf, lvl) - e.buf.WriteByte(' ') - e.lvl = lvl - return e -} - -// Timestamp appends the current timestamp to the log entry. Please note this -// CAN be called and append the timestamp multiple times -func (e *Entry) Timestamp() *Entry { - e.log.Format.AppendTimestamp(e.buf, clock.NowFormat()) - e.buf.WriteByte(' ') - return e -} - -// TimestampIf performs Entry.Timestamp() only IF timestamping is enabled for the Logger. -// Please note this CAN be called multiple times -func (e *Entry) TimestampIf() *Entry { - if e.log.Timestamp { - e.Timestamp() - } - return e -} - -// Hooks applies currently set Hooks to the Entry. Please note this CAN be -// called and perform the Hooks multiple times -func (e *Entry) Hooks() *Entry { - for _, hook := range e.log.Hooks { - hook.Do(e) - } - return e -} - -// Byte appends a byte value to the log entry -func (e *Entry) Byte(value byte) *Entry { - e.log.Format.AppendByte(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// ByteField appends a byte value as key-value pair to the log entry -func (e *Entry) ByteField(key string, value byte) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendByte(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Bytes appends a byte slice value as to the log entry -func (e *Entry) Bytes(value []byte) *Entry { - e.log.Format.AppendBytes(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// BytesField appends a byte slice value as key-value pair to the log entry -func (e *Entry) BytesField(key string, value []byte) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendBytes(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Str appends a string value to the log entry -func (e *Entry) Str(value string) *Entry { - e.log.Format.AppendString(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// StrField appends a string value as key-value pair to the log entry -func (e *Entry) StrField(key string, value string) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendString(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Strs appends a string slice value to the log entry -func (e *Entry) Strs(value []string) *Entry { - e.log.Format.AppendStrings(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// StrsField appends a string slice value as key-value pair to the log entry -func (e *Entry) StrsField(key string, value []string) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendStrings(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Int appends an int value to the log entry -func (e *Entry) Int(value int) *Entry { - e.log.Format.AppendInt(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// IntField appends an int value as key-value pair to the log entry -func (e *Entry) IntField(key string, value int) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendInt(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Ints appends an int slice value to the log entry -func (e *Entry) Ints(value []int) *Entry { - e.log.Format.AppendInts(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// IntsField appends an int slice value as key-value pair to the log entry -func (e *Entry) IntsField(key string, value []int) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendInts(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Uint appends a uint value to the log entry -func (e *Entry) Uint(value uint) *Entry { - e.log.Format.AppendUint(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// UintField appends a uint value as key-value pair to the log entry -func (e *Entry) UintField(key string, value uint) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendUint(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Uints appends a uint slice value to the log entry -func (e *Entry) Uints(value []uint) *Entry { - e.log.Format.AppendUints(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// UintsField appends a uint slice value as key-value pair to the log entry -func (e *Entry) UintsField(key string, value []uint) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendUints(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Float appends a float value to the log entry -func (e *Entry) Float(value float64) *Entry { - e.log.Format.AppendFloat(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// FloatField appends a float value as key-value pair to the log entry -func (e *Entry) FloatField(key string, value float64) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendFloat(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Floats appends a float slice value to the log entry -func (e *Entry) Floats(value []float64) *Entry { - e.log.Format.AppendFloats(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// FloatsField appends a float slice value as key-value pair to the log entry -func (e *Entry) FloatsField(key string, value []float64) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendFloats(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Bool appends a bool value to the log entry -func (e *Entry) Bool(value bool) *Entry { - e.log.Format.AppendBool(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// BoolField appends a bool value as key-value pair to the log entry -func (e *Entry) BoolField(key string, value bool) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendBool(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Bools appends a bool slice value to the log entry -func (e *Entry) Bools(value []bool) *Entry { - e.log.Format.AppendBools(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// BoolsField appends a bool slice value as key-value pair to the log entry -func (e *Entry) BoolsField(key string, value []bool) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendBools(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Time appends a time.Time value to the log entry -func (e *Entry) Time(value time.Time) *Entry { - e.log.Format.AppendTime(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// TimeField appends a time.Time value as key-value pair to the log entry -func (e *Entry) TimeField(key string, value time.Time) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendTime(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Times appends a time.Time slice value to the log entry -func (e *Entry) Times(value []time.Time) *Entry { - e.log.Format.AppendTimes(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// TimesField appends a time.Time slice value as key-value pair to the log entry -func (e *Entry) TimesField(key string, value []time.Time) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendTimes(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// DurationField appends a time.Duration value to the log entry -func (e *Entry) Duration(value time.Duration) *Entry { - e.log.Format.AppendDuration(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// DurationField appends a time.Duration value as key-value pair to the log entry -func (e *Entry) DurationField(key string, value time.Duration) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendDuration(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Durations appends a time.Duration slice value to the log entry -func (e *Entry) Durations(value []time.Duration) *Entry { - e.log.Format.AppendDurations(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// DurationsField appends a time.Duration slice value as key-value pair to the log entry -func (e *Entry) DurationsField(key string, value []time.Duration) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendDurations(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Field appends an interface value as key-value pair to the log entry -func (e *Entry) Field(key string, value interface{}) *Entry { - e.log.Format.AppendKey(e.buf, key) - e.log.Format.AppendValue(e.buf, value) - e.buf.WriteByte(' ') - return e -} - -// Fields appends a map of key-value pairs to the log entry -func (e *Entry) Fields(fields map[string]interface{}) *Entry { - for key, value := range fields { - e.Field(key, value) - } - return e -} - -// Values appends the given values to the log entry formatted as values, without a key. -func (e *Entry) Values(values ...interface{}) *Entry { - for _, value := range values { - e.log.Format.AppendValue(e.buf, value) - e.buf.WriteByte(' ') - } - return e -} - -// Append will append the given args formatted using fmt.Sprint(a...) to the Entry. -func (e *Entry) Append(a ...interface{}) *Entry { - fmt.Fprint(e.buf, a...) - e.buf.WriteByte(' ') - return e -} - -// Appendf will append the given format string and args using fmt.Sprintf(s, a...) to the Entry. -func (e *Entry) Appendf(s string, a ...interface{}) *Entry { - fmt.Fprintf(e.buf, s, a...) - e.buf.WriteByte(' ') - return e -} - -// Msg appends the fmt.Sprint() formatted final message to the log and calls .Send() -func (e *Entry) Msg(a ...interface{}) { - e.log.Format.AppendMsg(e.buf, a...) - e.Send() -} - -// Msgf appends the fmt.Sprintf() formatted final message to the log and calls .Send() -func (e *Entry) Msgf(s string, a ...interface{}) { - e.log.Format.AppendMsgf(e.buf, s, a...) - e.Send() -} - -// Send triggers write of the log entry, skipping if the entry's log LEVEL -// is below the currently set Logger level, and releases the Entry back to -// the Logger's Entry pool. So it is NOT safe to continue using this Entry -// object after calling .Send(), .Msg() or .Msgf() -func (e *Entry) Send() { - // If nothing to do, return - if e.lvl < e.log.Level || e.buf.Len() < 1 { - e.reset() - return - } - - // Ensure a final new line - if e.buf.B[e.buf.Len()-1] != '\n' { - e.buf.WriteByte('\n') - } - - // Write, reset and release - e.log.Output.Write(e.buf.B) - e.reset() -} - -func (e *Entry) reset() { - // Reset all - e.ctx = nil - e.buf.Reset() - e.lvl = unset - - // Release to pool - e.log.pool.Put(e) -} diff --git a/vendor/codeberg.org/gruf/go-logger/format.go b/vendor/codeberg.org/gruf/go-logger/format.go deleted file mode 100644 index 3901ea37f..000000000 --- a/vendor/codeberg.org/gruf/go-logger/format.go +++ /dev/null @@ -1,87 +0,0 @@ -package logger - -import ( - "time" - - "codeberg.org/gruf/go-bytes" -) - -// Check our types impl LogFormat -var _ LogFormat = &TextFormat{} - -// Formattable defines a type capable of writing a string formatted form -// of itself to a supplied byte buffer, and returning the resulting byte -// buffer. Implementing this will greatly speed up formatting of custom -// types passed to LogFormat (assuming they implement checking for this). -type Formattable interface { - AppendFormat([]byte) []byte -} - -// LogFormat defines a method of formatting log entries -type LogFormat interface { - // AppendKey appends given key to the log buffer - AppendKey(buf *bytes.Buffer, key string) - - // AppendLevel appends given log level as key-value pair to the log buffer - AppendLevel(buf *bytes.Buffer, lvl LEVEL) - - // AppendTimestamp appends given timestamp string as key-value pair to the log buffer - AppendTimestamp(buf *bytes.Buffer, fmtNow string) - - // AppendValue appends given interface formatted as value to the log buffer - AppendValue(buf *bytes.Buffer, value interface{}) - - // AppendByte appends given byte value to the log buffer - AppendByte(buf *bytes.Buffer, value byte) - - // AppendBytes appends given byte slice value to the log buffer - AppendBytes(buf *bytes.Buffer, value []byte) - - // AppendString appends given string value to the log buffer - AppendString(buf *bytes.Buffer, value string) - - // AppendStrings appends given string slice value to the log buffer - AppendStrings(buf *bytes.Buffer, value []string) - - // AppendBool appends given bool value to the log buffer - AppendBool(buf *bytes.Buffer, value bool) - - // AppendBools appends given bool slice value to the log buffer - AppendBools(buf *bytes.Buffer, value []bool) - - // AppendInt appends given int value to the log buffer - AppendInt(buf *bytes.Buffer, value int) - - // AppendInts appends given int slice value to the log buffer - AppendInts(buf *bytes.Buffer, value []int) - - // AppendUint appends given uint value to the log buffer - AppendUint(buf *bytes.Buffer, value uint) - - // AppendUints appends given uint slice value to the log buffer - AppendUints(buf *bytes.Buffer, value []uint) - - // AppendFloat appends given float value to the log buffer - AppendFloat(buf *bytes.Buffer, value float64) - - // AppendFloats appends given float slice value to the log buffer - AppendFloats(buf *bytes.Buffer, value []float64) - - // AppendTime appends given time value to the log buffer - AppendTime(buf *bytes.Buffer, value time.Time) - - // AppendTimes appends given time slice value to the log buffer - AppendTimes(buf *bytes.Buffer, value []time.Time) - - // AppendDuration appends given duration value to the log buffer - AppendDuration(buf *bytes.Buffer, value time.Duration) - - // AppendDurations appends given duration slice value to the log buffer - AppendDurations(buf *bytes.Buffer, value []time.Duration) - - // AppendMsg appends given msg as key-value pair to the log buffer using fmt.Sprint(...) formatting - AppendMsg(buf *bytes.Buffer, a ...interface{}) - - // AppendMsgf appends given msg format string as key-value pair to the log buffer using fmt.Sprintf(...) formatting - AppendMsgf(buf *bytes.Buffer, s string, a ...interface{}) -} diff --git a/vendor/codeberg.org/gruf/go-logger/format_text.go b/vendor/codeberg.org/gruf/go-logger/format_text.go deleted file mode 100644 index f9c90f887..000000000 --- a/vendor/codeberg.org/gruf/go-logger/format_text.go +++ /dev/null @@ -1,914 +0,0 @@ -package logger - -import ( - stdfmt "fmt" - "reflect" - "strconv" - "time" - "unsafe" - - "codeberg.org/gruf/go-bytes" -) - -// DefaultTextFormat is the default TextFormat instance -var DefaultTextFormat = TextFormat{ - Strict: false, - Verbose: false, - MaxDepth: 10, - Levels: DefaultLevels(), -} - -// TextFormat is the default LogFormat implementation, with very similar formatting to the -// standard "fmt" package's '%#v' operator. The main difference being that pointers are -// dereferenced as far as possible in order to reach a printable value. It is also *mildly* faster. -type TextFormat struct { - // Strict defines whether to use strict key-value pair formatting, i.e. should the level - // timestamp and msg be formatted as key-value pairs (with forced quoting for msg) - Strict bool - - // Verbose defines whether to increase output verbosity, i.e. include types with nil values - // and force values implementing .String() / .AppendFormat() to be printed as a struct etc. - Verbose bool - - // MaxDepth specifies the max depth of fields the formatter will iterate - MaxDepth uint8 - - // Levels defines the map of log LEVELs to level strings - Levels Levels -} - -// fmt returns a new format instance based on receiver TextFormat and given buffer -func (f TextFormat) fmt(buf *bytes.Buffer) format { - var flags uint8 - if f.Verbose { - flags |= vboseBit - } - return format{ - flags: flags, - curd: 0, - maxd: f.MaxDepth, - buf: buf, - } -} - -func (f TextFormat) AppendKey(buf *bytes.Buffer, key string) { - if len(key) > 0 { - // only append if key is non-zero length - appendString(f.fmt(buf).SetIsKey(true), key) - buf.WriteByte('=') - } -} - -func (f TextFormat) AppendLevel(buf *bytes.Buffer, lvl LEVEL) { - if f.Strict { - // Strict format, append level key - buf.WriteString(`level=`) - buf.WriteString(f.Levels.Get(lvl)) - return - } - - // Write level string - buf.WriteByte('[') - buf.WriteString(f.Levels.Get(lvl)) - buf.WriteByte(']') -} - -func (f TextFormat) AppendTimestamp(buf *bytes.Buffer, now string) { - if f.Strict { - // Strict format, use key and quote - buf.WriteString(`time=`) - appendString(f.fmt(buf), now) - return - } - - // Write time as-is - buf.WriteString(now) -} - -func (f TextFormat) AppendValue(buf *bytes.Buffer, value interface{}) { - appendIfaceOrRValue(f.fmt(buf).SetIsKey(false), value) -} - -func (f TextFormat) AppendByte(buf *bytes.Buffer, value byte) { - appendByte(f.fmt(buf), value) -} - -func (f TextFormat) AppendBytes(buf *bytes.Buffer, value []byte) { - appendBytes(f.fmt(buf), value) -} - -func (f TextFormat) AppendString(buf *bytes.Buffer, value string) { - appendString(f.fmt(buf), value) -} - -func (f TextFormat) AppendStrings(buf *bytes.Buffer, value []string) { - appendStringSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendBool(buf *bytes.Buffer, value bool) { - appendBool(f.fmt(buf), value) -} - -func (f TextFormat) AppendBools(buf *bytes.Buffer, value []bool) { - appendBoolSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendInt(buf *bytes.Buffer, value int) { - appendInt(f.fmt(buf), int64(value)) -} - -func (f TextFormat) AppendInts(buf *bytes.Buffer, value []int) { - appendIntSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendUint(buf *bytes.Buffer, value uint) { - appendUint(f.fmt(buf), uint64(value)) -} - -func (f TextFormat) AppendUints(buf *bytes.Buffer, value []uint) { - appendUintSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendFloat(buf *bytes.Buffer, value float64) { - appendFloat(f.fmt(buf), value) -} - -func (f TextFormat) AppendFloats(buf *bytes.Buffer, value []float64) { - appendFloatSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendTime(buf *bytes.Buffer, value time.Time) { - appendTime(f.fmt(buf), value) -} - -func (f TextFormat) AppendTimes(buf *bytes.Buffer, value []time.Time) { - appendTimeSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendDuration(buf *bytes.Buffer, value time.Duration) { - appendDuration(f.fmt(buf), value) -} - -func (f TextFormat) AppendDurations(buf *bytes.Buffer, value []time.Duration) { - appendDurationSlice(f.fmt(buf), value) -} - -func (f TextFormat) AppendMsg(buf *bytes.Buffer, a ...interface{}) { - if f.Strict { - // Strict format, use key and quote - buf.WriteString(`msg=`) - buf.B = strconv.AppendQuote(buf.B, stdfmt.Sprint(a...)) - return - } - - // Write message as-is - stdfmt.Fprint(buf, a...) -} - -func (f TextFormat) AppendMsgf(buf *bytes.Buffer, s string, a ...interface{}) { - if f.Strict { - // Strict format, use key and quote - buf.WriteString(`msg=`) - buf.B = strconv.AppendQuote(buf.B, stdfmt.Sprintf(s, a...)) - return - } - - // Write message as-is - stdfmt.Fprintf(buf, s, a...) -} - -// format is the object passed among the append___ formatting functions -type format struct { - flags uint8 // 'isKey' and 'verbose' flags - drefs uint8 // current value deref count - curd uint8 // current depth - maxd uint8 // maximum depth - buf *bytes.Buffer // out buffer -} - -const ( - // flag bit constants - isKeyBit = uint8(1) << 0 - vboseBit = uint8(1) << 1 -) - -// AtMaxDepth returns whether format is currently at max depth. -func (f format) AtMaxDepth() bool { - return f.curd >= f.maxd -} - -// Derefs returns no. times current value has been dereferenced. -func (f format) Derefs() uint8 { - return f.drefs -} - -// IsKey returns whether the isKey flag is set. -func (f format) IsKey() bool { - return (f.flags & isKeyBit) != 0 -} - -// Verbose returns whether the verbose flag is set. -func (f format) Verbose() bool { - return (f.flags & vboseBit) != 0 -} - -// SetIsKey returns format instance with the isKey bit set to value. -func (f format) SetIsKey(is bool) format { - flags := f.flags - if is { - flags |= isKeyBit - } else { - flags &= ^isKeyBit - } - return format{ - flags: flags, - drefs: f.drefs, - curd: f.curd, - maxd: f.maxd, - buf: f.buf, - } -} - -// IncrDepth returns format instance with depth incremented. -func (f format) IncrDepth() format { - return format{ - flags: f.flags, - drefs: f.drefs, - curd: f.curd + 1, - maxd: f.maxd, - buf: f.buf, - } -} - -// IncrDerefs returns format instance with dereference count incremented. -func (f format) IncrDerefs() format { - return format{ - flags: f.flags, - drefs: f.drefs + 1, - curd: f.curd, - maxd: f.maxd, - buf: f.buf, - } -} - -// appendType appends a type using supplied type str. -func appendType(fmt format, t string) { - for i := uint8(0); i < fmt.Derefs(); i++ { - fmt.buf.WriteByte('*') - } - fmt.buf.WriteString(t) -} - -// appendNilType writes nil to buf, type included if verbose. -func appendNilType(fmt format, t string) { - if fmt.Verbose() { - fmt.buf.WriteByte('(') - appendType(fmt, t) - fmt.buf.WriteString(`)(nil)`) - } else { - fmt.buf.WriteString(`nil`) - } -} - -// appendNilFace writes nil to buf, type included if verbose. -func appendNilIface(fmt format, i interface{}) { - if fmt.Verbose() { - fmt.buf.WriteByte('(') - appendType(fmt, reflect.TypeOf(i).String()) - fmt.buf.WriteString(`)(nil)`) - } else { - fmt.buf.WriteString(`nil`) - } -} - -// appendNilRValue writes nil to buf, type included if verbose. -func appendNilRValue(fmt format, v reflect.Value) { - if fmt.Verbose() { - fmt.buf.WriteByte('(') - appendType(fmt, v.Type().String()) - fmt.buf.WriteString(`)(nil)`) - } else { - fmt.buf.WriteString(`nil`) - } -} - -// appendByte writes a single byte to buf -func appendByte(fmt format, b byte) { - fmt.buf.WriteByte(b) -} - -// appendBytes writes a quoted byte slice to buf -func appendBytes(fmt format, b []byte) { - if !fmt.IsKey() && b == nil { - // Values CAN be nil formatted - appendNilType(fmt, `[]byte`) - } else { - // unsafe cast as string to prevent reallocation - appendString(fmt, *(*string)(unsafe.Pointer(&b))) - } -} - -// appendString writes an escaped, double-quoted string to buf -func appendString(fmt format, s string) { - if !fmt.IsKey() || !strconv.CanBackquote(s) { - // All non-keys and multiline keys get quoted + escaped - fmt.buf.B = strconv.AppendQuote(fmt.buf.B, s) - return - } else if containsSpaceOrTab(s) { - // Key containing spaces/tabs, quote this - fmt.buf.WriteByte('"') - fmt.buf.WriteString(s) - fmt.buf.WriteByte('"') - return - } - - // Safe to leave unquoted - fmt.buf.WriteString(s) -} - -// appendStringSlice writes a slice of strings to buf -func appendStringSlice(fmt format, s []string) { - // Check for nil slice - if s == nil { - appendNilType(fmt, `[]string`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, s := range s { - appendString(fmt.SetIsKey(false), s) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(s) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendBool writes a formatted bool to buf -func appendBool(fmt format, b bool) { - fmt.buf.B = strconv.AppendBool(fmt.buf.B, b) -} - -// appendBool writes a slice of formatted bools to buf -func appendBoolSlice(fmt format, b []bool) { - // Check for nil slice - if b == nil { - appendNilType(fmt, `[]bool`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, b := range b { - appendBool(fmt, b) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(b) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendInt writes a formatted int to buf -func appendInt(fmt format, i int64) { - fmt.buf.B = strconv.AppendInt(fmt.buf.B, i, 10) -} - -// appendIntSlice writes a slice of formatted int to buf -func appendIntSlice(fmt format, i []int) { - // Check for nil slice - if i == nil { - appendNilType(fmt, `[]int`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, i := range i { - appendInt(fmt, int64(i)) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(i) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendUint writes a formatted uint to buf -func appendUint(fmt format, u uint64) { - fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 10) -} - -// appendUintSlice writes a slice of formatted uint to buf -func appendUintSlice(fmt format, u []uint) { - // Check for nil slice - if u == nil { - appendNilType(fmt, `[]uint`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, u := range u { - appendUint(fmt, uint64(u)) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(u) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendFloat writes a formatted float to buf -func appendFloat(fmt format, f float64) { - fmt.buf.B = strconv.AppendFloat(fmt.buf.B, f, 'G', -1, 64) -} - -// appendFloatSlice writes a slice formatted floats to buf -func appendFloatSlice(fmt format, f []float64) { - // Check for nil slice - if f == nil { - appendNilType(fmt, `[]float64`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, f := range f { - appendFloat(fmt, f) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(f) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendTime writes a formatted, quoted time string to buf -func appendTime(fmt format, t time.Time) { - appendString(fmt.SetIsKey(true), t.Format(time.RFC1123)) -} - -// appendTimeSlice writes a slice of formatted time strings to buf -func appendTimeSlice(fmt format, t []time.Time) { - // Check for nil slice - if t == nil { - appendNilType(fmt, `[]time.Time`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, t := range t { - appendString(fmt.SetIsKey(true), t.Format(time.RFC1123)) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(t) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendDuration writes a formatted, quoted duration string to buf -func appendDuration(fmt format, d time.Duration) { - appendString(fmt.SetIsKey(true), d.String()) -} - -// appendDurationSlice writes a slice of formatted, quoted duration strings to buf -func appendDurationSlice(fmt format, d []time.Duration) { - // Check for nil slice - if d == nil { - appendNilType(fmt, `[]time.Duration`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, d := range d { - appendString(fmt.SetIsKey(true), d.String()) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(d) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendComplex writes a formatted complex128 to buf -func appendComplex(fmt format, c complex128) { - appendFloat(fmt, real(c)) - fmt.buf.WriteByte('+') - appendFloat(fmt, imag(c)) - fmt.buf.WriteByte('i') -} - -// appendComplexSlice writes a slice of formatted complex128s to buf -func appendComplexSlice(fmt format, c []complex128) { - // Check for nil slice - if c == nil { - appendNilType(fmt, `[]complex128`) - return - } - - fmt.buf.WriteByte('[') - - // Write elements - for _, c := range c { - appendComplex(fmt, c) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if len(c) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// notNil will safely check if 'v' is nil without dealing with weird Go interface nil bullshit. -func notNil(i interface{}) bool { - // cast to get fat pointer - e := *(*struct { - typeOf unsafe.Pointer // ignored - valueOf unsafe.Pointer - })(unsafe.Pointer(&i)) - - // check if value part is nil - return (e.valueOf != nil) -} - -// appendIfaceOrRValueNext performs appendIfaceOrRValue checking + incr depth -func appendIfaceOrRValueNext(fmt format, i interface{}) { - // Check we haven't hit max - if fmt.AtMaxDepth() { - fmt.buf.WriteString("...") - return - } - - // Incr the depth - fmt = fmt.IncrDepth() - - // Make actual call - appendIfaceOrRValue(fmt, i) -} - -// appendIfaceOrReflectValue will attempt to append as interface, falling back to reflection -func appendIfaceOrRValue(fmt format, i interface{}) { - if !appendIface(fmt, i) { - appendRValue(fmt, reflect.ValueOf(i)) - } -} - -// appendValueOrIfaceNext performs appendRValueOrIface checking + incr depth -func appendRValueOrIfaceNext(fmt format, v reflect.Value) { - // Check we haven't hit max - if fmt.AtMaxDepth() { - fmt.buf.WriteString("...") - return - } - - // Incr the depth - fmt = fmt.IncrDepth() - - // Make actual call - appendRValueOrIface(fmt, v) -} - -// appendRValueOrIface will attempt to interface the reflect.Value, falling back to using this directly -func appendRValueOrIface(fmt format, v reflect.Value) { - if !v.CanInterface() || !appendIface(fmt, v.Interface()) { - appendRValue(fmt, v) - } -} - -// appendIface parses and writes a formatted interface value to buf -func appendIface(fmt format, i interface{}) bool { - switch i := i.(type) { - case nil: - fmt.buf.WriteString(`nil`) - case byte: - appendByte(fmt, i) - case []byte: - appendBytes(fmt, i) - case string: - appendString(fmt, i) - case []string: - appendStringSlice(fmt, i) - case int: - appendInt(fmt, int64(i)) - case int8: - appendInt(fmt, int64(i)) - case int16: - appendInt(fmt, int64(i)) - case int32: - appendInt(fmt, int64(i)) - case int64: - appendInt(fmt, i) - case []int: - appendIntSlice(fmt, i) - case uint: - appendUint(fmt, uint64(i)) - case uint16: - appendUint(fmt, uint64(i)) - case uint32: - appendUint(fmt, uint64(i)) - case uint64: - appendUint(fmt, i) - case []uint: - appendUintSlice(fmt, i) - case float32: - appendFloat(fmt, float64(i)) - case float64: - appendFloat(fmt, i) - case []float64: - appendFloatSlice(fmt, i) - case bool: - appendBool(fmt, i) - case []bool: - appendBoolSlice(fmt, i) - case time.Time: - appendTime(fmt, i) - case []time.Time: - appendTimeSlice(fmt, i) - case time.Duration: - appendDuration(fmt, i) - case []time.Duration: - appendDurationSlice(fmt, i) - case complex64: - appendComplex(fmt, complex128(i)) - case complex128: - appendComplex(fmt, i) - case []complex128: - appendComplexSlice(fmt, i) - case map[string]interface{}: - appendIfaceMap(fmt, i) - case error: - if notNil(i) /* use safer nil check */ { - appendString(fmt, i.Error()) - } else { - appendNilIface(fmt, i) - } - case Formattable: - switch { - // catch nil case first - case !notNil(i): - appendNilIface(fmt, i) - - // not permitted - case fmt.Verbose(): - return false - - // use func - default: - fmt.buf.B = i.AppendFormat(fmt.buf.B) - } - case stdfmt.Stringer: - switch { - // catch nil case first - case !notNil(i): - appendNilIface(fmt, i) - - // not permitted - case fmt.Verbose(): - return false - - // use func - default: - appendString(fmt, i.String()) - } - default: - return false // could not handle - } - - return true -} - -// appendReflectValue will safely append a reflected value -func appendRValue(fmt format, v reflect.Value) { - switch v.Kind() { - case reflect.Float32, reflect.Float64: - appendFloat(fmt, v.Float()) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - appendInt(fmt, v.Int()) - case reflect.Uint8: - appendByte(fmt, uint8(v.Uint())) - case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64: - appendUint(fmt, v.Uint()) - case reflect.Bool: - appendBool(fmt, v.Bool()) - case reflect.Array: - appendArrayType(fmt, v) - case reflect.Slice: - appendSliceType(fmt, v) - case reflect.Map: - appendMapType(fmt, v) - case reflect.Struct: - appendStructType(fmt, v) - case reflect.Ptr: - if v.IsNil() { - appendNilRValue(fmt, v) - } else { - appendRValue(fmt.IncrDerefs(), v.Elem()) - } - case reflect.UnsafePointer: - fmt.buf.WriteString("(unsafe.Pointer)") - fmt.buf.WriteByte('(') - if u := v.Pointer(); u != 0 { - fmt.buf.WriteString("0x") - fmt.buf.B = strconv.AppendUint(fmt.buf.B, uint64(u), 16) - } else { - fmt.buf.WriteString(`nil`) - } - fmt.buf.WriteByte(')') - case reflect.Uintptr: - fmt.buf.WriteString("(uintptr)") - fmt.buf.WriteByte('(') - if u := v.Uint(); u != 0 { - fmt.buf.WriteString("0x") - fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 16) - } else { - fmt.buf.WriteString(`nil`) - } - fmt.buf.WriteByte(')') - case reflect.String: - appendString(fmt, v.String()) - case reflect.Complex64, reflect.Complex128: - appendComplex(fmt, v.Complex()) - case reflect.Func, reflect.Chan, reflect.Interface: - if v.IsNil() { - appendNilRValue(fmt, v) - } else { - fmt.buf.WriteString(v.String()) - } - default: - fmt.buf.WriteString(v.String()) - } -} - -// appendIfaceMap writes a map of key-value pairs (as a set of fields) to buf -func appendIfaceMap(fmt format, v map[string]interface{}) { - // Catch nil map - if v == nil { - appendNilType(fmt, `map[string]interface{}`) - return - } - - fmt.buf.WriteByte('{') - - // Write map pairs! - for key, value := range v { - appendString(fmt.SetIsKey(true), key) - fmt.buf.WriteByte('=') - appendIfaceOrRValueNext(fmt.SetIsKey(false), value) - fmt.buf.WriteByte(' ') - } - - // Drop last space - if len(v) > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte('}') -} - -// appendArrayType writes an array of unknown type (parsed by reflection) to buf, unlike appendSliceType does NOT catch nil slice -func appendArrayType(fmt format, v reflect.Value) { - // get no. elements - n := v.Len() - - fmt.buf.WriteByte('[') - - // Write values - for i := 0; i < n; i++ { - appendRValueOrIfaceNext(fmt.SetIsKey(false), v.Index(i)) - fmt.buf.WriteByte(',') - } - - // Drop last comma - if n > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte(']') -} - -// appendSliceType writes a slice of unknown type (parsed by reflection) to buf -func appendSliceType(fmt format, v reflect.Value) { - if v.IsNil() { - appendNilRValue(fmt, v) - } else { - appendArrayType(fmt, v) - } -} - -// appendMapType writes a map of unknown types (parsed by reflection) to buf -func appendMapType(fmt format, v reflect.Value) { - // Catch nil map - if v.IsNil() { - appendNilRValue(fmt, v) - return - } - - // Get a map iterator - r := v.MapRange() - n := v.Len() - - fmt.buf.WriteByte('{') - - // Iterate pairs - for r.Next() { - appendRValueOrIfaceNext(fmt.SetIsKey(true), r.Key()) - fmt.buf.WriteByte('=') - appendRValueOrIfaceNext(fmt.SetIsKey(false), r.Value()) - fmt.buf.WriteByte(' ') - } - - // Drop last space - if n > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte('}') -} - -// appendStructType writes a struct (as a set of key-value fields) to buf -func appendStructType(fmt format, v reflect.Value) { - // Get value type & no. fields - t := v.Type() - n := v.NumField() - w := 0 - - // If verbose, append the type - - fmt.buf.WriteByte('{') - - // Iterate fields - for i := 0; i < n; i++ { - vfield := v.Field(i) - name := t.Field(i).Name - - // Append field name - appendString(fmt.SetIsKey(true), name) - fmt.buf.WriteByte('=') - - if !vfield.CanInterface() { - // This is an unexported field - appendRValue(fmt.SetIsKey(false), vfield) - } else { - // This is an exported field! - appendRValueOrIfaceNext(fmt.SetIsKey(false), vfield) - } - - // Iter written count - fmt.buf.WriteByte(' ') - w++ - } - - // Drop last space - if w > 0 { - fmt.buf.Truncate(1) - } - - fmt.buf.WriteByte('}') -} - -// containsSpaceOrTab checks if "s" contains space or tabs -func containsSpaceOrTab(s string) bool { - for _, r := range s { - if r == ' ' || r == '\t' { - return true - } - } - return false -} diff --git a/vendor/codeberg.org/gruf/go-logger/hook.go b/vendor/codeberg.org/gruf/go-logger/hook.go deleted file mode 100644 index 2345ca93b..000000000 --- a/vendor/codeberg.org/gruf/go-logger/hook.go +++ /dev/null @@ -1,13 +0,0 @@ -package logger - -// Hook defines a log Entry modifier -type Hook interface { - Do(*Entry) -} - -// HookFunc is a simple adapter to allow functions to satisfy the Hook interface -type HookFunc func(*Entry) - -func (hook HookFunc) Do(entry *Entry) { - hook(entry) -} diff --git a/vendor/codeberg.org/gruf/go-logger/level.go b/vendor/codeberg.org/gruf/go-logger/level.go deleted file mode 100644 index 0a076c246..000000000 --- a/vendor/codeberg.org/gruf/go-logger/level.go +++ /dev/null @@ -1,38 +0,0 @@ -package logger - -// LEVEL defines a level of logging -type LEVEL uint8 - -// Available levels of logging. -const ( - unset LEVEL = ^LEVEL(0) - DEBUG LEVEL = 5 - INFO LEVEL = 10 - WARN LEVEL = 15 - ERROR LEVEL = 20 - FATAL LEVEL = 25 -) - -var unknownLevel = "unknown" - -// Levels defines a mapping of log LEVELs to formatted level strings -type Levels [^LEVEL(0)]string - -// DefaultLevels returns the default set of log levels -func DefaultLevels() Levels { - return Levels{ - DEBUG: "DEBUG", - INFO: "INFO", - WARN: "WARN", - ERROR: "ERROR", - FATAL: "FATAL", - } -} - -// Get fetches the level string for the provided value, or "unknown" -func (l Levels) Get(lvl LEVEL) string { - if str := l[int(lvl)]; str != "" { - return str - } - return unknownLevel -} diff --git a/vendor/codeberg.org/gruf/go-logger/logger.go b/vendor/codeberg.org/gruf/go-logger/logger.go deleted file mode 100644 index 94d3ab8ca..000000000 --- a/vendor/codeberg.org/gruf/go-logger/logger.go +++ /dev/null @@ -1,187 +0,0 @@ -package logger - -import ( - "context" - "fmt" - "io" - "os" - "sync" - "sync/atomic" - - "codeberg.org/gruf/go-bytes" -) - -type Logger struct { - // Hooks defines a list of hooks which are called before an entry - // is written. This should NOT be modified while the Logger is in use - Hooks []Hook - - // Level is the current log LEVEL, entries at level below the - // currently set level will not be output. This should NOT - // be modified while the Logger is in use - Level LEVEL - - // Timestamp defines whether to automatically append timestamps - // to entries written via Logger convience methods and specifically - // Entry.TimestampIf(). This should NOT be modified while Logger in use - Timestamp bool - - // Format is the log entry LogFormat to use. This should NOT - // be modified while the Logger is in use - Format LogFormat - - // BufferSize is the Entry buffer size to use when allocating - // new Entry objects. This should be modified atomically - BufSize int64 - - // Output is the log's output writer. This should NOT be - // modified while the Logger is in use - Output io.Writer - - // entry pool - pool sync.Pool -} - -// New returns a new Logger instance with defaults -func New(out io.Writer) *Logger { - return NewWith(0 /* all */, true, DefaultTextFormat, 512, out) -} - -// NewWith returns a new Logger instance with supplied configuration -func NewWith(lvl LEVEL, timestamp bool, fmt LogFormat, bufsize int64, out io.Writer) *Logger { - // Create new logger object - log := &Logger{ - Level: lvl, - Timestamp: timestamp, - Format: fmt, - BufSize: bufsize, - Output: out, - } - - // Ensure clock running - startClock() - - // Set-up logger Entry pool - log.pool.New = func() interface{} { - return &Entry{ - lvl: unset, - buf: &bytes.Buffer{B: make([]byte, 0, atomic.LoadInt64(&log.BufSize))}, - log: log, - } - } - - return log -} - -// Entry returns a new Entry from the Logger's pool with background context -func (l *Logger) Entry() *Entry { - entry, _ := l.pool.Get().(*Entry) - entry.ctx = context.Background() - return entry -} - -// Debug prints the provided arguments with the debug prefix -func (l *Logger) Debug(a ...interface{}) { - l.Log(DEBUG, a...) -} - -// Debugf prints the provided format string and arguments with the debug prefix -func (l *Logger) Debugf(s string, a ...interface{}) { - l.Logf(DEBUG, s, a...) -} - -// Info prints the provided arguments with the info prefix -func (l *Logger) Info(a ...interface{}) { - l.Log(INFO, a...) -} - -// Infof prints the provided format string and arguments with the info prefix -func (l *Logger) Infof(s string, a ...interface{}) { - l.Logf(INFO, s, a...) -} - -// Warn prints the provided arguments with the warn prefix -func (l *Logger) Warn(a ...interface{}) { - l.Log(WARN, a...) -} - -// Warnf prints the provided format string and arguments with the warn prefix -func (l *Logger) Warnf(s string, a ...interface{}) { - l.Logf(WARN, s, a...) -} - -// Error prints the provided arguments with the error prefix -func (l *Logger) Error(a ...interface{}) { - l.Log(ERROR, a...) -} - -// Errorf prints the provided format string and arguments with the error prefix -func (l *Logger) Errorf(s string, a ...interface{}) { - l.Logf(ERROR, s, a...) -} - -// Fatal prints provided arguments with the fatal prefix before exiting the program -// with os.Exit(1) -func (l *Logger) Fatal(a ...interface{}) { - defer os.Exit(1) - l.Log(FATAL, a...) -} - -// Fatalf prints provided the provided format string and arguments with the fatal prefix -// before exiting the program with os.Exit(1) -func (l *Logger) Fatalf(s string, a ...interface{}) { - defer os.Exit(1) - l.Logf(FATAL, s, a...) -} - -// Log prints the provided arguments at the supplied log level -func (l *Logger) Log(lvl LEVEL, a ...interface{}) { - if lvl >= l.Level { - l.Entry().TimestampIf().Level(lvl).Hooks().Msg(a...) - } -} - -// Logf prints the provided format string and arguments at the supplied log level -func (l *Logger) Logf(lvl LEVEL, s string, a ...interface{}) { - if lvl >= l.Level { - l.Entry().TimestampIf().Level(lvl).Hooks().Msgf(s, a...) - } -} - -// LogFields prints the provided fields formatted as key-value pairs at the supplied log level -func (l *Logger) LogFields(lvl LEVEL, fields map[string]interface{}) { - if lvl >= l.Level { - l.Entry().TimestampIf().Level(lvl).Fields(fields).Hooks().Send() - } -} - -// LogValues prints the provided values formatted as-so at the supplied log level -func (l *Logger) LogValues(lvl LEVEL, a ...interface{}) { - if lvl >= l.Level { - l.Entry().TimestampIf().Level(lvl).Values(a...).Hooks().Send() - } -} - -// Print simply prints provided arguments -func (l *Logger) Print(a ...interface{}) { - e := l.Entry().TimestampIf() - fmt.Fprint(e.buf, a...) - e.Send() -} - -// Printf simply prints provided the provided format string and arguments -func (l *Logger) Printf(s string, a ...interface{}) { - e := l.Entry().TimestampIf() - fmt.Fprintf(e.buf, s, a...) - e.Send() -} - -// PrintFields prints the provided fields formatted as key-value pairs -func (l *Logger) PrintFields(fields map[string]interface{}) { - l.Entry().TimestampIf().Fields(fields).Send() -} - -// PrintValues prints the provided values formatted as-so -func (l *Logger) PrintValues(a ...interface{}) { - l.Entry().TimestampIf().Values(a...).Send() -} diff --git a/vendor/codeberg.org/gruf/go-logger/writer.go b/vendor/codeberg.org/gruf/go-logger/writer.go deleted file mode 100644 index 72321f518..000000000 --- a/vendor/codeberg.org/gruf/go-logger/writer.go +++ /dev/null @@ -1,29 +0,0 @@ -package logger - -import ( - "io" - "io/ioutil" - "sync" -) - -// AddSafety wraps an io.Writer to provide mutex locking protection -func AddSafety(w io.Writer) io.Writer { - if w == nil { - w = ioutil.Discard - } else if sw, ok := w.(*safeWriter); ok { - return sw - } - return &safeWriter{wr: w} -} - -// safeWriter wraps an io.Writer to provide mutex locking on write -type safeWriter struct { - wr io.Writer - mu sync.Mutex -} - -func (w *safeWriter) Write(b []byte) (int, error) { - w.mu.Lock() - defer w.mu.Unlock() - return w.wr.Write(b) -} diff --git a/vendor/codeberg.org/gruf/go-store/kv/iterator.go b/vendor/codeberg.org/gruf/go-store/kv/iterator.go index d3999273f..ddaaf60cf 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/iterator.go +++ b/vendor/codeberg.org/gruf/go-store/kv/iterator.go @@ -60,5 +60,5 @@ func (i *KVIterator) Value() ([]byte, error) { } // Attempt to fetch from store - return i.store.get(i.key) + return i.store.get(i.store.mutexMap.RLock, i.key) } diff --git a/vendor/codeberg.org/gruf/go-store/kv/state.go b/vendor/codeberg.org/gruf/go-store/kv/state.go index 20a3e951d..330928bce 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/state.go +++ b/vendor/codeberg.org/gruf/go-store/kv/state.go @@ -30,7 +30,7 @@ func (st *StateRO) Get(key string) ([]byte, error) { } // Pass request to store - return st.store.get(key) + return st.store.get(st.store.mutexMap.RLock, key) } func (st *StateRO) GetStream(key string) (io.ReadCloser, error) { @@ -44,7 +44,7 @@ func (st *StateRO) GetStream(key string) (io.ReadCloser, error) { } // Pass request to store - return st.store.getStream(key) + return st.store.getStream(st.store.mutexMap.RLock, key) } func (st *StateRO) Has(key string) (bool, error) { @@ -58,7 +58,7 @@ func (st *StateRO) Has(key string) (bool, error) { } // Pass request to store - return st.store.has(key) + return st.store.has(st.store.mutexMap.RLock, key) } func (st *StateRO) Release() { @@ -94,7 +94,7 @@ func (st *StateRW) Get(key string) ([]byte, error) { } // Pass request to store - return st.store.get(key) + return st.store.get(st.store.mutexMap.RLock, key) } func (st *StateRW) GetStream(key string) (io.ReadCloser, error) { @@ -108,7 +108,7 @@ func (st *StateRW) GetStream(key string) (io.ReadCloser, error) { } // Pass request to store - return st.store.getStream(key) + return st.store.getStream(st.store.mutexMap.RLock, key) } func (st *StateRW) Put(key string, value []byte) error { @@ -122,7 +122,7 @@ func (st *StateRW) Put(key string, value []byte) error { } // Pass request to store - return st.store.put(key, value) + return st.store.put(st.store.mutexMap.Lock, key, value) } func (st *StateRW) PutStream(key string, r io.Reader) error { @@ -136,7 +136,7 @@ func (st *StateRW) PutStream(key string, r io.Reader) error { } // Pass request to store - return st.store.putStream(key, r) + return st.store.putStream(st.store.mutexMap.Lock, key, r) } func (st *StateRW) Has(key string) (bool, error) { @@ -150,7 +150,7 @@ func (st *StateRW) Has(key string) (bool, error) { } // Pass request to store - return st.store.has(key) + return st.store.has(st.store.mutexMap.RLock, key) } func (st *StateRW) Delete(key string) error { @@ -164,7 +164,7 @@ func (st *StateRW) Delete(key string) error { } // Pass request to store - return st.store.delete(key) + return st.store.delete(st.store.mutexMap.Lock, key) } func (st *StateRW) Release() { diff --git a/vendor/codeberg.org/gruf/go-store/kv/store.go b/vendor/codeberg.org/gruf/go-store/kv/store.go index 34fe91987..4c3a31140 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/store.go +++ b/vendor/codeberg.org/gruf/go-store/kv/store.go @@ -53,19 +53,30 @@ func OpenStorage(storage storage.Storage) (*KVStore, error) { }, nil } -// Get fetches the bytes for supplied key in the store -func (st *KVStore) Get(key string) ([]byte, error) { - // Acquire store read lock +// RLock acquires a read-lock on supplied key, returning unlock function. +func (st *KVStore) RLock(key string) (runlock func()) { st.mutex.RLock() - defer st.mutex.RUnlock() + runlock = st.mutexMap.RLock(key) + st.mutex.RUnlock() + return runlock +} - // Pass to unprotected fn - return st.get(key) +// Lock acquires a write-lock on supplied key, returning unlock function. +func (st *KVStore) Lock(key string) (unlock func()) { + st.mutex.Lock() + unlock = st.mutexMap.Lock(key) + st.mutex.Unlock() + return unlock } -func (st *KVStore) get(key string) ([]byte, error) { +// Get fetches the bytes for supplied key in the store +func (st *KVStore) Get(key string) ([]byte, error) { + return st.get(st.RLock, key) +} + +func (st *KVStore) get(rlock func(string) func(), key string) ([]byte, error) { // Acquire read lock for key - runlock := st.mutexMap.RLock(key) + runlock := rlock(key) defer runlock() // Read file bytes @@ -74,17 +85,12 @@ func (st *KVStore) get(key string) ([]byte, error) { // GetStream fetches a ReadCloser for the bytes at the supplied key location in the store func (st *KVStore) GetStream(key string) (io.ReadCloser, error) { - // Acquire store read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - - // Pass to unprotected fn - return st.getStream(key) + return st.getStream(st.RLock, key) } -func (st *KVStore) getStream(key string) (io.ReadCloser, error) { +func (st *KVStore) getStream(rlock func(string) func(), key string) (io.ReadCloser, error) { // Acquire read lock for key - runlock := st.mutexMap.RLock(key) + runlock := rlock(key) // Attempt to open stream for read rd, err := st.storage.ReadStream(key) @@ -99,17 +105,12 @@ func (st *KVStore) getStream(key string) (io.ReadCloser, error) { // Put places the bytes at the supplied key location in the store func (st *KVStore) Put(key string, value []byte) error { - // Acquire store write lock - st.mutex.Lock() - defer st.mutex.Unlock() - - // Pass to unprotected fn - return st.put(key, value) + return st.put(st.Lock, key, value) } -func (st *KVStore) put(key string, value []byte) error { +func (st *KVStore) put(lock func(string) func(), key string, value []byte) error { // Acquire write lock for key - unlock := st.mutexMap.Lock(key) + unlock := lock(key) defer unlock() // Write file bytes @@ -118,17 +119,12 @@ func (st *KVStore) put(key string, value []byte) error { // PutStream writes the bytes from the supplied Reader at the supplied key location in the store func (st *KVStore) PutStream(key string, r io.Reader) error { - // Acquire store write lock - st.mutex.Lock() - defer st.mutex.Unlock() - - // Pass to unprotected fn - return st.putStream(key, r) + return st.putStream(st.Lock, key, r) } -func (st *KVStore) putStream(key string, r io.Reader) error { +func (st *KVStore) putStream(lock func(string) func(), key string, r io.Reader) error { // Acquire write lock for key - unlock := st.mutexMap.Lock(key) + unlock := lock(key) defer unlock() // Write file stream @@ -137,17 +133,12 @@ func (st *KVStore) putStream(key string, r io.Reader) error { // Has checks whether the supplied key exists in the store func (st *KVStore) Has(key string) (bool, error) { - // Acquire store read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - - // Pass to unprotected fn - return st.has(key) + return st.has(st.RLock, key) } -func (st *KVStore) has(key string) (bool, error) { +func (st *KVStore) has(rlock func(string) func(), key string) (bool, error) { // Acquire read lock for key - runlock := st.mutexMap.RLock(key) + runlock := rlock(key) defer runlock() // Stat file on disk @@ -156,17 +147,12 @@ func (st *KVStore) has(key string) (bool, error) { // Delete removes the supplied key-value pair from the store func (st *KVStore) Delete(key string) error { - // Acquire store write lock - st.mutex.Lock() - defer st.mutex.Unlock() - - // Pass to unprotected fn - return st.delete(key) + return st.delete(st.Lock, key) } -func (st *KVStore) delete(key string) error { +func (st *KVStore) delete(lock func(string) func(), key string) error { // Acquire write lock for key - unlock := st.mutexMap.Lock(key) + unlock := lock(key) defer unlock() // Remove file from disk diff --git a/vendor/codeberg.org/gruf/go-store/storage/block.go b/vendor/codeberg.org/gruf/go-store/storage/block.go index 9a8c4dc7d..bc35b07ac 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/block.go +++ b/vendor/codeberg.org/gruf/go-store/storage/block.go @@ -1,7 +1,6 @@ package storage import ( - "crypto/sha256" "io" "io/fs" "os" @@ -14,6 +13,7 @@ import ( "codeberg.org/gruf/go-hashenc" "codeberg.org/gruf/go-pools" "codeberg.org/gruf/go-store/util" + "github.com/zeebo/blake3" ) var ( @@ -77,7 +77,7 @@ func getBlockConfig(cfg *BlockConfig) BlockConfig { // BlockStorage is a Storage implementation that stores input data as chunks on // a filesystem. Each value is chunked into blocks of configured size and these -// blocks are stored with name equal to their base64-encoded SHA256 hash-sum. A +// blocks are stored with name equal to their base64-encoded BLAKE3 hash-sum. A // "node" file is finally created containing an array of hashes contained within // this value type BlockStorage struct { @@ -87,6 +87,7 @@ type BlockStorage struct { config BlockConfig // cfg is the supplied configuration for this store hashPool sync.Pool // hashPool is this store's hashEncoder pool bufpool pools.BufferPool // bufpool is this store's bytes.Buffer pool + lock *LockableFile // lock is the opened lockfile for this storage instance // NOTE: // BlockStorage does not need to lock each of the underlying block files @@ -138,6 +139,14 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) { return nil, errPathIsFile } + // Open and acquire storage lock for path + lock, err := OpenLock(pb.Join(path, LockFile)) + if err != nil { + return nil, err + } else if err := lock.Lock(); err != nil { + return nil, err + } + // Figure out the largest size for bufpool slices bufSz := encodedHashLen if bufSz < config.BlockSize { @@ -159,6 +168,7 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) { }, }, bufpool: pools.NewBufferPool(bufSz), + lock: lock, }, nil } @@ -443,11 +453,16 @@ loop: continue loop } - // Write in separate goroutine + // Check if reached EOF + atEOF := (n < buf.Len()) + wg.Add(1) go func() { - // Defer buffer release + signal done + // Perform writes in goroutine + defer func() { + // Defer release + + // signal we're done st.bufpool.Put(buf) wg.Done() }() @@ -460,8 +475,8 @@ loop: } }() - // We reached EOF - if n < buf.Len() { + // Break at end + if atEOF { break loop } } @@ -568,6 +583,12 @@ func (st *BlockStorage) Remove(key string) error { return os.Remove(kpath) } +// Close implements Storage.Close() +func (st *BlockStorage) Close() error { + defer st.lock.Close() + return st.lock.Unlock() +} + // WalkKeys implements Storage.WalkKeys() func (st *BlockStorage) WalkKeys(opts WalkKeysOptions) error { // Acquire path builder @@ -610,7 +631,7 @@ func (st *BlockStorage) blockPathForKey(hash string) string { } // hashSeparator is the separating byte between block hashes -const hashSeparator = byte(':') +const hashSeparator = byte('\n') // node represents the contents of a node file in storage type node struct { @@ -773,24 +794,28 @@ func (r *blockReader) Read(b []byte) (int, error) { } } +var ( + // base64Encoding is our base64 encoding object. + base64Encoding = hashenc.Base64() + + // encodedHashLen is the once-calculated encoded hash-sum length + encodedHashLen = base64Encoding.EncodedLen( + blake3.New().Size(), + ) +) + // hashEncoder is a HashEncoder with built-in encode buffer type hashEncoder struct { henc hashenc.HashEncoder ebuf []byte } -// encodedHashLen is the once-calculated encoded hash-sum length -var encodedHashLen = hashenc.Base64().EncodedLen( - sha256.New().Size(), -) - // newHashEncoder returns a new hashEncoder instance func newHashEncoder() *hashEncoder { - hash := sha256.New() - enc := hashenc.Base64() + hash := blake3.New() return &hashEncoder{ - henc: hashenc.New(hash, enc), - ebuf: make([]byte, enc.EncodedLen(hash.Size())), + henc: hashenc.New(hash, base64Encoding), + ebuf: make([]byte, encodedHashLen), } } diff --git a/vendor/codeberg.org/gruf/go-store/storage/disk.go b/vendor/codeberg.org/gruf/go-store/storage/disk.go index 060d56688..9b5430437 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/disk.go +++ b/vendor/codeberg.org/gruf/go-store/storage/disk.go @@ -71,6 +71,7 @@ type DiskStorage struct { path string // path is the root path of this store bufp pools.BufferPool // bufp is the buffer pool for this DiskStorage config DiskConfig // cfg is the supplied configuration for this store + lock *LockableFile // lock is the opened lockfile for this storage instance } // OpenFile opens a DiskStorage instance for given folder path and configuration @@ -81,13 +82,13 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { // Clean provided path, ensure ends in '/' (should // be dir, this helps with file path trimming later) - path = pb.Clean(path) + "/" + storePath := pb.Join(path, "store") + "/" // Get checked config config := getDiskConfig(cfg) // Attempt to open dir path - file, err := os.OpenFile(path, defaultFileROFlags, defaultDirPerms) + file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms) if err != nil { // If not a not-exist error, return if !os.IsNotExist(err) { @@ -95,13 +96,13 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { } // Attempt to make store path dirs - err = os.MkdirAll(path, defaultDirPerms) + err = os.MkdirAll(storePath, defaultDirPerms) if err != nil { return nil, err } // Reopen dir now it's been created - file, err = os.OpenFile(path, defaultFileROFlags, defaultDirPerms) + file, err = os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms) if err != nil { return nil, err } @@ -116,11 +117,20 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { return nil, errPathIsFile } + // Open and acquire storage lock for path + lock, err := OpenLock(pb.Join(path, LockFile)) + if err != nil { + return nil, err + } else if err := lock.Lock(); err != nil { + return nil, err + } + // Return new DiskStorage return &DiskStorage{ - path: path, + path: storePath, bufp: pools.NewBufferPool(config.WriteBufSize), config: config, + lock: lock, }, nil } @@ -248,6 +258,12 @@ func (st *DiskStorage) Remove(key string) error { return os.Remove(kpath) } +// Close implements Storage.Close() +func (st *DiskStorage) Close() error { + defer st.lock.Close() + return st.lock.Unlock() +} + // WalkKeys implements Storage.WalkKeys() func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error { // Acquire path builder @@ -256,8 +272,9 @@ func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error { // Walk dir for entries return util.WalkDir(pb, st.path, func(kpath string, fsentry fs.DirEntry) { - // Only deal with regular files if fsentry.Type().IsRegular() { + // Only deal with regular files + // Get full item path (without root) kpath = pb.Join(kpath, fsentry.Name())[len(st.path):] diff --git a/vendor/codeberg.org/gruf/go-store/storage/fs.go b/vendor/codeberg.org/gruf/go-store/storage/fs.go index 444cee4b0..ff4c857c3 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/fs.go +++ b/vendor/codeberg.org/gruf/go-store/storage/fs.go @@ -39,7 +39,7 @@ func stat(path string) (bool, error) { return syscall.Stat(path, &stat) }) if err != nil { - if err == syscall.ENOENT { + if err == syscall.ENOENT { //nolint err = nil } return false, err diff --git a/vendor/codeberg.org/gruf/go-store/storage/lock.go b/vendor/codeberg.org/gruf/go-store/storage/lock.go index 3d794cda9..a757830cc 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/lock.go +++ b/vendor/codeberg.org/gruf/go-store/storage/lock.go @@ -7,27 +7,31 @@ import ( "codeberg.org/gruf/go-store/util" ) -type lockableFile struct { +// LockFile is our standard lockfile name. +const LockFile = "store.lock" + +type LockableFile struct { *os.File } -func openLock(path string) (*lockableFile, error) { +// OpenLock opens a lockfile at path. +func OpenLock(path string) (*LockableFile, error) { file, err := open(path, defaultFileLockFlags) if err != nil { return nil, err } - return &lockableFile{file}, nil + return &LockableFile{file}, nil } -func (f *lockableFile) lock() error { +func (f *LockableFile) Lock() error { return f.flock(syscall.LOCK_EX | syscall.LOCK_NB) } -func (f *lockableFile) unlock() error { +func (f *LockableFile) Unlock() error { return f.flock(syscall.LOCK_UN | syscall.LOCK_NB) } -func (f *lockableFile) flock(how int) error { +func (f *LockableFile) flock(how int) error { return util.RetryOnEINTR(func() error { return syscall.Flock(int(f.Fd()), how) }) diff --git a/vendor/codeberg.org/gruf/go-store/storage/memory.go b/vendor/codeberg.org/gruf/go-store/storage/memory.go index be60fa464..7daa4a483 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/memory.go +++ b/vendor/codeberg.org/gruf/go-store/storage/memory.go @@ -2,6 +2,7 @@ package storage import ( "io" + "sync" "codeberg.org/gruf/go-bytes" "codeberg.org/gruf/go-store/util" @@ -10,13 +11,17 @@ import ( // MemoryStorage is a storage implementation that simply stores key-value // pairs in a Go map in-memory. The map is protected by a mutex. type MemoryStorage struct { + ow bool // overwrites fs map[string][]byte + mu sync.Mutex } // OpenMemory opens a new MemoryStorage instance with internal map of 'size'. -func OpenMemory(size int) *MemoryStorage { +func OpenMemory(size int, overwrites bool) *MemoryStorage { return &MemoryStorage{ fs: make(map[string][]byte, size), + mu: sync.Mutex{}, + ow: overwrites, } } @@ -27,19 +32,33 @@ func (st *MemoryStorage) Clean() error { // ReadBytes implements Storage.ReadBytes(). func (st *MemoryStorage) ReadBytes(key string) ([]byte, error) { + // Safely check store + st.mu.Lock() b, ok := st.fs[key] + st.mu.Unlock() + + // Return early if not exist if !ok { return nil, ErrNotFound } + + // Create return copy return bytes.Copy(b), nil } // ReadStream implements Storage.ReadStream(). func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error) { + // Safely check store + st.mu.Lock() b, ok := st.fs[key] + st.mu.Unlock() + + // Return early if not exist if !ok { return nil, ErrNotFound } + + // Create io.ReadCloser from 'b' copy b = bytes.Copy(b) r := bytes.NewReader(b) return util.NopReadCloser(r), nil @@ -47,43 +66,73 @@ func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error) { // WriteBytes implements Storage.WriteBytes(). func (st *MemoryStorage) WriteBytes(key string, b []byte) error { + // Safely check store + st.mu.Lock() _, ok := st.fs[key] - if ok { + + // Check for already exist + if ok && !st.ow { + st.mu.Unlock() return ErrAlreadyExists } + + // Write + unlock st.fs[key] = bytes.Copy(b) + st.mu.Unlock() return nil } // WriteStream implements Storage.WriteStream(). func (st *MemoryStorage) WriteStream(key string, r io.Reader) error { + // Read all from reader b, err := io.ReadAll(r) if err != nil { return err } + + // Write to storage return st.WriteBytes(key, b) } // Stat implements Storage.Stat(). func (st *MemoryStorage) Stat(key string) (bool, error) { + st.mu.Lock() _, ok := st.fs[key] + st.mu.Unlock() return ok, nil } // Remove implements Storage.Remove(). func (st *MemoryStorage) Remove(key string) error { + // Safely check store + st.mu.Lock() _, ok := st.fs[key] + + // Check in store if !ok { + st.mu.Unlock() return ErrNotFound } + + // Delete + unlock delete(st.fs, key) + st.mu.Unlock() + return nil +} + +// Close implements Storage.Close(). +func (st *MemoryStorage) Close() error { return nil } // WalkKeys implements Storage.WalkKeys(). func (st *MemoryStorage) WalkKeys(opts WalkKeysOptions) error { + // Safely walk storage keys + st.mu.Lock() for key := range st.fs { opts.WalkFn(entry(key)) } + st.mu.Unlock() + return nil } diff --git a/vendor/codeberg.org/gruf/go-store/storage/storage.go b/vendor/codeberg.org/gruf/go-store/storage/storage.go index b160267a4..346aff097 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/storage.go +++ b/vendor/codeberg.org/gruf/go-store/storage/storage.go @@ -19,9 +19,6 @@ func (e entry) Key() string { // Storage defines a means of storing and accessing key value pairs type Storage interface { - // Clean removes unused values and unclutters the storage (e.g. removing empty folders) - Clean() error - // ReadBytes returns the byte value for key in storage ReadBytes(key string) ([]byte, error) @@ -40,6 +37,12 @@ type Storage interface { // Remove attempts to remove the supplied key-value pair from storage Remove(key string) error + // Close will close the storage, releasing any file locks + Close() error + + // Clean removes unused values and unclutters the storage (e.g. removing empty folders) + Clean() error + // WalkKeys walks the keys in the storage WalkKeys(opts WalkKeysOptions) error } diff --git a/vendor/github.com/zeebo/blake3/.gitignore b/vendor/github.com/zeebo/blake3/.gitignore new file mode 100644 index 000000000..c6bfdf2c3 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/.gitignore @@ -0,0 +1,6 @@ +*.pprof +*.test +*.txt +*.out + +/upstream diff --git a/vendor/github.com/zeebo/blake3/LICENSE b/vendor/github.com/zeebo/blake3/LICENSE new file mode 100644 index 000000000..3a63575d3 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/LICENSE @@ -0,0 +1,125 @@ +This work is released into the public domain with CC0 1.0. + +------------------------------------------------------------------------------- + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/vendor/github.com/zeebo/blake3/Makefile b/vendor/github.com/zeebo/blake3/Makefile new file mode 100644 index 000000000..f98f0f093 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/Makefile @@ -0,0 +1,11 @@ +asm: internal/alg/hash/hash_avx2/impl_amd64.s internal/alg/compress/compress_sse41/impl_amd64.s + +internal/alg/hash/hash_avx2/impl_amd64.s: avo/avx2/*.go + ( cd avo; go run ./avx2 ) > internal/alg/hash/hash_avx2/impl_amd64.s + +internal/alg/compress/compress_sse41/impl_amd64.s: avo/sse41/*.go + ( cd avo; go run ./sse41 ) > internal/alg/compress/compress_sse41/impl_amd64.s + +.PHONY: test +test: + go test -race -bench=. -benchtime=1x diff --git a/vendor/github.com/zeebo/blake3/README.md b/vendor/github.com/zeebo/blake3/README.md new file mode 100644 index 000000000..0a0f2e186 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/README.md @@ -0,0 +1,77 @@ +# BLAKE3 + +

+ go.dev + Go Report Card + SourceGraph +

+ +Pure Go implementation of [BLAKE3](https://blake3.io) with AVX2 and SSE4.1 acceleration. + +Special thanks to the excellent [avo](https://github.com/mmcloughlin/avo) making writing vectorized version much easier. + +# Benchmarks + +## Caveats + +This library makes some different design decisions than the upstream Rust crate around internal buffering. Specifically, because it does not target the embedded system space, nor does it support multithreading, it elects to do its own internal buffering. This means that a user does not have to worry about providing large enough buffers to get the best possible performance, but it does worse on smaller input sizes. So some notes: + +- The Rust benchmarks below are all single-threaded to match this Go implementation. +- I make no attempt to get precise measurements (cpu throttling, noisy environment, etc.) so please benchmark on your own systems. +- These benchmarks are run on an i7-6700K which does not support AVX-512, so Rust is limited to use AVX2 at sizes above 8 kib. +- I tried my best to make them benchmark the same thing, but who knows? :smile: + +## Charts + +In this case, both libraries are able to avoid a lot of data copying and will use vectorized instructions to hash as fast as possible, and perform similarly. + +![Large Full Buffer](/assets/large-full-buffer.svg) + +For incremental writes, you must provide the Rust version large enough buffers so that it can use vectorized instructions. This Go library performs consistently regardless of the size being sent into the update function. + +![Incremental](/assets/incremental.svg) + +The downside of internal buffering is most apparent with small sizes as most time is spent initializing the hasher state. In terms of hashing rate, the difference is 3-4x, but in an absolute sense it's ~100ns (see tables below). If you wish to hash a large number of very small strings and you care about those nanoseconds, be sure to use the Reset method to avoid re-initializing the state. + +![Small Full Buffer](/assets/small-full-buffer.svg) + +## Timing Tables + +### Small + +| Size | Full Buffer | Reset | | Full Buffer Rate | Reset Rate | +|--------|-------------|------------|-|------------------|--------------| +| 64 b | `205ns` | `86.5ns` | | `312MB/s` | `740MB/s` | +| 256 b | `364ns` | `250ns` | | `703MB/s` | `1.03GB/s` | +| 512 b | `575ns` | `468ns` | | `892MB/s` | `1.10GB/s` | +| 768 b | `795ns` | `682ns` | | `967MB/s` | `1.13GB/s` | + +### Large + +| Size | Incremental | Full Buffer | Reset | | Incremental Rate | Full Buffer Rate | Reset Rate | +|----------|-------------|-------------|------------|-|------------------|------------------|--------------| +| 1 kib | `1.02µs` | `1.01µs` | `891ns` | | `1.00GB/s` | `1.01GB/s` | `1.15GB/s` | +| 2 kib | `2.11µs` | `2.07µs` | `1.95µs` | | `968MB/s` | `990MB/s` | `1.05GB/s` | +| 4 kib | `2.28µs` | `2.15µs` | `2.05µs` | | `1.80GB/s` | `1.90GB/s` | `2.00GB/s` | +| 8 kib | `2.64µs` | `2.52µs` | `2.44µs` | | `3.11GB/s` | `3.25GB/s` | `3.36GB/s` | +| 16 kib | `4.93µs` | `4.54µs` | `4.48µs` | | `3.33GB/s` | `3.61GB/s` | `3.66GB/s` | +| 32 kib | `9.41µs` | `8.62µs` | `8.54µs` | | `3.48GB/s` | `3.80GB/s` | `3.84GB/s` | +| 64 kib | `18.2µs` | `16.7µs` | `16.6µs` | | `3.59GB/s` | `3.91GB/s` | `3.94GB/s` | +| 128 kib | `36.3µs` | `32.9µs` | `33.1µs` | | `3.61GB/s` | `3.99GB/s` | `3.96GB/s` | +| 256 kib | `72.5µs` | `65.7µs` | `66.0µs` | | `3.62GB/s` | `3.99GB/s` | `3.97GB/s` | +| 512 kib | `145µs` | `131µs` | `132µs` | | `3.60GB/s` | `4.00GB/s` | `3.97GB/s` | +| 1024 kib | `290µs` | `262µs` | `262µs` | | `3.62GB/s` | `4.00GB/s` | `4.00GB/s` | + +### No ASM + +| Size | Incremental | Full Buffer | Reset | | Incremental Rate | Full Buffer Rate | Reset Rate | +|----------|-------------|-------------|------------|-|------------------|------------------|-------------| +| 64 b | `253ns` | `254ns` | `134ns` | | `253MB/s` | `252MB/s` | `478MB/s` | +| 256 b | `553ns` | `557ns` | `441ns` | | `463MB/s` | `459MB/s` | `580MB/s` | +| 512 b | `948ns` | `953ns` | `841ns` | | `540MB/s` | `538MB/s` | `609MB/s` | +| 768 b | `1.38µs` | `1.40µs` | `1.35µs` | | `558MB/s` | `547MB/s` | `570MB/s` | +| 1 kib | `1.77µs` | `1.77µs` | `1.70µs` | | `577MB/s` | `580MB/s` | `602MB/s` | +| | | | | | | | | +| 1024 kib | `880µs` | `883µs` | `878µs` | | `596MB/s` | `595MB/s` | `598MB/s` | + +The speed caps out at around 1 kib, so most rows have been elided from the presentation. diff --git a/vendor/github.com/zeebo/blake3/api.go b/vendor/github.com/zeebo/blake3/api.go new file mode 100644 index 000000000..5de263f08 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/api.go @@ -0,0 +1,166 @@ +// Package blake3 provides an SSE4.1/AVX2 accelerated BLAKE3 implementation. +package blake3 + +import ( + "errors" + + "github.com/zeebo/blake3/internal/consts" + "github.com/zeebo/blake3/internal/utils" +) + +// Hasher is a hash.Hash for BLAKE3. +type Hasher struct { + size int + h hasher +} + +// New returns a new Hasher that has a digest size of 32 bytes. +// +// If you need more or less output bytes than that, use Digest method. +func New() *Hasher { + return &Hasher{ + size: 32, + h: hasher{ + key: consts.IV, + }, + } +} + +// NewKeyed returns a new Hasher that uses the 32 byte input key and has +// a digest size of 32 bytes. +// +// If you need more or less output bytes than that, use the Digest method. +func NewKeyed(key []byte) (*Hasher, error) { + if len(key) != 32 { + return nil, errors.New("invalid key size") + } + + h := &Hasher{ + size: 32, + h: hasher{ + flags: consts.Flag_Keyed, + }, + } + utils.KeyFromBytes(key, &h.h.key) + + return h, nil +} + +// DeriveKey derives a key based on reusable key material of any +// length, in the given context. The key will be stored in out, using +// all of its current length. +// +// Context strings must be hardcoded constants, and the recommended +// format is "[application] [commit timestamp] [purpose]", e.g., +// "example.com 2019-12-25 16:18:03 session tokens v1". +func DeriveKey(context string, material []byte, out []byte) { + h := NewDeriveKey(context) + _, _ = h.Write(material) + _, _ = h.Digest().Read(out) +} + +// NewDeriveKey returns a Hasher that is initialized with the context +// string. See DeriveKey for details. It has a digest size of 32 bytes. +// +// If you need more or less output bytes than that, use the Digest method. +func NewDeriveKey(context string) *Hasher { + // hash the context string and use that instead of IV + h := &Hasher{ + size: 32, + h: hasher{ + key: consts.IV, + flags: consts.Flag_DeriveKeyContext, + }, + } + + var buf [32]byte + _, _ = h.WriteString(context) + _, _ = h.Digest().Read(buf[:]) + + h.Reset() + utils.KeyFromBytes(buf[:], &h.h.key) + h.h.flags = consts.Flag_DeriveKeyMaterial + + return h +} + +// Write implements part of the hash.Hash interface. It never returns an error. +func (h *Hasher) Write(p []byte) (int, error) { + h.h.update(p) + return len(p), nil +} + +// WriteString is like Write but specialized to strings to avoid allocations. +func (h *Hasher) WriteString(p string) (int, error) { + h.h.updateString(p) + return len(p), nil +} + +// Reset implements part of the hash.Hash interface. It causes the Hasher to +// act as if it was newly created. +func (h *Hasher) Reset() { + h.h.reset() +} + +// Clone returns a new Hasher with the same internal state. +// +// Modifying the resulting Hasher will not modify the original Hasher, and vice versa. +func (h *Hasher) Clone() *Hasher { + return &Hasher{size: h.size, h: h.h} +} + +// Size implements part of the hash.Hash interface. It returns the number of +// bytes the hash will output in Sum. +func (h *Hasher) Size() int { + return h.size +} + +// BlockSize implements part of the hash.Hash interface. It returns the most +// natural size to write to the Hasher. +func (h *Hasher) BlockSize() int { + // TODO: is there a downside to picking this large size? + return 8192 +} + +// Sum implements part of the hash.Hash interface. It appends the digest of +// the Hasher to the provided buffer and returns it. +func (h *Hasher) Sum(b []byte) []byte { + if top := len(b) + h.size; top <= cap(b) && top >= len(b) { + h.h.finalize(b[len(b):top]) + return b[:top] + } + + tmp := make([]byte, h.size) + h.h.finalize(tmp) + return append(b, tmp...) +} + +// Digest takes a snapshot of the hash state and returns an object that can +// be used to read and seek through 2^64 bytes of digest output. +func (h *Hasher) Digest() *Digest { + var d Digest + h.h.finalizeDigest(&d) + return &d +} + +// Sum256 returns the first 256 bits of the unkeyed digest of the data. +func Sum256(data []byte) (sum [32]byte) { + out := Sum512(data) + copy(sum[:], out[:32]) + return sum +} + +// Sum512 returns the first 512 bits of the unkeyed digest of the data. +func Sum512(data []byte) (sum [64]byte) { + if len(data) <= consts.ChunkLen { + var d Digest + compressAll(&d, data, 0, consts.IV) + _, _ = d.Read(sum[:]) + return sum + } else { + h := hasher{key: consts.IV} + h.update(data) + h.finalize(sum[:]) + return sum + } +} diff --git a/vendor/github.com/zeebo/blake3/blake3.go b/vendor/github.com/zeebo/blake3/blake3.go new file mode 100644 index 000000000..98dedcabe --- /dev/null +++ b/vendor/github.com/zeebo/blake3/blake3.go @@ -0,0 +1,285 @@ +package blake3 + +import ( + "math/bits" + "unsafe" + + "github.com/zeebo/blake3/internal/alg" + "github.com/zeebo/blake3/internal/consts" + "github.com/zeebo/blake3/internal/utils" +) + +// +// hasher contains state for a blake3 hash +// + +type hasher struct { + len uint64 + chunks uint64 + flags uint32 + key [8]uint32 + stack cvstack + buf [8192]byte +} + +func (a *hasher) reset() { + a.len = 0 + a.chunks = 0 + a.stack.occ = 0 + a.stack.lvls = [8]uint8{} + a.stack.bufn = 0 +} + +func (a *hasher) update(buf []byte) { + // relies on the first two words of a string being the same as a slice + a.updateString(*(*string)(unsafe.Pointer(&buf))) +} + +func (a *hasher) updateString(buf string) { + var input *[8192]byte + + for len(buf) > 0 { + if a.len == 0 && len(buf) > 8192 { + // relies on the data pointer being the first word in the string header + input = (*[8192]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&buf))) + buf = buf[8192:] + } else if a.len < 8192 { + n := copy(a.buf[a.len:], buf) + a.len += uint64(n) + buf = buf[n:] + continue + } else { + input = &a.buf + } + + a.consume(input) + a.len = 0 + a.chunks += 8 + } +} + +func (a *hasher) consume(input *[8192]byte) { + var out chainVector + var chain [8]uint32 + alg.HashF(input, 8192, a.chunks, a.flags, &a.key, &out, &chain) + a.stack.pushN(0, &out, 8, a.flags, &a.key) +} + +func (a *hasher) finalize(p []byte) { + var d Digest + a.finalizeDigest(&d) + _, _ = d.Read(p) +} + +func (a *hasher) finalizeDigest(d *Digest) { + if a.chunks == 0 && a.len <= consts.ChunkLen { + compressAll(d, a.buf[:a.len], a.flags, a.key) + return + } + + d.chain = a.key + d.flags = a.flags | consts.Flag_ChunkEnd + + if a.len > 64 { + var buf chainVector + alg.HashF(&a.buf, a.len, a.chunks, a.flags, &a.key, &buf, &d.chain) + + if a.len > consts.ChunkLen { + complete := (a.len - 1) / consts.ChunkLen + a.stack.pushN(0, &buf, int(complete), a.flags, &a.key) + a.chunks += complete + a.len = uint64(copy(a.buf[:], a.buf[complete*consts.ChunkLen:a.len])) + } + } + + if a.len <= 64 { + d.flags |= consts.Flag_ChunkStart + } + + d.counter = a.chunks + d.blen = uint32(a.len) % 64 + + base := a.len / 64 * 64 + if a.len > 0 && d.blen == 0 { + d.blen = 64 + base -= 64 + } + + if consts.IsLittleEndian { + copy((*[64]byte)(unsafe.Pointer(&d.block[0]))[:], a.buf[base:a.len]) + } else { + var tmp [64]byte + copy(tmp[:], a.buf[base:a.len]) + utils.BytesToWords(&tmp, &d.block) + } + + for a.stack.bufn > 0 { + a.stack.flush(a.flags, &a.key) + } + + var tmp [16]uint32 + for occ := a.stack.occ; occ != 0; occ &= occ - 1 { + col := uint(bits.TrailingZeros64(occ)) % 64 + + alg.Compress(&d.chain, &d.block, d.counter, d.blen, d.flags, &tmp) + + *(*[8]uint32)(unsafe.Pointer(&d.block[0])) = a.stack.stack[col] + *(*[8]uint32)(unsafe.Pointer(&d.block[8])) = *(*[8]uint32)(unsafe.Pointer(&tmp[0])) + + if occ == a.stack.occ { + d.chain = a.key + d.counter = 0 + d.blen = consts.BlockLen + d.flags = a.flags | consts.Flag_Parent + } + } + + d.flags |= consts.Flag_Root +} + +// +// chain value stack +// + +type chainVector = [64]uint32 + +type cvstack struct { + occ uint64 // which levels in stack are occupied + lvls [8]uint8 // what level the buf input was in + bufn int // how many pairs are loaded into buf + buf [2]chainVector + stack [64][8]uint32 +} + +func (a *cvstack) pushN(l uint8, cv *chainVector, n int, flags uint32, key *[8]uint32) { + for i := 0; i < n; i++ { + a.pushL(l, cv, i) + for a.bufn == 8 { + a.flush(flags, key) + } + } +} + +func (a *cvstack) pushL(l uint8, cv *chainVector, n int) { + bit := uint64(1) << (l & 63) + if a.occ&bit == 0 { + readChain(cv, n, &a.stack[l&63]) + a.occ ^= bit + return + } + + a.lvls[a.bufn&7] = l + writeChain(&a.stack[l&63], &a.buf[0], a.bufn) + copyChain(cv, n, &a.buf[1], a.bufn) + a.bufn++ + a.occ ^= bit +} + +func (a *cvstack) flush(flags uint32, key *[8]uint32) { + var out chainVector + alg.HashP(&a.buf[0], &a.buf[1], flags|consts.Flag_Parent, key, &out, a.bufn) + + bufn, lvls := a.bufn, a.lvls + a.bufn, a.lvls = 0, [8]uint8{} + + for i := 0; i < bufn; i++ { + a.pushL(lvls[i]+1, &out, i) + } +} + +// +// helpers to deal with reading/writing transposed values +// + +func copyChain(in *chainVector, icol int, out *chainVector, ocol int) { + type u = uintptr + type p = unsafe.Pointer + type a = *uint32 + + i := p(u(p(in)) + u(icol*4)) + o := p(u(p(out)) + u(ocol*4)) + + *a(p(u(o) + 0*32)) = *a(p(u(i) + 0*32)) + *a(p(u(o) + 1*32)) = *a(p(u(i) + 1*32)) + *a(p(u(o) + 2*32)) = *a(p(u(i) + 2*32)) + *a(p(u(o) + 3*32)) = *a(p(u(i) + 3*32)) + *a(p(u(o) + 4*32)) = *a(p(u(i) + 4*32)) + *a(p(u(o) + 5*32)) = *a(p(u(i) + 5*32)) + *a(p(u(o) + 6*32)) = *a(p(u(i) + 6*32)) + *a(p(u(o) + 7*32)) = *a(p(u(i) + 7*32)) +} + +func readChain(in *chainVector, col int, out *[8]uint32) { + type u = uintptr + type p = unsafe.Pointer + type a = *uint32 + + i := p(u(p(in)) + u(col*4)) + + out[0] = *a(p(u(i) + 0*32)) + out[1] = *a(p(u(i) + 1*32)) + out[2] = *a(p(u(i) + 2*32)) + out[3] = *a(p(u(i) + 3*32)) + out[4] = *a(p(u(i) + 4*32)) + out[5] = *a(p(u(i) + 5*32)) + out[6] = *a(p(u(i) + 6*32)) + out[7] = *a(p(u(i) + 7*32)) +} + +func writeChain(in *[8]uint32, out *chainVector, col int) { + type u = uintptr + type p = unsafe.Pointer + type a = *uint32 + + o := p(u(p(out)) + u(col*4)) + + *a(p(u(o) + 0*32)) = in[0] + *a(p(u(o) + 1*32)) = in[1] + *a(p(u(o) + 2*32)) = in[2] + *a(p(u(o) + 3*32)) = in[3] + *a(p(u(o) + 4*32)) = in[4] + *a(p(u(o) + 5*32)) = in[5] + *a(p(u(o) + 6*32)) = in[6] + *a(p(u(o) + 7*32)) = in[7] +} + +// +// compress <= chunkLen bytes in one shot +// + +func compressAll(d *Digest, in []byte, flags uint32, key [8]uint32) { + var compressed [16]uint32 + + d.chain = key + d.flags = flags | consts.Flag_ChunkStart + + for len(in) > 64 { + buf := (*[64]byte)(unsafe.Pointer(&in[0])) + + var block *[16]uint32 + if consts.IsLittleEndian { + block = (*[16]uint32)(unsafe.Pointer(buf)) + } else { + block = &d.block + utils.BytesToWords(buf, block) + } + + alg.Compress(&d.chain, block, 0, consts.BlockLen, d.flags, &compressed) + + d.chain = *(*[8]uint32)(unsafe.Pointer(&compressed[0])) + d.flags &^= consts.Flag_ChunkStart + + in = in[64:] + } + + if consts.IsLittleEndian { + copy((*[64]byte)(unsafe.Pointer(&d.block[0]))[:], in) + } else { + var tmp [64]byte + copy(tmp[:], in) + utils.BytesToWords(&tmp, &d.block) + } + + d.blen = uint32(len(in)) + d.flags |= consts.Flag_ChunkEnd | consts.Flag_Root +} diff --git a/vendor/github.com/zeebo/blake3/digest.go b/vendor/github.com/zeebo/blake3/digest.go new file mode 100644 index 000000000..58365d5ab --- /dev/null +++ b/vendor/github.com/zeebo/blake3/digest.go @@ -0,0 +1,100 @@ +package blake3 + +import ( + "fmt" + "io" + "unsafe" + + "github.com/zeebo/blake3/internal/alg" + "github.com/zeebo/blake3/internal/consts" + "github.com/zeebo/blake3/internal/utils" +) + +// Digest captures the state of a Hasher allowing reading and seeking through +// the output stream. +type Digest struct { + counter uint64 + chain [8]uint32 + block [16]uint32 + blen uint32 + flags uint32 + buf [16]uint32 + bufn int +} + +// Read reads data frm the hasher into out. It always fills the entire buffer and +// never errors. The stream will wrap around when reading past 2^64 bytes. +func (d *Digest) Read(p []byte) (n int, err error) { + n = len(p) + + if d.bufn > 0 { + n := d.slowCopy(p) + p = p[n:] + d.bufn -= n + } + + for len(p) >= 64 { + d.fillBuf() + + if consts.IsLittleEndian { + *(*[64]byte)(unsafe.Pointer(&p[0])) = *(*[64]byte)(unsafe.Pointer(&d.buf[0])) + } else { + utils.WordsToBytes(&d.buf, p) + } + + p = p[64:] + d.bufn = 0 + } + + if len(p) == 0 { + return n, nil + } + + d.fillBuf() + d.bufn -= d.slowCopy(p) + + return n, nil +} + +// Seek sets the position to the provided location. Only SeekStart and +// SeekCurrent are allowed. +func (d *Digest) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekEnd: + return 0, fmt.Errorf("seek from end not supported") + case io.SeekCurrent: + offset += int64(consts.BlockLen*d.counter) - int64(d.bufn) + default: + return 0, fmt.Errorf("invalid whence: %d", whence) + } + if offset < 0 { + return 0, fmt.Errorf("seek before start") + } + d.setPosition(uint64(offset)) + return offset, nil +} + +func (d *Digest) setPosition(pos uint64) { + d.counter = pos / consts.BlockLen + d.fillBuf() + d.bufn -= int(pos % consts.BlockLen) +} + +func (d *Digest) slowCopy(p []byte) (n int) { + off := uint(consts.BlockLen-d.bufn) % consts.BlockLen + if consts.IsLittleEndian { + n = copy(p, (*[consts.BlockLen]byte)(unsafe.Pointer(&d.buf[0]))[off:]) + } else { + var tmp [consts.BlockLen]byte + utils.WordsToBytes(&d.buf, tmp[:]) + n = copy(p, tmp[off:]) + } + return n +} + +func (d *Digest) fillBuf() { + alg.Compress(&d.chain, &d.block, d.counter, d.blen, d.flags, &d.buf) + d.counter++ + d.bufn = consts.BlockLen +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/alg.go b/vendor/github.com/zeebo/blake3/internal/alg/alg.go new file mode 100644 index 000000000..239fdec5b --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/alg.go @@ -0,0 +1,18 @@ +package alg + +import ( + "github.com/zeebo/blake3/internal/alg/compress" + "github.com/zeebo/blake3/internal/alg/hash" +) + +func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { + hash.HashF(input, length, counter, flags, key, out, chain) +} + +func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { + hash.HashP(left, right, flags, key, out, n) +} + +func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { + compress.Compress(chain, block, counter, blen, flags, out) +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go new file mode 100644 index 000000000..0b2685408 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go @@ -0,0 +1,15 @@ +package compress + +import ( + "github.com/zeebo/blake3/internal/alg/compress/compress_pure" + "github.com/zeebo/blake3/internal/alg/compress/compress_sse41" + "github.com/zeebo/blake3/internal/consts" +) + +func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { + if consts.HasSSE41 { + compress_sse41.Compress(chain, block, counter, blen, flags, out) + } else { + compress_pure.Compress(chain, block, counter, blen, flags, out) + } +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go new file mode 100644 index 000000000..66ea1fb75 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go @@ -0,0 +1,135 @@ +package compress_pure + +import ( + "math/bits" + + "github.com/zeebo/blake3/internal/consts" +) + +func Compress( + chain *[8]uint32, + block *[16]uint32, + counter uint64, + blen uint32, + flags uint32, + out *[16]uint32, +) { + + *out = [16]uint32{ + chain[0], chain[1], chain[2], chain[3], + chain[4], chain[5], chain[6], chain[7], + consts.IV0, consts.IV1, consts.IV2, consts.IV3, + uint32(counter), uint32(counter >> 32), blen, flags, + } + + rcompress(out, block) +} + +func g(a, b, c, d, mx, my uint32) (uint32, uint32, uint32, uint32) { + a += b + mx + d = bits.RotateLeft32(d^a, -16) + c += d + b = bits.RotateLeft32(b^c, -12) + a += b + my + d = bits.RotateLeft32(d^a, -8) + c += d + b = bits.RotateLeft32(b^c, -7) + return a, b, c, d +} + +func rcompress(s *[16]uint32, m *[16]uint32) { + const ( + a = 10 + b = 11 + c = 12 + d = 13 + e = 14 + f = 15 + ) + + s0, s1, s2, s3 := s[0+0], s[0+1], s[0+2], s[0+3] + s4, s5, s6, s7 := s[0+4], s[0+5], s[0+6], s[0+7] + s8, s9, sa, sb := s[8+0], s[8+1], s[8+2], s[8+3] + sc, sd, se, sf := s[8+4], s[8+5], s[8+6], s[8+7] + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[0], m[1]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[2], m[3]) + s2, s6, sa, se = g(s2, s6, sa, se, m[4], m[5]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[6], m[7]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[8], m[9]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[a], m[b]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[c], m[d]) + s3, s4, s9, se = g(s3, s4, s9, se, m[e], m[f]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[2], m[6]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[3], m[a]) + s2, s6, sa, se = g(s2, s6, sa, se, m[7], m[0]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[4], m[d]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[1], m[b]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[c], m[5]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[9], m[e]) + s3, s4, s9, se = g(s3, s4, s9, se, m[f], m[8]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[3], m[4]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[a], m[c]) + s2, s6, sa, se = g(s2, s6, sa, se, m[d], m[2]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[7], m[e]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[6], m[5]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[9], m[0]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[b], m[f]) + s3, s4, s9, se = g(s3, s4, s9, se, m[8], m[1]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[a], m[7]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[c], m[9]) + s2, s6, sa, se = g(s2, s6, sa, se, m[e], m[3]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[d], m[f]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[4], m[0]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[b], m[2]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[5], m[8]) + s3, s4, s9, se = g(s3, s4, s9, se, m[1], m[6]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[c], m[d]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[9], m[b]) + s2, s6, sa, se = g(s2, s6, sa, se, m[f], m[a]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[e], m[8]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[7], m[2]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[5], m[3]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[0], m[1]) + s3, s4, s9, se = g(s3, s4, s9, se, m[6], m[4]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[9], m[e]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[b], m[5]) + s2, s6, sa, se = g(s2, s6, sa, se, m[8], m[c]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[f], m[1]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[d], m[3]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[0], m[a]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[2], m[6]) + s3, s4, s9, se = g(s3, s4, s9, se, m[4], m[7]) + + s0, s4, s8, sc = g(s0, s4, s8, sc, m[b], m[f]) + s1, s5, s9, sd = g(s1, s5, s9, sd, m[5], m[0]) + s2, s6, sa, se = g(s2, s6, sa, se, m[1], m[9]) + s3, s7, sb, sf = g(s3, s7, sb, sf, m[8], m[6]) + s0, s5, sa, sf = g(s0, s5, sa, sf, m[e], m[a]) + s1, s6, sb, sc = g(s1, s6, sb, sc, m[2], m[c]) + s2, s7, s8, sd = g(s2, s7, s8, sd, m[3], m[4]) + s3, s4, s9, se = g(s3, s4, s9, se, m[7], m[d]) + + s[8+0] = s8 ^ s[0] + s[8+1] = s9 ^ s[1] + s[8+2] = sa ^ s[2] + s[8+3] = sb ^ s[3] + s[8+4] = sc ^ s[4] + s[8+5] = sd ^ s[5] + s[8+6] = se ^ s[6] + s[8+7] = sf ^ s[7] + + s[0] = s0 ^ s8 + s[1] = s1 ^ s9 + s[2] = s2 ^ sa + s[3] = s3 ^ sb + s[4] = s4 ^ sc + s[5] = s5 ^ sd + s[6] = s6 ^ se + s[7] = s7 ^ sf +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s new file mode 100644 index 000000000..0fedf0b3a --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s @@ -0,0 +1,560 @@ +// Code generated by command: go run compress.go. DO NOT EDIT. + +#include "textflag.h" + +DATA iv<>+0(SB)/4, $0x6a09e667 +DATA iv<>+4(SB)/4, $0xbb67ae85 +DATA iv<>+8(SB)/4, $0x3c6ef372 +DATA iv<>+12(SB)/4, $0xa54ff53a +DATA iv<>+16(SB)/4, $0x510e527f +DATA iv<>+20(SB)/4, $0x9b05688c +DATA iv<>+24(SB)/4, $0x1f83d9ab +DATA iv<>+28(SB)/4, $0x5be0cd19 +GLOBL iv<>(SB), RODATA|NOPTR, $32 + +DATA rot16_shuf<>+0(SB)/1, $0x02 +DATA rot16_shuf<>+1(SB)/1, $0x03 +DATA rot16_shuf<>+2(SB)/1, $0x00 +DATA rot16_shuf<>+3(SB)/1, $0x01 +DATA rot16_shuf<>+4(SB)/1, $0x06 +DATA rot16_shuf<>+5(SB)/1, $0x07 +DATA rot16_shuf<>+6(SB)/1, $0x04 +DATA rot16_shuf<>+7(SB)/1, $0x05 +DATA rot16_shuf<>+8(SB)/1, $0x0a +DATA rot16_shuf<>+9(SB)/1, $0x0b +DATA rot16_shuf<>+10(SB)/1, $0x08 +DATA rot16_shuf<>+11(SB)/1, $0x09 +DATA rot16_shuf<>+12(SB)/1, $0x0e +DATA rot16_shuf<>+13(SB)/1, $0x0f +DATA rot16_shuf<>+14(SB)/1, $0x0c +DATA rot16_shuf<>+15(SB)/1, $0x0d +DATA rot16_shuf<>+16(SB)/1, $0x12 +DATA rot16_shuf<>+17(SB)/1, $0x13 +DATA rot16_shuf<>+18(SB)/1, $0x10 +DATA rot16_shuf<>+19(SB)/1, $0x11 +DATA rot16_shuf<>+20(SB)/1, $0x16 +DATA rot16_shuf<>+21(SB)/1, $0x17 +DATA rot16_shuf<>+22(SB)/1, $0x14 +DATA rot16_shuf<>+23(SB)/1, $0x15 +DATA rot16_shuf<>+24(SB)/1, $0x1a +DATA rot16_shuf<>+25(SB)/1, $0x1b +DATA rot16_shuf<>+26(SB)/1, $0x18 +DATA rot16_shuf<>+27(SB)/1, $0x19 +DATA rot16_shuf<>+28(SB)/1, $0x1e +DATA rot16_shuf<>+29(SB)/1, $0x1f +DATA rot16_shuf<>+30(SB)/1, $0x1c +DATA rot16_shuf<>+31(SB)/1, $0x1d +GLOBL rot16_shuf<>(SB), RODATA|NOPTR, $32 + +DATA rot8_shuf<>+0(SB)/1, $0x01 +DATA rot8_shuf<>+1(SB)/1, $0x02 +DATA rot8_shuf<>+2(SB)/1, $0x03 +DATA rot8_shuf<>+3(SB)/1, $0x00 +DATA rot8_shuf<>+4(SB)/1, $0x05 +DATA rot8_shuf<>+5(SB)/1, $0x06 +DATA rot8_shuf<>+6(SB)/1, $0x07 +DATA rot8_shuf<>+7(SB)/1, $0x04 +DATA rot8_shuf<>+8(SB)/1, $0x09 +DATA rot8_shuf<>+9(SB)/1, $0x0a +DATA rot8_shuf<>+10(SB)/1, $0x0b +DATA rot8_shuf<>+11(SB)/1, $0x08 +DATA rot8_shuf<>+12(SB)/1, $0x0d +DATA rot8_shuf<>+13(SB)/1, $0x0e +DATA rot8_shuf<>+14(SB)/1, $0x0f +DATA rot8_shuf<>+15(SB)/1, $0x0c +DATA rot8_shuf<>+16(SB)/1, $0x11 +DATA rot8_shuf<>+17(SB)/1, $0x12 +DATA rot8_shuf<>+18(SB)/1, $0x13 +DATA rot8_shuf<>+19(SB)/1, $0x10 +DATA rot8_shuf<>+20(SB)/1, $0x15 +DATA rot8_shuf<>+21(SB)/1, $0x16 +DATA rot8_shuf<>+22(SB)/1, $0x17 +DATA rot8_shuf<>+23(SB)/1, $0x14 +DATA rot8_shuf<>+24(SB)/1, $0x19 +DATA rot8_shuf<>+25(SB)/1, $0x1a +DATA rot8_shuf<>+26(SB)/1, $0x1b +DATA rot8_shuf<>+27(SB)/1, $0x18 +DATA rot8_shuf<>+28(SB)/1, $0x1d +DATA rot8_shuf<>+29(SB)/1, $0x1e +DATA rot8_shuf<>+30(SB)/1, $0x1f +DATA rot8_shuf<>+31(SB)/1, $0x1c +GLOBL rot8_shuf<>(SB), RODATA|NOPTR, $32 + +// func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) +// Requires: SSE, SSE2, SSE4.1, SSSE3 +TEXT ·Compress(SB), NOSPLIT, $0-40 + MOVQ chain+0(FP), AX + MOVQ block+8(FP), CX + MOVQ counter+16(FP), DX + MOVL blen+24(FP), BX + MOVL flags+28(FP), BP + MOVQ out+32(FP), SI + MOVUPS (AX), X0 + MOVUPS 16(AX), X1 + MOVUPS iv<>+0(SB), X2 + PINSRD $0x00, DX, X3 + SHRQ $0x20, DX + PINSRD $0x01, DX, X3 + PINSRD $0x02, BX, X3 + PINSRD $0x03, BP, X3 + MOVUPS (CX), X4 + MOVUPS 16(CX), X5 + MOVUPS 32(CX), X6 + MOVUPS 48(CX), X7 + MOVUPS rot16_shuf<>+0(SB), X8 + MOVUPS rot8_shuf<>+0(SB), X9 + + // round 1 + MOVAPS X4, X10 + SHUFPS $0x88, X5, X10 + PADDD X10, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X11 + PSRLL $0x0c, X1 + PSLLL $0x14, X11 + POR X11, X1 + MOVAPS X4, X4 + SHUFPS $0xdd, X5, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X6, X5 + SHUFPS $0x88, X7, X5 + SHUFPS $0x93, X5, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X11 + PSRLL $0x0c, X1 + PSLLL $0x14, X11 + POR X11, X1 + MOVAPS X6, X6 + SHUFPS $0xdd, X7, X6 + SHUFPS $0x93, X6, X6 + PADDD X6, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x07, X1 + PSLLL $0x19, X7 + POR X7, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 2 + MOVAPS X10, X7 + SHUFPS $0xd6, X4, X7 + SHUFPS $0x39, X7, X7 + PADDD X7, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X11 + PSRLL $0x0c, X1 + PSLLL $0x14, X11 + POR X11, X1 + MOVAPS X5, X11 + SHUFPS $0xfa, X6, X11 + PSHUFD $0x0f, X10, X10 + PBLENDW $0x33, X10, X11 + PADDD X11, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X10 + PSRLL $0x07, X1 + PSLLL $0x19, X10 + POR X10, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X6, X12 + PUNPCKLLQ X4, X12 + PBLENDW $0xc0, X5, X12 + SHUFPS $0xb4, X12, X12 + PADDD X12, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X10 + PSRLL $0x0c, X1 + PSLLL $0x14, X10 + POR X10, X1 + MOVAPS X4, X10 + PUNPCKHLQ X6, X10 + MOVAPS X5, X4 + PUNPCKLLQ X10, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 3 + MOVAPS X7, X5 + SHUFPS $0xd6, X11, X5 + SHUFPS $0x39, X5, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X6 + PSRLL $0x0c, X1 + PSLLL $0x14, X6 + POR X6, X1 + MOVAPS X12, X6 + SHUFPS $0xfa, X4, X6 + PSHUFD $0x0f, X7, X7 + PBLENDW $0x33, X7, X6 + PADDD X6, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x07, X1 + PSLLL $0x19, X7 + POR X7, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X4, X10 + PUNPCKLLQ X11, X10 + PBLENDW $0xc0, X12, X10 + SHUFPS $0xb4, X10, X10 + PADDD X10, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x0c, X1 + PSLLL $0x14, X7 + POR X7, X1 + MOVAPS X11, X7 + PUNPCKHLQ X4, X7 + MOVAPS X12, X4 + PUNPCKLLQ X7, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x07, X1 + PSLLL $0x19, X7 + POR X7, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 4 + MOVAPS X5, X7 + SHUFPS $0xd6, X6, X7 + SHUFPS $0x39, X7, X7 + PADDD X7, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X11 + PSRLL $0x0c, X1 + PSLLL $0x14, X11 + POR X11, X1 + MOVAPS X10, X11 + SHUFPS $0xfa, X4, X11 + PSHUFD $0x0f, X5, X5 + PBLENDW $0x33, X5, X11 + PADDD X11, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X4, X12 + PUNPCKLLQ X6, X12 + PBLENDW $0xc0, X10, X12 + SHUFPS $0xb4, X12, X12 + PADDD X12, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x0c, X1 + PSLLL $0x14, X5 + POR X5, X1 + MOVAPS X6, X5 + PUNPCKHLQ X4, X5 + MOVAPS X10, X4 + PUNPCKLLQ X5, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 5 + MOVAPS X7, X5 + SHUFPS $0xd6, X11, X5 + SHUFPS $0x39, X5, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X6 + PSRLL $0x0c, X1 + PSLLL $0x14, X6 + POR X6, X1 + MOVAPS X12, X6 + SHUFPS $0xfa, X4, X6 + PSHUFD $0x0f, X7, X7 + PBLENDW $0x33, X7, X6 + PADDD X6, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x07, X1 + PSLLL $0x19, X7 + POR X7, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X4, X10 + PUNPCKLLQ X11, X10 + PBLENDW $0xc0, X12, X10 + SHUFPS $0xb4, X10, X10 + PADDD X10, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x0c, X1 + PSLLL $0x14, X7 + POR X7, X1 + MOVAPS X11, X7 + PUNPCKHLQ X4, X7 + MOVAPS X12, X4 + PUNPCKLLQ X7, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X7 + PSRLL $0x07, X1 + PSLLL $0x19, X7 + POR X7, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 6 + MOVAPS X5, X7 + SHUFPS $0xd6, X6, X7 + SHUFPS $0x39, X7, X7 + PADDD X7, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X11 + PSRLL $0x0c, X1 + PSLLL $0x14, X11 + POR X11, X1 + MOVAPS X10, X11 + SHUFPS $0xfa, X4, X11 + PSHUFD $0x0f, X5, X5 + PBLENDW $0x33, X5, X11 + PADDD X11, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X4, X12 + PUNPCKLLQ X6, X12 + PBLENDW $0xc0, X10, X12 + SHUFPS $0xb4, X12, X12 + PADDD X12, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x0c, X1 + PSLLL $0x14, X5 + POR X5, X1 + MOVAPS X6, X5 + PUNPCKHLQ X4, X5 + MOVAPS X10, X4 + PUNPCKLLQ X5, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // round 7 + MOVAPS X7, X5 + SHUFPS $0xd6, X11, X5 + SHUFPS $0x39, X5, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x0c, X1 + PSLLL $0x14, X5 + POR X5, X1 + MOVAPS X12, X5 + SHUFPS $0xfa, X4, X5 + PSHUFD $0x0f, X7, X6 + PBLENDW $0x33, X6, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x07, X1 + PSLLL $0x19, X5 + POR X5, X1 + PSHUFD $0x93, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x39, X2, X2 + MOVAPS X4, X5 + PUNPCKLLQ X11, X5 + PBLENDW $0xc0, X12, X5 + SHUFPS $0xb4, X5, X5 + PADDD X5, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X8, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X5 + PSRLL $0x0c, X1 + PSLLL $0x14, X5 + POR X5, X1 + MOVAPS X11, X6 + PUNPCKHLQ X4, X6 + MOVAPS X12, X4 + PUNPCKLLQ X6, X4 + SHUFPS $0x1e, X4, X4 + PADDD X4, X0 + PADDD X1, X0 + PXOR X0, X3 + PSHUFB X9, X3 + PADDD X3, X2 + PXOR X2, X1 + MOVAPS X1, X4 + PSRLL $0x07, X1 + PSLLL $0x19, X4 + POR X4, X1 + PSHUFD $0x39, X0, X0 + PSHUFD $0x4e, X3, X3 + PSHUFD $0x93, X2, X2 + + // finalize + PXOR X2, X0 + PXOR X3, X1 + MOVUPS (AX), X4 + PXOR X4, X2 + MOVUPS 16(AX), X4 + PXOR X4, X3 + MOVUPS X0, (SI) + MOVUPS X1, 16(SI) + MOVUPS X2, 32(SI) + MOVUPS X3, 48(SI) + RET diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go new file mode 100644 index 000000000..cd63e9740 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go @@ -0,0 +1,9 @@ +// +build !amd64 + +package compress_sse41 + +import "github.com/zeebo/blake3/internal/alg/compress/compress_pure" + +func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { + compress_pure.Compress(chain, block, counter, blen, flags, out) +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go new file mode 100644 index 000000000..ffd932d3c --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go @@ -0,0 +1,6 @@ +// +build amd64 + +package compress_sse41 + +//go:noescape +func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go new file mode 100644 index 000000000..ac43abb69 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go @@ -0,0 +1,23 @@ +package hash + +import ( + "github.com/zeebo/blake3/internal/alg/hash/hash_avx2" + "github.com/zeebo/blake3/internal/alg/hash/hash_pure" + "github.com/zeebo/blake3/internal/consts" +) + +func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { + if consts.HasAVX2 && length > 2*consts.ChunkLen { + hash_avx2.HashF(input, length, counter, flags, key, out, chain) + } else { + hash_pure.HashF(input, length, counter, flags, key, out, chain) + } +} + +func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { + if consts.HasAVX2 && n >= 2 { + hash_avx2.HashP(left, right, flags, key, out, n) + } else { + hash_pure.HashP(left, right, flags, key, out, n) + } +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s new file mode 100644 index 000000000..d7531664b --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s @@ -0,0 +1,2561 @@ +// Code generated by command: go run main.go. DO NOT EDIT. + +#include "textflag.h" + +DATA iv<>+0(SB)/4, $0x6a09e667 +DATA iv<>+4(SB)/4, $0xbb67ae85 +DATA iv<>+8(SB)/4, $0x3c6ef372 +DATA iv<>+12(SB)/4, $0xa54ff53a +DATA iv<>+16(SB)/4, $0x510e527f +DATA iv<>+20(SB)/4, $0x9b05688c +DATA iv<>+24(SB)/4, $0x1f83d9ab +DATA iv<>+28(SB)/4, $0x5be0cd19 +GLOBL iv<>(SB), RODATA|NOPTR, $32 + +DATA rot16_shuf<>+0(SB)/1, $0x02 +DATA rot16_shuf<>+1(SB)/1, $0x03 +DATA rot16_shuf<>+2(SB)/1, $0x00 +DATA rot16_shuf<>+3(SB)/1, $0x01 +DATA rot16_shuf<>+4(SB)/1, $0x06 +DATA rot16_shuf<>+5(SB)/1, $0x07 +DATA rot16_shuf<>+6(SB)/1, $0x04 +DATA rot16_shuf<>+7(SB)/1, $0x05 +DATA rot16_shuf<>+8(SB)/1, $0x0a +DATA rot16_shuf<>+9(SB)/1, $0x0b +DATA rot16_shuf<>+10(SB)/1, $0x08 +DATA rot16_shuf<>+11(SB)/1, $0x09 +DATA rot16_shuf<>+12(SB)/1, $0x0e +DATA rot16_shuf<>+13(SB)/1, $0x0f +DATA rot16_shuf<>+14(SB)/1, $0x0c +DATA rot16_shuf<>+15(SB)/1, $0x0d +DATA rot16_shuf<>+16(SB)/1, $0x12 +DATA rot16_shuf<>+17(SB)/1, $0x13 +DATA rot16_shuf<>+18(SB)/1, $0x10 +DATA rot16_shuf<>+19(SB)/1, $0x11 +DATA rot16_shuf<>+20(SB)/1, $0x16 +DATA rot16_shuf<>+21(SB)/1, $0x17 +DATA rot16_shuf<>+22(SB)/1, $0x14 +DATA rot16_shuf<>+23(SB)/1, $0x15 +DATA rot16_shuf<>+24(SB)/1, $0x1a +DATA rot16_shuf<>+25(SB)/1, $0x1b +DATA rot16_shuf<>+26(SB)/1, $0x18 +DATA rot16_shuf<>+27(SB)/1, $0x19 +DATA rot16_shuf<>+28(SB)/1, $0x1e +DATA rot16_shuf<>+29(SB)/1, $0x1f +DATA rot16_shuf<>+30(SB)/1, $0x1c +DATA rot16_shuf<>+31(SB)/1, $0x1d +GLOBL rot16_shuf<>(SB), RODATA|NOPTR, $32 + +DATA rot8_shuf<>+0(SB)/1, $0x01 +DATA rot8_shuf<>+1(SB)/1, $0x02 +DATA rot8_shuf<>+2(SB)/1, $0x03 +DATA rot8_shuf<>+3(SB)/1, $0x00 +DATA rot8_shuf<>+4(SB)/1, $0x05 +DATA rot8_shuf<>+5(SB)/1, $0x06 +DATA rot8_shuf<>+6(SB)/1, $0x07 +DATA rot8_shuf<>+7(SB)/1, $0x04 +DATA rot8_shuf<>+8(SB)/1, $0x09 +DATA rot8_shuf<>+9(SB)/1, $0x0a +DATA rot8_shuf<>+10(SB)/1, $0x0b +DATA rot8_shuf<>+11(SB)/1, $0x08 +DATA rot8_shuf<>+12(SB)/1, $0x0d +DATA rot8_shuf<>+13(SB)/1, $0x0e +DATA rot8_shuf<>+14(SB)/1, $0x0f +DATA rot8_shuf<>+15(SB)/1, $0x0c +DATA rot8_shuf<>+16(SB)/1, $0x11 +DATA rot8_shuf<>+17(SB)/1, $0x12 +DATA rot8_shuf<>+18(SB)/1, $0x13 +DATA rot8_shuf<>+19(SB)/1, $0x10 +DATA rot8_shuf<>+20(SB)/1, $0x15 +DATA rot8_shuf<>+21(SB)/1, $0x16 +DATA rot8_shuf<>+22(SB)/1, $0x17 +DATA rot8_shuf<>+23(SB)/1, $0x14 +DATA rot8_shuf<>+24(SB)/1, $0x19 +DATA rot8_shuf<>+25(SB)/1, $0x1a +DATA rot8_shuf<>+26(SB)/1, $0x1b +DATA rot8_shuf<>+27(SB)/1, $0x18 +DATA rot8_shuf<>+28(SB)/1, $0x1d +DATA rot8_shuf<>+29(SB)/1, $0x1e +DATA rot8_shuf<>+30(SB)/1, $0x1f +DATA rot8_shuf<>+31(SB)/1, $0x1c +GLOBL rot8_shuf<>(SB), RODATA|NOPTR, $32 + +DATA block_len<>+0(SB)/4, $0x00000040 +DATA block_len<>+4(SB)/4, $0x00000040 +DATA block_len<>+8(SB)/4, $0x00000040 +DATA block_len<>+12(SB)/4, $0x00000040 +DATA block_len<>+16(SB)/4, $0x00000040 +DATA block_len<>+20(SB)/4, $0x00000040 +DATA block_len<>+24(SB)/4, $0x00000040 +DATA block_len<>+28(SB)/4, $0x00000040 +GLOBL block_len<>(SB), RODATA|NOPTR, $32 + +DATA zero<>+0(SB)/4, $0x00000000 +DATA zero<>+4(SB)/4, $0x00000000 +DATA zero<>+8(SB)/4, $0x00000000 +DATA zero<>+12(SB)/4, $0x00000000 +DATA zero<>+16(SB)/4, $0x00000000 +DATA zero<>+20(SB)/4, $0x00000000 +DATA zero<>+24(SB)/4, $0x00000000 +DATA zero<>+28(SB)/4, $0x00000000 +GLOBL zero<>(SB), RODATA|NOPTR, $32 + +DATA counter<>+0(SB)/8, $0x0000000000000000 +DATA counter<>+8(SB)/8, $0x0000000000000001 +DATA counter<>+16(SB)/8, $0x0000000000000002 +DATA counter<>+24(SB)/8, $0x0000000000000003 +DATA counter<>+32(SB)/8, $0x0000000000000004 +DATA counter<>+40(SB)/8, $0x0000000000000005 +DATA counter<>+48(SB)/8, $0x0000000000000006 +DATA counter<>+56(SB)/8, $0x0000000000000007 +GLOBL counter<>(SB), RODATA|NOPTR, $64 + +// func HashF(input *[8192]byte, length uint64, counter uint64, flags uint32, key *[8]uint32, out *[32]uint32, chain *[8]uint32) +// Requires: AVX, AVX2 +TEXT ·HashF(SB), $688-56 + MOVQ input+0(FP), AX + MOVQ length+8(FP), CX + MOVQ counter+16(FP), DX + MOVL flags+24(FP), BX + MOVQ key+32(FP), BP + MOVQ out+40(FP), SI + MOVQ chain+48(FP), DI + + // Allocate local space and align it + LEAQ 31(SP), R10 + MOVQ $0x000000000000001f, R8 + NOTQ R8 + ANDQ R8, R10 + + // Skip if the length is zero + XORQ R8, R8 + XORQ R9, R9 + TESTQ CX, CX + JZ skip_compute + + // Compute complete chunks and blocks + SUBQ $0x01, CX + MOVQ CX, R8 + SHRQ $0x0a, R8 + MOVQ CX, R9 + ANDQ $0x000003c0, R9 + +skip_compute: + // Load some params into the stack (avo improvment?) + MOVL BX, 64(SP) + MOVQ DX, 72(SP) + + // Load IV into vectors + VPBROADCASTD (BP), Y0 + VPBROADCASTD 4(BP), Y1 + VPBROADCASTD 8(BP), Y2 + VPBROADCASTD 12(BP), Y3 + VPBROADCASTD 16(BP), Y4 + VPBROADCASTD 20(BP), Y5 + VPBROADCASTD 24(BP), Y6 + VPBROADCASTD 28(BP), Y7 + + // Build and store counter data on the stack + VPBROADCASTQ 72(SP), Y8 + VPADDQ counter<>+0(SB), Y8, Y8 + VPBROADCASTQ 72(SP), Y9 + VPADDQ counter<>+32(SB), Y9, Y9 + VPUNPCKLDQ Y9, Y8, Y10 + VPUNPCKHDQ Y9, Y8, Y8 + VPUNPCKLDQ Y8, Y10, Y9 + VPUNPCKHDQ Y8, Y10, Y8 + VPERMQ $0xd8, Y9, Y9 + VPERMQ $0xd8, Y8, Y8 + VMOVDQU Y9, 112(SP) + VMOVDQU Y8, 144(SP) + + // Set up block flags and variables for iteration + XORQ CX, CX + ORL $0x01, 64(SP) + +loop: + // Include end flags if last block + CMPQ CX, $0x000003c0 + JNE round_setup + ORL $0x02, 64(SP) + +round_setup: + // Load and transpose message vectors + VMOVDQU (AX)(CX*1), Y8 + VMOVDQU 1024(AX)(CX*1), Y9 + VMOVDQU 2048(AX)(CX*1), Y10 + VMOVDQU 3072(AX)(CX*1), Y11 + VMOVDQU 4096(AX)(CX*1), Y12 + VMOVDQU 5120(AX)(CX*1), Y13 + VMOVDQU 6144(AX)(CX*1), Y14 + VMOVDQU 7168(AX)(CX*1), Y15 + VMOVDQA Y0, (R10) + VPUNPCKLDQ Y9, Y8, Y0 + VPUNPCKHDQ Y9, Y8, Y8 + VPUNPCKLDQ Y11, Y10, Y9 + VPUNPCKHDQ Y11, Y10, Y10 + VPUNPCKLDQ Y13, Y12, Y11 + VPUNPCKHDQ Y13, Y12, Y12 + VPUNPCKLDQ Y15, Y14, Y13 + VPUNPCKHDQ Y15, Y14, Y14 + VPUNPCKLQDQ Y9, Y0, Y15 + VPUNPCKHQDQ Y9, Y0, Y0 + VPUNPCKLQDQ Y10, Y8, Y9 + VPUNPCKHQDQ Y10, Y8, Y8 + VPUNPCKLQDQ Y13, Y11, Y10 + VPUNPCKHQDQ Y13, Y11, Y11 + VPUNPCKLQDQ Y14, Y12, Y13 + VPUNPCKHQDQ Y14, Y12, Y12 + VINSERTI128 $0x01, X10, Y15, Y14 + VPERM2I128 $0x31, Y10, Y15, Y10 + VINSERTI128 $0x01, X11, Y0, Y15 + VPERM2I128 $0x31, Y11, Y0, Y0 + VINSERTI128 $0x01, X13, Y9, Y11 + VPERM2I128 $0x31, Y13, Y9, Y9 + VINSERTI128 $0x01, X12, Y8, Y13 + VPERM2I128 $0x31, Y12, Y8, Y8 + VMOVDQU Y14, 176(SP) + VMOVDQU Y15, 208(SP) + VMOVDQU Y11, 240(SP) + VMOVDQU Y13, 272(SP) + VMOVDQU Y10, 304(SP) + VMOVDQU Y0, 336(SP) + VMOVDQU Y9, 368(SP) + VMOVDQU Y8, 400(SP) + VMOVDQU 32(AX)(CX*1), Y0 + VMOVDQU 1056(AX)(CX*1), Y8 + VMOVDQU 2080(AX)(CX*1), Y9 + VMOVDQU 3104(AX)(CX*1), Y10 + VMOVDQU 4128(AX)(CX*1), Y11 + VMOVDQU 5152(AX)(CX*1), Y12 + VMOVDQU 6176(AX)(CX*1), Y13 + VMOVDQU 7200(AX)(CX*1), Y14 + VPUNPCKLDQ Y8, Y0, Y15 + VPUNPCKHDQ Y8, Y0, Y0 + VPUNPCKLDQ Y10, Y9, Y8 + VPUNPCKHDQ Y10, Y9, Y9 + VPUNPCKLDQ Y12, Y11, Y10 + VPUNPCKHDQ Y12, Y11, Y11 + VPUNPCKLDQ Y14, Y13, Y12 + VPUNPCKHDQ Y14, Y13, Y13 + VPUNPCKLQDQ Y8, Y15, Y14 + VPUNPCKHQDQ Y8, Y15, Y8 + VPUNPCKLQDQ Y9, Y0, Y15 + VPUNPCKHQDQ Y9, Y0, Y0 + VPUNPCKLQDQ Y12, Y10, Y9 + VPUNPCKHQDQ Y12, Y10, Y10 + VPUNPCKLQDQ Y13, Y11, Y12 + VPUNPCKHQDQ Y13, Y11, Y11 + VINSERTI128 $0x01, X9, Y14, Y13 + VPERM2I128 $0x31, Y9, Y14, Y9 + VINSERTI128 $0x01, X10, Y8, Y14 + VPERM2I128 $0x31, Y10, Y8, Y8 + VINSERTI128 $0x01, X12, Y15, Y10 + VPERM2I128 $0x31, Y12, Y15, Y12 + VINSERTI128 $0x01, X11, Y0, Y15 + VPERM2I128 $0x31, Y11, Y0, Y0 + VMOVDQU Y13, 432(SP) + VMOVDQU Y14, 464(SP) + VMOVDQU Y10, 496(SP) + VMOVDQU Y15, 528(SP) + VMOVDQU Y9, 560(SP) + VMOVDQU Y8, 592(SP) + VMOVDQU Y12, 624(SP) + VMOVDQU Y0, 656(SP) + + // Load constants for the round + VMOVDQA (R10), Y0 + VMOVDQU block_len<>+0(SB), Y8 + VPBROADCASTD 64(SP), Y9 + VPBROADCASTD iv<>+0(SB), Y10 + VPBROADCASTD iv<>+4(SB), Y11 + VPBROADCASTD iv<>+8(SB), Y12 + VPBROADCASTD iv<>+12(SB), Y13 + VMOVDQU 112(SP), Y14 + VMOVDQU 144(SP), Y15 + + // Save state for partial chunk if necessary + CMPQ CX, R9 + JNE begin_rounds + VMOVDQU Y0, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, (DI) + VMOVDQU Y1, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 4(DI) + VMOVDQU Y2, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 8(DI) + VMOVDQU Y3, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 12(DI) + VMOVDQU Y4, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 16(DI) + VMOVDQU Y5, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 20(DI) + VMOVDQU Y6, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 24(DI) + VMOVDQU Y7, 80(SP) + MOVL 80(SP)(R8*4), DX + MOVL DX, 28(DI) + +begin_rounds: + // Perform the rounds + // Round 1 + VPADDD 176(SP), Y0, Y0 + VPADDD 240(SP), Y1, Y1 + VPADDD 304(SP), Y2, Y2 + VPADDD 368(SP), Y3, Y3 + VPADDD Y4, Y0, Y0 + VPXOR Y0, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y7, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y4, Y4 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y5, Y5 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y6, Y6 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y7, Y7 + VMOVDQA Y0, (R10) + VPSRLD $0x0c, Y4, Y0 + VPSLLD $0x14, Y4, Y4 + VPOR Y0, Y4, Y0 + VPSRLD $0x0c, Y5, Y4 + VPSLLD $0x14, Y5, Y5 + VPOR Y4, Y5, Y4 + VPSRLD $0x0c, Y6, Y5 + VPSLLD $0x14, Y6, Y6 + VPOR Y5, Y6, Y5 + VPSRLD $0x0c, Y7, Y6 + VPSLLD $0x14, Y7, Y7 + VPOR Y6, Y7, Y6 + VMOVDQA (R10), Y7 + VPADDD 208(SP), Y7, Y7 + VPADDD 272(SP), Y1, Y1 + VPADDD 336(SP), Y2, Y2 + VPADDD 400(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 432(SP), Y7, Y7 + VPADDD 496(SP), Y1, Y1 + VPADDD 560(SP), Y2, Y2 + VPADDD 624(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 464(SP), Y7, Y7 + VPADDD 528(SP), Y1, Y1 + VPADDD 592(SP), Y2, Y2 + VPADDD 656(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 2 + VMOVDQA (R10), Y7 + VPADDD 240(SP), Y7, Y7 + VPADDD 272(SP), Y1, Y1 + VPADDD 400(SP), Y2, Y2 + VPADDD 304(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 368(SP), Y7, Y7 + VPADDD 496(SP), Y1, Y1 + VPADDD 176(SP), Y2, Y2 + VPADDD 592(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 208(SP), Y7, Y7 + VPADDD 560(SP), Y1, Y1 + VPADDD 464(SP), Y2, Y2 + VPADDD 656(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 528(SP), Y7, Y7 + VPADDD 336(SP), Y1, Y1 + VPADDD 624(SP), Y2, Y2 + VPADDD 432(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 3 + VMOVDQA (R10), Y7 + VPADDD 272(SP), Y7, Y7 + VPADDD 496(SP), Y1, Y1 + VPADDD 592(SP), Y2, Y2 + VPADDD 400(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 304(SP), Y7, Y7 + VPADDD 560(SP), Y1, Y1 + VPADDD 240(SP), Y2, Y2 + VPADDD 624(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 368(SP), Y7, Y7 + VPADDD 464(SP), Y1, Y1 + VPADDD 528(SP), Y2, Y2 + VPADDD 432(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 336(SP), Y7, Y7 + VPADDD 176(SP), Y1, Y1 + VPADDD 656(SP), Y2, Y2 + VPADDD 208(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 4 + VMOVDQA (R10), Y7 + VPADDD 496(SP), Y7, Y7 + VPADDD 560(SP), Y1, Y1 + VPADDD 624(SP), Y2, Y2 + VPADDD 592(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 400(SP), Y7, Y7 + VPADDD 464(SP), Y1, Y1 + VPADDD 272(SP), Y2, Y2 + VPADDD 656(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 304(SP), Y7, Y7 + VPADDD 528(SP), Y1, Y1 + VPADDD 336(SP), Y2, Y2 + VPADDD 208(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 176(SP), Y7, Y7 + VPADDD 240(SP), Y1, Y1 + VPADDD 432(SP), Y2, Y2 + VPADDD 368(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 5 + VMOVDQA (R10), Y7 + VPADDD 560(SP), Y7, Y7 + VPADDD 464(SP), Y1, Y1 + VPADDD 656(SP), Y2, Y2 + VPADDD 624(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 592(SP), Y7, Y7 + VPADDD 528(SP), Y1, Y1 + VPADDD 496(SP), Y2, Y2 + VPADDD 432(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 400(SP), Y7, Y7 + VPADDD 336(SP), Y1, Y1 + VPADDD 176(SP), Y2, Y2 + VPADDD 368(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 240(SP), Y7, Y7 + VPADDD 272(SP), Y1, Y1 + VPADDD 208(SP), Y2, Y2 + VPADDD 304(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 6 + VMOVDQA (R10), Y7 + VPADDD 464(SP), Y7, Y7 + VPADDD 528(SP), Y1, Y1 + VPADDD 432(SP), Y2, Y2 + VPADDD 656(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 624(SP), Y7, Y7 + VPADDD 336(SP), Y1, Y1 + VPADDD 560(SP), Y2, Y2 + VPADDD 208(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 592(SP), Y7, Y7 + VPADDD 176(SP), Y1, Y1 + VPADDD 240(SP), Y2, Y2 + VPADDD 304(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 272(SP), Y7, Y7 + VPADDD 496(SP), Y1, Y1 + VPADDD 368(SP), Y2, Y2 + VPADDD 400(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Round 7 + VMOVDQA (R10), Y7 + VPADDD 528(SP), Y7, Y7 + VPADDD 336(SP), Y1, Y1 + VPADDD 208(SP), Y2, Y2 + VPADDD 432(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 656(SP), Y7, Y7 + VPADDD 176(SP), Y1, Y1 + VPADDD 464(SP), Y2, Y2 + VPADDD 368(SP), Y3, Y3 + VPADDD Y0, Y7, Y7 + VPXOR Y7, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y5, Y2, Y2 + VPXOR Y2, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y6, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y14, Y10, Y10 + VPXOR Y10, Y0, Y0 + VPADDD Y15, Y11, Y11 + VPXOR Y11, Y4, Y4 + VPADDD Y8, Y12, Y12 + VPXOR Y12, Y5, Y5 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y6, Y6 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VMOVDQA (R10), Y7 + VPADDD 624(SP), Y7, Y7 + VPADDD 240(SP), Y1, Y1 + VPADDD 272(SP), Y2, Y2 + VPADDD 400(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x0c, Y4, Y7 + VPSLLD $0x14, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x0c, Y5, Y7 + VPSLLD $0x14, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x0c, Y6, Y7 + VPSLLD $0x14, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x0c, Y0, Y7 + VPSLLD $0x14, Y0, Y0 + VPOR Y7, Y0, Y0 + VMOVDQA (R10), Y7 + VPADDD 496(SP), Y7, Y7 + VPADDD 560(SP), Y1, Y1 + VPADDD 304(SP), Y2, Y2 + VPADDD 592(SP), Y3, Y3 + VPADDD Y4, Y7, Y7 + VPXOR Y7, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y5, Y1, Y1 + VPXOR Y1, Y14, Y14 + VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y15, Y15 + VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y8, Y8 + VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 + VPADDD Y9, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPADDD Y14, Y13, Y13 + VPXOR Y13, Y5, Y5 + VPADDD Y15, Y10, Y10 + VPXOR Y10, Y6, Y6 + VPADDD Y8, Y11, Y11 + VPXOR Y11, Y0, Y0 + VMOVDQA Y7, (R10) + VPSRLD $0x07, Y4, Y7 + VPSLLD $0x19, Y4, Y4 + VPOR Y7, Y4, Y4 + VPSRLD $0x07, Y5, Y7 + VPSLLD $0x19, Y5, Y5 + VPOR Y7, Y5, Y5 + VPSRLD $0x07, Y6, Y7 + VPSLLD $0x19, Y6, Y6 + VPOR Y7, Y6, Y6 + VPSRLD $0x07, Y0, Y7 + VPSLLD $0x19, Y0, Y0 + VPOR Y7, Y0, Y0 + + // Finalize rounds + VPXOR Y9, Y6, Y6 + VPXOR (R10), Y10, Y7 + VPXOR Y11, Y1, Y1 + VPXOR Y12, Y2, Y2 + VPXOR Y13, Y3, Y3 + VPXOR Y14, Y0, Y0 + VPXOR Y15, Y4, Y4 + VPXOR Y8, Y5, Y5 + + // Fix up registers for next iteration + VMOVDQU Y7, Y8 + VMOVDQU Y6, Y7 + VMOVDQU Y5, Y6 + VMOVDQU Y4, Y5 + VMOVDQU Y0, Y4 + VMOVDQU Y8, Y0 + + // If we have zero complete chunks, we're done + CMPQ R8, $0x00 + JNE loop_trailer + CMPQ R9, CX + JEQ finalize + +loop_trailer: + // Increment, reset flags, and loop + CMPQ CX, $0x000003c0 + JEQ finalize + ADDQ $0x40, CX + MOVL BX, 64(SP) + JMP loop + +finalize: + // Store result into out + VMOVDQU Y0, (SI) + VMOVDQU Y1, 32(SI) + VMOVDQU Y2, 64(SI) + VMOVDQU Y3, 96(SI) + VMOVDQU Y4, 128(SI) + VMOVDQU Y5, 160(SI) + VMOVDQU Y6, 192(SI) + VMOVDQU Y7, 224(SI) + VZEROUPPER + RET + +// func HashP(left *[32]uint32, right *[32]uint32, flags uint8, key *[8]uint32, out *[32]uint32, n int) +// Requires: AVX, AVX2 +TEXT ·HashP(SB), NOSPLIT, $72-48 + MOVQ left+0(FP), AX + MOVQ right+8(FP), CX + MOVBLZX flags+16(FP), DX + MOVQ key+24(FP), BX + MOVQ out+32(FP), BP + + // Allocate local space and align it + LEAQ 31(SP), SI + MOVQ $0x000000000000001f, DI + NOTQ DI + ANDQ DI, SI + + // Set up flags value + MOVL DX, 64(SP) + + // Perform the rounds + // Round 1 + VPBROADCASTD (BX), Y0 + VPADDD (AX), Y0, Y0 + VPBROADCASTD 4(BX), Y1 + VPADDD 64(AX), Y1, Y1 + VPBROADCASTD 8(BX), Y2 + VPADDD 128(AX), Y2, Y2 + VPBROADCASTD 12(BX), Y3 + VPADDD 192(AX), Y3, Y3 + VPBROADCASTD 16(BX), Y4 + VPADDD Y4, Y0, Y0 + VMOVDQU zero<>+0(SB), Y5 + VPXOR Y0, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPBROADCASTD 20(BX), Y6 + VPADDD Y6, Y1, Y1 + VMOVDQU zero<>+0(SB), Y7 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPBROADCASTD 24(BX), Y8 + VPADDD Y8, Y2, Y2 + VMOVDQU block_len<>+0(SB), Y9 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPBROADCASTD 28(BX), Y10 + VPADDD Y10, Y3, Y3 + VPBROADCASTD 64(SP), Y11 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPBROADCASTD iv<>+0(SB), Y12 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y4, Y4 + VPBROADCASTD iv<>+4(SB), Y13 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y6, Y6 + VPBROADCASTD iv<>+8(SB), Y14 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y8, Y8 + VPBROADCASTD iv<>+12(SB), Y15 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y10, Y10 + VMOVDQA Y0, (SI) + VPSRLD $0x0c, Y4, Y0 + VPSLLD $0x14, Y4, Y4 + VPOR Y0, Y4, Y0 + VPSRLD $0x0c, Y6, Y4 + VPSLLD $0x14, Y6, Y6 + VPOR Y4, Y6, Y4 + VPSRLD $0x0c, Y8, Y6 + VPSLLD $0x14, Y8, Y8 + VPOR Y6, Y8, Y6 + VPSRLD $0x0c, Y10, Y8 + VPSLLD $0x14, Y10, Y10 + VPOR Y8, Y10, Y8 + VMOVDQA (SI), Y10 + VPADDD 32(AX), Y10, Y10 + VPADDD 96(AX), Y1, Y1 + VPADDD 160(AX), Y2, Y2 + VPADDD 224(AX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD (CX), Y10, Y10 + VPADDD 64(CX), Y1, Y1 + VPADDD 128(CX), Y2, Y2 + VPADDD 192(CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 32(CX), Y10, Y10 + VPADDD 96(CX), Y1, Y1 + VPADDD 160(CX), Y2, Y2 + VPADDD 224(CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 2 + VMOVDQA (SI), Y10 + VPADDD 64(AX), Y10, Y10 + VPADDD 96(AX), Y1, Y1 + VPADDD 224(AX), Y2, Y2 + VPADDD 128(AX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 192(AX), Y10, Y10 + VPADDD 64(CX), Y1, Y1 + VPADDD (AX), Y2, Y2 + VPADDD 160(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 32(AX), Y10, Y10 + VPADDD 128(CX), Y1, Y1 + VPADDD 32(CX), Y2, Y2 + VPADDD 224(CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 96(CX), Y10, Y10 + VPADDD 160(AX), Y1, Y1 + VPADDD 192(CX), Y2, Y2 + VPADDD (CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 3 + VMOVDQA (SI), Y10 + VPADDD 96(AX), Y10, Y10 + VPADDD 64(CX), Y1, Y1 + VPADDD 160(CX), Y2, Y2 + VPADDD 224(AX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 128(AX), Y10, Y10 + VPADDD 128(CX), Y1, Y1 + VPADDD 64(AX), Y2, Y2 + VPADDD 192(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 192(AX), Y10, Y10 + VPADDD 32(CX), Y1, Y1 + VPADDD 96(CX), Y2, Y2 + VPADDD (CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 160(AX), Y10, Y10 + VPADDD (AX), Y1, Y1 + VPADDD 224(CX), Y2, Y2 + VPADDD 32(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 4 + VMOVDQA (SI), Y10 + VPADDD 64(CX), Y10, Y10 + VPADDD 128(CX), Y1, Y1 + VPADDD 192(CX), Y2, Y2 + VPADDD 160(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 224(AX), Y10, Y10 + VPADDD 32(CX), Y1, Y1 + VPADDD 96(AX), Y2, Y2 + VPADDD 224(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 128(AX), Y10, Y10 + VPADDD 96(CX), Y1, Y1 + VPADDD 160(AX), Y2, Y2 + VPADDD 32(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD (AX), Y10, Y10 + VPADDD 64(AX), Y1, Y1 + VPADDD (CX), Y2, Y2 + VPADDD 192(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 5 + VMOVDQA (SI), Y10 + VPADDD 128(CX), Y10, Y10 + VPADDD 32(CX), Y1, Y1 + VPADDD 224(CX), Y2, Y2 + VPADDD 192(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 160(CX), Y10, Y10 + VPADDD 96(CX), Y1, Y1 + VPADDD 64(CX), Y2, Y2 + VPADDD (CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 224(AX), Y10, Y10 + VPADDD 160(AX), Y1, Y1 + VPADDD (AX), Y2, Y2 + VPADDD 192(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 64(AX), Y10, Y10 + VPADDD 96(AX), Y1, Y1 + VPADDD 32(AX), Y2, Y2 + VPADDD 128(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 6 + VMOVDQA (SI), Y10 + VPADDD 32(CX), Y10, Y10 + VPADDD 96(CX), Y1, Y1 + VPADDD (CX), Y2, Y2 + VPADDD 224(CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 192(CX), Y10, Y10 + VPADDD 160(AX), Y1, Y1 + VPADDD 128(CX), Y2, Y2 + VPADDD 32(AX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 160(CX), Y10, Y10 + VPADDD (AX), Y1, Y1 + VPADDD 64(AX), Y2, Y2 + VPADDD 128(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 96(AX), Y10, Y10 + VPADDD 64(CX), Y1, Y1 + VPADDD 192(AX), Y2, Y2 + VPADDD 224(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Round 7 + VMOVDQA (SI), Y10 + VPADDD 96(CX), Y10, Y10 + VPADDD 160(AX), Y1, Y1 + VPADDD 32(AX), Y2, Y2 + VPADDD (CX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 224(CX), Y10, Y10 + VPADDD (AX), Y1, Y1 + VPADDD 32(CX), Y2, Y2 + VPADDD 192(AX), Y3, Y3 + VPADDD Y0, Y10, Y10 + VPXOR Y10, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y4, Y1, Y1 + VPXOR Y1, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y6, Y2, Y2 + VPXOR Y2, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y8, Y3, Y3 + VPXOR Y3, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y5, Y12, Y12 + VPXOR Y12, Y0, Y0 + VPADDD Y7, Y13, Y13 + VPXOR Y13, Y4, Y4 + VPADDD Y9, Y14, Y14 + VPXOR Y14, Y6, Y6 + VPADDD Y11, Y15, Y15 + VPXOR Y15, Y8, Y8 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VMOVDQA (SI), Y10 + VPADDD 192(CX), Y10, Y10 + VPADDD 64(AX), Y1, Y1 + VPADDD 96(AX), Y2, Y2 + VPADDD 224(AX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x0c, Y4, Y10 + VPSLLD $0x14, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x0c, Y6, Y10 + VPSLLD $0x14, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x0c, Y8, Y10 + VPSLLD $0x14, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x0c, Y0, Y10 + VPSLLD $0x14, Y0, Y0 + VPOR Y10, Y0, Y0 + VMOVDQA (SI), Y10 + VPADDD 64(CX), Y10, Y10 + VPADDD 128(CX), Y1, Y1 + VPADDD 128(AX), Y2, Y2 + VPADDD 160(CX), Y3, Y3 + VPADDD Y4, Y10, Y10 + VPXOR Y10, Y11, Y11 + VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 + VPADDD Y6, Y1, Y1 + VPXOR Y1, Y5, Y5 + VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 + VPADDD Y8, Y2, Y2 + VPXOR Y2, Y7, Y7 + VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 + VPADDD Y0, Y3, Y3 + VPXOR Y3, Y9, Y9 + VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 + VPADDD Y11, Y14, Y14 + VPXOR Y14, Y4, Y4 + VPADDD Y5, Y15, Y15 + VPXOR Y15, Y6, Y6 + VPADDD Y7, Y12, Y12 + VPXOR Y12, Y8, Y8 + VPADDD Y9, Y13, Y13 + VPXOR Y13, Y0, Y0 + VMOVDQA Y10, (SI) + VPSRLD $0x07, Y4, Y10 + VPSLLD $0x19, Y4, Y4 + VPOR Y10, Y4, Y4 + VPSRLD $0x07, Y6, Y10 + VPSLLD $0x19, Y6, Y6 + VPOR Y10, Y6, Y6 + VPSRLD $0x07, Y8, Y10 + VPSLLD $0x19, Y8, Y8 + VPOR Y10, Y8, Y8 + VPSRLD $0x07, Y0, Y10 + VPSLLD $0x19, Y0, Y0 + VPOR Y10, Y0, Y0 + + // Finalize + VPXOR (SI), Y12, Y10 + VPXOR Y13, Y1, Y1 + VPXOR Y14, Y2, Y2 + VPXOR Y15, Y3, Y3 + VPXOR Y5, Y0, Y0 + VPXOR Y7, Y4, Y4 + VPXOR Y9, Y6, Y5 + VPXOR Y11, Y8, Y6 + + // Store result into out + VMOVDQU Y10, (BP) + VMOVDQU Y1, 32(BP) + VMOVDQU Y2, 64(BP) + VMOVDQU Y3, 96(BP) + VMOVDQU Y0, 128(BP) + VMOVDQU Y4, 160(BP) + VMOVDQU Y5, 192(BP) + VMOVDQU Y6, 224(BP) + VZEROUPPER + RET diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go new file mode 100644 index 000000000..613972814 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go @@ -0,0 +1,13 @@ +// +build !amd64 + +package hash_avx2 + +import "github.com/zeebo/blake3/internal/alg/hash/hash_pure" + +func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { + hash_pure.HashF(input, length, counter, flags, key, out, chain) +} + +func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { + hash_pure.HashP(left, right, flags, key, out, n) +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go new file mode 100644 index 000000000..10e949550 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go @@ -0,0 +1,9 @@ +// +build amd64 + +package hash_avx2 + +//go:noescape +func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) + +//go:noescape +func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go new file mode 100644 index 000000000..0c6fd63cd --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go @@ -0,0 +1,56 @@ +package hash_pure + +import ( + "unsafe" + + "github.com/zeebo/blake3/internal/alg/compress" + "github.com/zeebo/blake3/internal/consts" + "github.com/zeebo/blake3/internal/utils" +) + +func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { + var tmp [16]uint32 + + for i := uint64(0); consts.ChunkLen*i < length && i < 8; i++ { + bchain := *key + bflags := flags | consts.Flag_ChunkStart + start := consts.ChunkLen * i + + for n := uint64(0); n < 16; n++ { + if n == 15 { + bflags |= consts.Flag_ChunkEnd + } + if start+64*n >= length { + break + } + if start+64+64*n >= length { + *chain = bchain + } + + var blockPtr *[16]uint32 + if consts.IsLittleEndian { + blockPtr = (*[16]uint32)(unsafe.Pointer(&input[consts.ChunkLen*i+consts.BlockLen*n])) + } else { + var block [16]uint32 + utils.BytesToWords((*[64]uint8)(unsafe.Pointer(&input[consts.ChunkLen*i+consts.BlockLen*n])), &block) + blockPtr = &block + } + + compress.Compress(&bchain, blockPtr, counter, consts.BlockLen, bflags, &tmp) + + bchain = *(*[8]uint32)(unsafe.Pointer(&tmp[0])) + bflags = flags + } + + out[i+0] = bchain[0] + out[i+8] = bchain[1] + out[i+16] = bchain[2] + out[i+24] = bchain[3] + out[i+32] = bchain[4] + out[i+40] = bchain[5] + out[i+48] = bchain[6] + out[i+56] = bchain[7] + + counter++ + } +} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go new file mode 100644 index 000000000..bee5d8dd0 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go @@ -0,0 +1,38 @@ +package hash_pure + +import "github.com/zeebo/blake3/internal/alg/compress" + +func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { + var tmp [16]uint32 + var block [16]uint32 + + for i := 0; i < n && i < 8; i++ { + block[0] = left[i+0] + block[1] = left[i+8] + block[2] = left[i+16] + block[3] = left[i+24] + block[4] = left[i+32] + block[5] = left[i+40] + block[6] = left[i+48] + block[7] = left[i+56] + block[8] = right[i+0] + block[9] = right[i+8] + block[10] = right[i+16] + block[11] = right[i+24] + block[12] = right[i+32] + block[13] = right[i+40] + block[14] = right[i+48] + block[15] = right[i+56] + + compress.Compress(key, &block, 0, 64, flags, &tmp) + + out[i+0] = tmp[0] + out[i+8] = tmp[1] + out[i+16] = tmp[2] + out[i+24] = tmp[3] + out[i+32] = tmp[4] + out[i+40] = tmp[5] + out[i+48] = tmp[6] + out[i+56] = tmp[7] + } +} diff --git a/vendor/github.com/zeebo/blake3/internal/consts/consts.go b/vendor/github.com/zeebo/blake3/internal/consts/consts.go new file mode 100644 index 000000000..89f08fe10 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/consts/consts.go @@ -0,0 +1,29 @@ +package consts + +var IV = [...]uint32{IV0, IV1, IV2, IV3, IV4, IV5, IV6, IV7} + +const ( + IV0 = 0x6A09E667 + IV1 = 0xBB67AE85 + IV2 = 0x3C6EF372 + IV3 = 0xA54FF53A + IV4 = 0x510E527F + IV5 = 0x9B05688C + IV6 = 0x1F83D9AB + IV7 = 0x5BE0CD19 +) + +const ( + Flag_ChunkStart uint32 = 1 << 0 + Flag_ChunkEnd uint32 = 1 << 1 + Flag_Parent uint32 = 1 << 2 + Flag_Root uint32 = 1 << 3 + Flag_Keyed uint32 = 1 << 4 + Flag_DeriveKeyContext uint32 = 1 << 5 + Flag_DeriveKeyMaterial uint32 = 1 << 6 +) + +const ( + BlockLen = 64 + ChunkLen = 1024 +) diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu.go new file mode 100644 index 000000000..1eebff943 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/consts/cpu.go @@ -0,0 +1,17 @@ +package consts + +import ( + "os" + + "golang.org/x/sys/cpu" +) + +var ( + HasAVX2 = cpu.X86.HasAVX2 && + os.Getenv("BLAKE3_DISABLE_AVX2") == "" && + os.Getenv("BLAKE3_PUREGO") == "" + + HasSSE41 = cpu.X86.HasSSE41 && + os.Getenv("BLAKE3_DISABLE_SSE41") == "" && + os.Getenv("BLAKE3_PUREGO") == "" +) diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go new file mode 100644 index 000000000..fb730464f --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go @@ -0,0 +1,5 @@ +// +build mips mips64 ppc64 s390x + +package consts + +const IsLittleEndian = false diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go new file mode 100644 index 000000000..1bae02a74 --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go @@ -0,0 +1,5 @@ +// +build amd64 386 arm arm64 mipsle mips64le ppc64le riscv64 wasm + +package consts + +const IsLittleEndian = true diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go new file mode 100644 index 000000000..5f7407a6a --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go @@ -0,0 +1,7 @@ +// +build !mips,!mips64,!ppc64,!s390x,!amd64,!386,!arm,!arm64,!mipsle,!mips64le,!ppc64le,!riscv64,!wasm + +package consts + +import "unsafe" + +var IsLittleEndian = *(*uint16)(unsafe.Pointer(&[2]byte{0, 1})) != 1 diff --git a/vendor/github.com/zeebo/blake3/internal/utils/utils.go b/vendor/github.com/zeebo/blake3/internal/utils/utils.go new file mode 100644 index 000000000..0b36f0f0f --- /dev/null +++ b/vendor/github.com/zeebo/blake3/internal/utils/utils.go @@ -0,0 +1,60 @@ +package utils + +import ( + "encoding/binary" + "unsafe" +) + +func SliceToArray32(bytes []byte) *[32]uint8 { return (*[32]uint8)(unsafe.Pointer(&bytes[0])) } +func SliceToArray64(bytes []byte) *[64]uint8 { return (*[64]uint8)(unsafe.Pointer(&bytes[0])) } + +func BytesToWords(bytes *[64]uint8, words *[16]uint32) { + words[0] = binary.LittleEndian.Uint32(bytes[0*4:]) + words[1] = binary.LittleEndian.Uint32(bytes[1*4:]) + words[2] = binary.LittleEndian.Uint32(bytes[2*4:]) + words[3] = binary.LittleEndian.Uint32(bytes[3*4:]) + words[4] = binary.LittleEndian.Uint32(bytes[4*4:]) + words[5] = binary.LittleEndian.Uint32(bytes[5*4:]) + words[6] = binary.LittleEndian.Uint32(bytes[6*4:]) + words[7] = binary.LittleEndian.Uint32(bytes[7*4:]) + words[8] = binary.LittleEndian.Uint32(bytes[8*4:]) + words[9] = binary.LittleEndian.Uint32(bytes[9*4:]) + words[10] = binary.LittleEndian.Uint32(bytes[10*4:]) + words[11] = binary.LittleEndian.Uint32(bytes[11*4:]) + words[12] = binary.LittleEndian.Uint32(bytes[12*4:]) + words[13] = binary.LittleEndian.Uint32(bytes[13*4:]) + words[14] = binary.LittleEndian.Uint32(bytes[14*4:]) + words[15] = binary.LittleEndian.Uint32(bytes[15*4:]) +} + +func WordsToBytes(words *[16]uint32, bytes []byte) { + bytes = bytes[:64] + binary.LittleEndian.PutUint32(bytes[0*4:1*4], words[0]) + binary.LittleEndian.PutUint32(bytes[1*4:2*4], words[1]) + binary.LittleEndian.PutUint32(bytes[2*4:3*4], words[2]) + binary.LittleEndian.PutUint32(bytes[3*4:4*4], words[3]) + binary.LittleEndian.PutUint32(bytes[4*4:5*4], words[4]) + binary.LittleEndian.PutUint32(bytes[5*4:6*4], words[5]) + binary.LittleEndian.PutUint32(bytes[6*4:7*4], words[6]) + binary.LittleEndian.PutUint32(bytes[7*4:8*4], words[7]) + binary.LittleEndian.PutUint32(bytes[8*4:9*4], words[8]) + binary.LittleEndian.PutUint32(bytes[9*4:10*4], words[9]) + binary.LittleEndian.PutUint32(bytes[10*4:11*4], words[10]) + binary.LittleEndian.PutUint32(bytes[11*4:12*4], words[11]) + binary.LittleEndian.PutUint32(bytes[12*4:13*4], words[12]) + binary.LittleEndian.PutUint32(bytes[13*4:14*4], words[13]) + binary.LittleEndian.PutUint32(bytes[14*4:15*4], words[14]) + binary.LittleEndian.PutUint32(bytes[15*4:16*4], words[15]) +} + +func KeyFromBytes(key []byte, out *[8]uint32) { + key = key[:32] + out[0] = binary.LittleEndian.Uint32(key[0:]) + out[1] = binary.LittleEndian.Uint32(key[4:]) + out[2] = binary.LittleEndian.Uint32(key[8:]) + out[3] = binary.LittleEndian.Uint32(key[12:]) + out[4] = binary.LittleEndian.Uint32(key[16:]) + out[5] = binary.LittleEndian.Uint32(key[20:]) + out[6] = binary.LittleEndian.Uint32(key[24:]) + out[7] = binary.LittleEndian.Uint32(key[28:]) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c403b7d2d..8637feaec 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,18 +1,20 @@ # codeberg.org/gruf/go-bytes v1.0.2 ## explicit; go 1.14 codeberg.org/gruf/go-bytes -# codeberg.org/gruf/go-errors v1.0.4 +# codeberg.org/gruf/go-errors v1.0.5 ## explicit; go 1.15 codeberg.org/gruf/go-errors # codeberg.org/gruf/go-fastpath v1.0.2 ## explicit; go 1.14 codeberg.org/gruf/go-fastpath +# codeberg.org/gruf/go-format v1.0.3 +## explicit; go 1.17 +codeberg.org/gruf/go-format # codeberg.org/gruf/go-hashenc v1.0.1 ## explicit; go 1.16 codeberg.org/gruf/go-hashenc # codeberg.org/gruf/go-logger v1.3.2 ## explicit; go 1.14 -codeberg.org/gruf/go-logger # codeberg.org/gruf/go-mutexes v1.0.1 ## explicit; go 1.14 codeberg.org/gruf/go-mutexes @@ -25,7 +27,7 @@ codeberg.org/gruf/go-pools # codeberg.org/gruf/go-runners v1.2.0 ## explicit; go 1.14 codeberg.org/gruf/go-runners -# codeberg.org/gruf/go-store v1.1.5 +# codeberg.org/gruf/go-store v1.2.2 ## explicit; go 1.14 codeberg.org/gruf/go-store/kv codeberg.org/gruf/go-store/storage @@ -524,6 +526,18 @@ github.com/vmihailenco/tagparser/v2/internal/parser # github.com/wagslane/go-password-validator v0.3.0 ## explicit; go 1.16 github.com/wagslane/go-password-validator +# github.com/zeebo/blake3 v0.2.1 +## explicit; go 1.13 +github.com/zeebo/blake3 +github.com/zeebo/blake3/internal/alg +github.com/zeebo/blake3/internal/alg/compress +github.com/zeebo/blake3/internal/alg/compress/compress_pure +github.com/zeebo/blake3/internal/alg/compress/compress_sse41 +github.com/zeebo/blake3/internal/alg/hash +github.com/zeebo/blake3/internal/alg/hash/hash_avx2 +github.com/zeebo/blake3/internal/alg/hash/hash_pure +github.com/zeebo/blake3/internal/consts +github.com/zeebo/blake3/internal/utils # golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b ## explicit; go 1.17 golang.org/x/crypto/acme -- cgit v1.3 From 589bb9df0275457b5f9c3790e67517ec1be1745d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 16 Jan 2022 18:52:55 +0100 Subject: pass reader around instead of []byte --- internal/federation/dereferencing/account.go | 5 +- internal/federation/dereferencing/media.go | 3 +- internal/media/image.go | 46 ++++--- internal/media/manager_test.go | 32 +++-- internal/media/processingemoji.go | 168 +++++++++++-------------- internal/media/processingmedia.go | 179 ++++++++++++++------------- internal/media/types.go | 5 +- internal/media/util.go | 11 +- internal/processing/account/update.go | 42 +------ internal/processing/admin/emoji.go | 20 +-- internal/processing/media/create.go | 19 +-- internal/transport/derefmedia.go | 7 +- internal/transport/transport.go | 5 +- testrig/storage.go | 82 +----------- 14 files changed, 238 insertions(+), 386 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index b9efbfa45..6ea8256d5 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -23,6 +23,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/url" "strings" @@ -251,7 +252,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) ([]byte, error) { + data := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, avatarIRI) } @@ -273,7 +274,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) ([]byte, error) { + data := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, headerIRI) } diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 46cb4a5f4..c427f2507 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -21,6 +21,7 @@ package dereferencing import ( "context" "fmt" + "io" "net/url" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -41,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) ([]byte, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/media/image.go b/internal/media/image.go index de4b71210..b8f00024f 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -26,6 +26,7 @@ import ( "image/gif" "image/jpeg" "image/png" + "io" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" @@ -37,17 +38,17 @@ const ( thumbnailMaxHeight = 512 ) -type ImageMeta struct { - image []byte +type imageMeta struct { width int height int size int aspect float64 - blurhash string + blurhash string // defined only for calls to deriveThumbnail if createBlurhash is true + small []byte // defined only for calls to deriveStaticEmoji or deriveThumbnail } -func decodeGif(b []byte) (*ImageMeta, error) { - gif, err := gif.DecodeAll(bytes.NewReader(b)) +func decodeGif(r io.Reader) (*imageMeta, error) { + gif, err := gif.DecodeAll(r) if err != nil { return nil, err } @@ -58,8 +59,7 @@ func decodeGif(b []byte) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &ImageMeta{ - image: b, + return &imageMeta{ width: width, height: height, size: size, @@ -67,15 +67,15 @@ func decodeGif(b []byte) (*ImageMeta, error) { }, nil } -func decodeImage(b []byte, contentType string) (*ImageMeta, error) { +func decodeImage(r io.Reader, contentType string) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) + i, err = jpeg.Decode(r) case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) default: err = fmt.Errorf("content type %s not recognised", contentType) } @@ -93,8 +93,7 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { size := width * height aspect := float64(width) / float64(height) - return &ImageMeta{ - image: b, + return &imageMeta{ width: width, height: height, size: size, @@ -111,17 +110,17 @@ func decodeImage(b []byte, contentType string) (*ImageMeta, error) { // // If createBlurhash is false, then the blurhash field on the returned ImageAndMeta // will be an empty string. -func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageMeta, error) { +func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImageJpeg: - i, err = jpeg.Decode(bytes.NewReader(b)) + i, err = jpeg.Decode(r) case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) + i, err = gif.Decode(r) default: err = fmt.Errorf("content type %s can't be thumbnailed", contentType) } @@ -140,7 +139,7 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM size := width * height aspect := float64(width) / float64(height) - im := &ImageMeta{ + im := &imageMeta{ width: width, height: height, size: size, @@ -165,25 +164,24 @@ func deriveThumbnail(b []byte, contentType string, createBlurhash bool) (*ImageM }); err != nil { return nil, err } - - im.image = out.Bytes() + im.small = out.Bytes() return im, nil } // deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(b []byte, contentType string) (*ImageMeta, error) { +func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) { var i image.Image var err error switch contentType { case mimeImagePng: - i, err = png.Decode(bytes.NewReader(b)) + i, err = png.Decode(r) if err != nil { return nil, err } case mimeImageGif: - i, err = gif.Decode(bytes.NewReader(b)) + i, err = gif.Decode(r) if err != nil { return nil, err } @@ -195,8 +193,8 @@ func deriveStaticEmoji(b []byte, contentType string) (*ImageMeta, error) { if err := png.Encode(out, i); err != nil { return nil, err } - return &ImageMeta{ - image: out.Bytes(), + return &imageMeta{ + small: out.Bytes(), }, nil } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 0fadceb37..5380b83b1 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -19,8 +19,10 @@ package media_test import ( + "bytes" "context" "fmt" + "io" "os" "testing" "time" @@ -37,9 +39,13 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) ([]byte, error) { + data := func(_ context.Context) (io.Reader, error) { // load bytes from a test image - return os.ReadFile("./test/test-jpeg.jpg") + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -103,9 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - data := func(_ context.Context) ([]byte, error) { + data := func(_ context.Context) (io.Reader, error) { // load bytes from a test image - return os.ReadFile("./test/test-jpeg.jpg") + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -175,16 +185,16 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { // in this test, we spam the manager queue with 50 new media requests, just to see how it holds up - ctx := context.Background() - // load bytes from a test image - testBytes, err := os.ReadFile("./test/test-jpeg.jpg") - suite.NoError(err) - suite.NotEmpty(testBytes) + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } - data := func(_ context.Context) ([]byte, error) { - return testBytes, nil + data := func(_ context.Context) (io.Reader, error) { + // load bytes from a test image + return bytes.NewReader(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 467b500fc..147b6b5b3 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -19,8 +19,10 @@ package media import ( + "bytes" "context" "fmt" + "io" "strings" "sync" "time" @@ -46,22 +48,14 @@ type ProcessingEmoji struct { emoji *gtsmodel.Emoji data DataFunc - - rawData []byte // will be set once the fetchRawData function has been called + read bool // bool indicating that data function has been triggered already /* - below fields represent the processing state of the static version of the emoji - */ - - staticState processState - static *ImageMeta - - /* - below fields represent the processing state of the emoji image + below fields represent the processing state of the static of the emoji */ + staticState processState fullSizeState processState - fullSize *ImageMeta /* below pointers to database and storage are maintained so that @@ -85,21 +79,18 @@ func (p *ProcessingEmoji) EmojiID() string { // LoadEmoji blocks until the static and fullsize image // has been processed, and then returns the completed emoji. func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - if err := p.fetchRawData(ctx); err != nil { - return nil, err - } + p.mu.Lock() + defer p.mu.Unlock() - if _, err := p.loadStatic(ctx); err != nil { + if err := p.store(ctx); err != nil { return nil, err } - if _, err := p.loadFullSize(ctx); err != nil { + if err := p.loadStatic(ctx); err != nil { return nil, err } // store the result in the database before returning it - p.mu.Lock() - defer p.mu.Unlock() if !p.insertedInDB { if err := p.database.Put(ctx, p.emoji); err != nil { return nil, err @@ -116,118 +107,85 @@ func (p *ProcessingEmoji) Finished() bool { return p.staticState == complete && p.fullSizeState == complete } -func (p *ProcessingEmoji) loadStatic(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { switch p.staticState { case received: - // we haven't processed a static version of this emoji yet so do it now - static, err := deriveStaticEmoji(p.rawData, p.emoji.ImageContentType) + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.emoji.ImagePath) if err != nil { - p.err = fmt.Errorf("error deriving static: %s", err) + p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) p.staticState = errored - return nil, p.err + return p.err } - // put the static in storage - if err := p.storage.Put(p.emoji.ImageStaticPath, static.image); err != nil { - p.err = fmt.Errorf("error storing static: %s", err) + // we haven't processed a static version of this emoji yet so do it now + static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType) + if err != nil { + p.err = fmt.Errorf("loadStatic: error deriving static: %s", err) p.staticState = errored - return nil, p.err + return p.err } - // set appropriate fields on the emoji based on the static version we derived - p.emoji.ImageStaticFileSize = len(static.image) - - // set the static on the processing emoji - p.static = static - - // we're done processing the static version of the emoji! - p.staticState = complete - fallthrough - case complete: - return p.static, nil - case errored: - return nil, p.err - } - - return nil, fmt.Errorf("static processing status %d unknown", p.staticState) -} - -func (p *ProcessingEmoji) loadFullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - - switch p.fullSizeState { - case received: - var err error - var decoded *ImageMeta - - ct := p.emoji.ImageContentType - switch ct { - case mimeImagePng: - decoded, err = decodeImage(p.rawData, ct) - case mimeImageGif: - decoded, err = decodeGif(p.rawData) - default: - err = fmt.Errorf("content type %s not a processible emoji type", ct) - } - - if err != nil { - p.err = err - p.fullSizeState = errored - return nil, err + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadStatic: error closing stored full size: %s", err) + p.staticState = errored + return p.err } - // put the full size emoji in storage - if err := p.storage.Put(p.emoji.ImagePath, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size emoji: %s", err) - p.fullSizeState = errored - return nil, p.err + // put the static in storage + if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { + p.err = fmt.Errorf("loadStatic: error storing static: %s", err) + p.staticState = errored + return p.err } - // set the fullsize of this media - p.fullSize = decoded + p.emoji.ImageStaticFileSize = len(static.small) - // we're done processing the full-size emoji - p.fullSizeState = complete + // we're done processing the static version of the emoji! + p.staticState = complete fallthrough case complete: - return p.fullSize, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) + return fmt.Errorf("static processing status %d unknown", p.staticState) } -// fetchRawData calls the data function attached to p if it hasn't been called yet, -// and updates the underlying emoji fields as necessary. -// It should only be called from within a function that already has a lock on p! -func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { +// store calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. It will then stream +// bytes from p's reader directly into storage so that it can be retrieved later. +func (p *ProcessingEmoji) store(ctx context.Context) error { // check if we've already done this and bail early if we have - if p.rawData != nil { + if p.read { return nil } - // execute the data function and pin the raw bytes for further processing - b, err := p.data(ctx) + // execute the data function to get the reader out of it + reader, err := p.data(ctx) if err != nil { - return fmt.Errorf("fetchRawData: error executing data function: %s", err) + return fmt.Errorf("store: error executing data function: %s", err) + } + + // extract no more than 261 bytes from the beginning of the file -- this is the header + firstBytes := make([]byte, maxFileHeaderBytes) + if _, err := reader.Read(firstBytes); err != nil { + return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) } - p.rawData = b - // now we have the data we can work out the content type - contentType, err := parseContentType(p.rawData) + // now we have the file header we can work out the content type from it + contentType, err := parseContentType(firstBytes) if err != nil { - return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + return fmt.Errorf("store: error parsing content type: %s", err) } + // bail if this is a type we can't process if !supportedEmoji(contentType) { - return fmt.Errorf("fetchRawData: content type %s was not valid for an emoji", contentType) + return fmt.Errorf("store: content type %s was not valid for an emoji", contentType) } + // extract the file extension split := strings.Split(contentType, "/") extension := split[1] // something like 'gif' @@ -236,8 +194,24 @@ func (p *ProcessingEmoji) fetchRawData(ctx context.Context) error { p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) p.emoji.ImageContentType = contentType - p.emoji.ImageFileSize = len(p.rawData) + // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // store this for now -- other processes can pull it out of storage as they please + if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { + return fmt.Errorf("store: error storing stream: %s", err) + } + p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream + + // if the original reader is a readcloser, close it since we're done with it now + if rc, ok := reader.(io.ReadCloser); ok { + if err := rc.Close(); err != nil { + return fmt.Errorf("store: error closing readcloser: %s", err) + } + } + + p.read = true return nil } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 082c58607..82db863e0 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -19,8 +19,10 @@ package media import ( + "bytes" "context" "fmt" + "io" "strings" "sync" "time" @@ -44,22 +46,10 @@ type ProcessingMedia struct { attachment *gtsmodel.MediaAttachment data DataFunc + read bool // bool indicating that data function has been triggered already - rawData []byte // will be set once the fetchRawData function has been called - - /* - below fields represent the processing state of the media thumbnail - */ - - thumbstate processState - thumb *ImageMeta - - /* - below fields represent the processing state of the full-sized media - */ - - fullSizeState processState - fullSize *ImageMeta + thumbstate processState // the processing state of the media thumbnail + fullSizeState processState // the processing state of the full-sized media /* below pointers to database and storage are maintained so that @@ -83,21 +73,22 @@ func (p *ProcessingMedia) AttachmentID() string { // LoadAttachment blocks until the thumbnail and fullsize content // has been processed, and then returns the completed attachment. func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - if err := p.fetchRawData(ctx); err != nil { + p.mu.Lock() + defer p.mu.Unlock() + + if err := p.store(ctx); err != nil { return nil, err } - if _, err := p.loadThumb(ctx); err != nil { + if err := p.loadThumb(ctx); err != nil { return nil, err } - if _, err := p.loadFullSize(ctx); err != nil { + if err := p.loadFullSize(ctx); err != nil { return nil, err } // store the result in the database before returning it - p.mu.Lock() - defer p.mu.Unlock() if !p.insertedInDB { if err := p.database.Put(ctx, p.attachment); err != nil { return nil, err @@ -114,10 +105,7 @@ func (p *ProcessingMedia) Finished() bool { return p.thumbstate == complete && p.fullSizeState == complete } -func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingMedia) loadThumb(ctx context.Context) error { switch p.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now @@ -129,87 +117,94 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) (*ImageMeta, error) { createBlurhash = true } - thumb, err := deriveThumbnail(p.rawData, p.attachment.File.ContentType, createBlurhash) + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.attachment.File.Path) + if err != nil { + p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) + p.thumbstate = errored + return p.err + } + + // ... and into the derive thumbnail function + thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash) if err != nil { - p.err = fmt.Errorf("error deriving thumbnail: %s", err) + p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) + p.thumbstate = errored + return p.err + } + + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadThumb: error closing stored full size: %s", err) p.thumbstate = errored - return nil, p.err + return p.err } // put the thumbnail in storage - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.image); err != nil { - p.err = fmt.Errorf("error storing thumbnail: %s", err) + if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { + p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) p.thumbstate = errored - return nil, p.err + return p.err } // set appropriate fields on the attachment based on the thumbnail we derived if createBlurhash { p.attachment.Blurhash = thumb.blurhash } - p.attachment.FileMeta.Small = gtsmodel.Small{ Width: thumb.width, Height: thumb.height, Size: thumb.size, Aspect: thumb.aspect, } - p.attachment.Thumbnail.FileSize = len(thumb.image) - - // set the thumbnail of this media - p.thumb = thumb + p.attachment.Thumbnail.FileSize = len(thumb.small) // we're done processing the thumbnail! p.thumbstate = complete fallthrough case complete: - return p.thumb, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", p.thumbstate) + return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbstate) } -func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) { - p.mu.Lock() - defer p.mu.Unlock() - +func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { switch p.fullSizeState { case received: - var clean []byte var err error - var decoded *ImageMeta + var decoded *imageMeta + + // stream the original file out of storage... + stored, err := p.storage.GetStream(p.attachment.File.Path) + if err != nil { + p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) + p.fullSizeState = errored + return p.err + } + // decode the image ct := p.attachment.File.ContentType switch ct { case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(p.rawData); exifErr != nil { - err = exifErr - break - } - decoded, err = decodeImage(clean, ct) + decoded, err = decodeImage(stored, ct) case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = p.rawData - decoded, err = decodeGif(clean) + decoded, err = decodeGif(stored) default: - err = fmt.Errorf("content type %s not a processible image type", ct) + err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct) } if err != nil { p.err = err p.fullSizeState = errored - return nil, err + return p.err } - // put the full size in storage - if err := p.storage.Put(p.attachment.File.Path, decoded.image); err != nil { - p.err = fmt.Errorf("error storing full size image: %s", err) - p.fullSizeState = errored - return nil, p.err + if err := stored.Close(); err != nil { + p.err = fmt.Errorf("loadFullSize: error closing stored full size: %s", err) + p.thumbstate = errored + return p.err } // set appropriate fields on the attachment based on the image we derived @@ -219,56 +214,58 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) (*ImageMeta, error) Size: decoded.size, Aspect: decoded.aspect, } - p.attachment.File.FileSize = len(decoded.image) p.attachment.File.UpdatedAt = time.Now() p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - // set the fullsize of this media - p.fullSize = decoded - // we're done processing the full-size image p.fullSizeState = complete fallthrough case complete: - return p.fullSize, nil + return nil case errored: - return nil, p.err + return p.err } - return nil, fmt.Errorf("full size processing status %d unknown", p.fullSizeState) + return fmt.Errorf("loadFullSize: full size processing status %d unknown", p.fullSizeState) } -// fetchRawData calls the data function attached to p if it hasn't been called yet, -// and updates the underlying attachment fields as necessary. -// It should only be called from within a function that already has a lock on p! -func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { +// store calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. It will then stream +// bytes from p's reader directly into storage so that it can be retrieved later. +func (p *ProcessingMedia) store(ctx context.Context) error { // check if we've already done this and bail early if we have - if p.rawData != nil { + if p.read { return nil } - // execute the data function and pin the raw bytes for further processing - b, err := p.data(ctx) + // execute the data function to get the reader out of it + reader, err := p.data(ctx) if err != nil { - return fmt.Errorf("fetchRawData: error executing data function: %s", err) + return fmt.Errorf("store: error executing data function: %s", err) + } + + // extract no more than 261 bytes from the beginning of the file -- this is the header + firstBytes := make([]byte, maxFileHeaderBytes) + if _, err := reader.Read(firstBytes); err != nil { + return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) } - p.rawData = b - // now we have the data we can work out the content type - contentType, err := parseContentType(p.rawData) + // now we have the file header we can work out the content type from it + contentType, err := parseContentType(firstBytes) if err != nil { - return fmt.Errorf("fetchRawData: error parsing content type: %s", err) + return fmt.Errorf("store: error parsing content type: %s", err) } + // bail if this is a type we can't process if !supportedImage(contentType) { - return fmt.Errorf("fetchRawData: media type %s not (yet) supported", contentType) + return fmt.Errorf("store: media type %s not (yet) supported", contentType) } + // extract the file extension split := strings.Split(contentType, "/") if len(split) != 2 { - return fmt.Errorf("fetchRawData: content type %s was not valid", contentType) + return fmt.Errorf("store: content type %s was not valid", contentType) } - extension := split[1] // something like 'jpeg' // set some additional fields on the attachment now that @@ -282,6 +279,22 @@ func (p *ProcessingMedia) fetchRawData(ctx context.Context) error { p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) p.attachment.File.ContentType = contentType + // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // store this for now -- other processes can pull it out of storage as they please + if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil { + return fmt.Errorf("store: error storing stream: %s", err) + } + + // if the original reader is a readcloser, close it since we're done with it now + if rc, ok := reader.(io.ReadCloser); ok { + if err := rc.Close(); err != nil { + return fmt.Errorf("store: error closing readcloser: %s", err) + } + } + + p.read = true return nil } diff --git a/internal/media/types.go b/internal/media/types.go index 5b3fe4a41..0a7f60d66 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,6 +20,7 @@ package media import ( "context" + "io" "time" ) @@ -28,7 +29,7 @@ import ( // // See: https://en.wikipedia.org/wiki/File_format#File_header // and https://github.com/h2non/filetype -const maxFileHeaderBytes = 262 +const maxFileHeaderBytes = 261 // mime consts const ( @@ -117,4 +118,4 @@ type AdditionalEmojiInfo struct { } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. -type DataFunc func(ctx context.Context) ([]byte, error) +type DataFunc func(ctx context.Context) (io.Reader, error) diff --git a/internal/media/util.go b/internal/media/util.go index 7a3d81c0f..248d5fb19 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -19,7 +19,6 @@ package media import ( - "bytes" "errors" "fmt" @@ -28,11 +27,11 @@ import ( // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). // Returns an error if the content type is not something we can process. -func parseContentType(content []byte) (string, error) { - // read in the first bytes of the file - fileHeader := make([]byte, maxFileHeaderBytes) - if _, err := bytes.NewReader(content).Read(fileHeader); err != nil { - return "", fmt.Errorf("could not read first magic bytes of file: %s", err) +// +// Fileheader should be no longer than 262 bytes; anything more than this is inefficient. +func parseContentType(fileHeader []byte) (string, error) { + if fhLength := len(fileHeader); fhLength > maxFileHeaderBytes { + return "", fmt.Errorf("parseContentType requires %d bytes max, we got %d", maxFileHeaderBytes, fhLength) } kind, err := filetype.Match(fileHeader) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 7b305dc95..5a0a3e5a1 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -19,9 +19,7 @@ package account import ( - "bytes" "context" - "errors" "fmt" "io" "mime/multipart" @@ -142,24 +140,8 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(ctx context.Context) ([]byte, error) { - // pop open the fileheader - f, err := avatar.Open() - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateAvatar: could not read provided avatar: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateAvatar: could not read provided avatar: size 0 bytes") - } - - return buf.Bytes(), f.Close() + dataFunc := func(ctx context.Context) (io.Reader, error) { + return avatar.Open() } isAvatar := true @@ -184,24 +166,8 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(ctx context.Context) ([]byte, error) { - // pop open the fileheader - f, err := header.Open() - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - - // extract the bytes - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("UpdateHeader: could not read provided header: %s", err) - } - if size == 0 { - return nil, errors.New("UpdateHeader: could not read provided header: size 0 bytes") - } - - return buf.Bytes(), f.Close() + dataFunc := func(ctx context.Context) (io.Reader, error) { + return header.Open() } isHeader := true diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index fcc17c4be..e0068858b 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -19,9 +19,7 @@ package admin import ( - "bytes" "context" - "errors" "fmt" "io" @@ -38,22 +36,8 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } - data := func(innerCtx context.Context) ([]byte, error) { - // open the emoji and extract the bytes from it - f, err := form.Image.Open() - if err != nil { - return nil, fmt.Errorf("error opening emoji: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading emoji: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided emoji: size 0 bytes") - } - - return buf.Bytes(), f.Close() + data := func(innerCtx context.Context) (io.Reader, error) { + return form.Image.Open() } emojiID, err := id.NewRandomULID() diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 0896315b1..0fda4c27b 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -19,9 +19,7 @@ package media import ( - "bytes" "context" - "errors" "fmt" "io" @@ -31,21 +29,8 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - data := func(innerCtx context.Context) ([]byte, error) { - // open the attachment and extract the bytes from it - f, err := form.File.Open() - if err != nil { - return nil, fmt.Errorf("error opening attachment: %s", err) - } - buf := new(bytes.Buffer) - size, err := io.Copy(buf, f) - if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) - } - if size == 0 { - return nil, errors.New("could not read provided attachment: size 0 bytes") - } - return buf.Bytes(), f.Close() + data := func(innerCtx context.Context) (io.Reader, error) { + return form.File.Open() } focusX, focusY, err := parseFocus(form.Focus) diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index 3fa4a89e4..ed32f20c6 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -21,14 +21,14 @@ package transport import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "github.com/sirupsen/logrus" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) { l := logrus.WithField("func", "DereferenceMedia") l.Debugf("performing GET to %s", iri.String()) req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil) @@ -50,9 +50,8 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, if err != nil { return nil, err } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) } - return ioutil.ReadAll(resp.Body) + return resp.Body, nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index c43515a42..d9650d952 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -21,6 +21,7 @@ package transport import ( "context" "crypto" + "io" "net/url" "sync" @@ -33,8 +34,8 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the bytes of the given media attachment IRI. - DereferenceMedia(ctx context.Context, iri *url.URL) ([]byte, error) + // DereferenceMedia fetches the given media attachment IRI. + DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. diff --git a/testrig/storage.go b/testrig/storage.go index a8cf0d838..0e91d7dbe 100644 --- a/testrig/storage.go +++ b/testrig/storage.go @@ -19,20 +19,16 @@ package testrig import ( - "bytes" - "errors" "fmt" - "io" "os" "codeberg.org/gruf/go-store/kv" "codeberg.org/gruf/go-store/storage" - "codeberg.org/gruf/go-store/util" ) // NewTestStorage returns a new in memory storage with the default test config func NewTestStorage() *kv.KVStore { - storage, err := kv.OpenStorage(&inMemStorage{storage: map[string][]byte{}, overwrite: false}) + storage, err := kv.OpenStorage(storage.OpenMemory(200, false)) if err != nil { panic(err) } @@ -113,79 +109,3 @@ func StandardStorageTeardown(s *kv.KVStore) { } } } - -type inMemStorage struct { - storage map[string][]byte - overwrite bool -} - -func (s *inMemStorage) Clean() error { - return nil -} - -func (s *inMemStorage) ReadBytes(key string) ([]byte, error) { - b, ok := s.storage[key] - if !ok { - return nil, errors.New("key not found") - } - return b, nil -} - -func (s *inMemStorage) ReadStream(key string) (io.ReadCloser, error) { - b, err := s.ReadBytes(key) - if err != nil { - return nil, err - } - return util.NopReadCloser(bytes.NewReader(b)), nil -} - -func (s *inMemStorage) WriteBytes(key string, value []byte) error { - if _, ok := s.storage[key]; ok && !s.overwrite { - return errors.New("key already in storage") - } - s.storage[key] = copyBytes(value) - return nil -} - -func (s *inMemStorage) WriteStream(key string, r io.Reader) error { - b, err := io.ReadAll(r) - if err != nil { - return err - } - return s.WriteBytes(key, b) -} - -func (s *inMemStorage) Stat(key string) (bool, error) { - _, ok := s.storage[key] - return ok, nil -} - -func (s *inMemStorage) Remove(key string) error { - if _, ok := s.storage[key]; !ok { - return errors.New("key not found") - } - delete(s.storage, key) - return nil -} - -func (s *inMemStorage) WalkKeys(opts storage.WalkKeysOptions) error { - if opts.WalkFn == nil { - return errors.New("invalid walkfn") - } - for key := range s.storage { - opts.WalkFn(entry(key)) - } - return nil -} - -type entry string - -func (e entry) Key() string { - return string(e) -} - -func copyBytes(b []byte) []byte { - p := make([]byte, len(b)) - copy(p, b) - return p -} -- cgit v1.3 From 7d024ce74d29a14bc8db60495751e674bdb24463 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 23 Jan 2022 14:41:31 +0100 Subject: use exif-terminator --- README.md | 2 +- go.mod | 12 +- go.sum | 40 +- vendor/github.com/dsoprea/go-exif/.travis.yml | 24 - vendor/github.com/dsoprea/go-exif/LICENSE | 9 - vendor/github.com/dsoprea/go-exif/README.md | 206 --- vendor/github.com/dsoprea/go-exif/error.go | 10 - vendor/github.com/dsoprea/go-exif/exif.go | 247 --- vendor/github.com/dsoprea/go-exif/gps.go | 56 - vendor/github.com/dsoprea/go-exif/ifd.go | 407 ----- vendor/github.com/dsoprea/go-exif/ifd_builder.go | 1265 --------------- .../dsoprea/go-exif/ifd_builder_encode.go | 530 ------- vendor/github.com/dsoprea/go-exif/ifd_enumerate.go | 1356 ---------------- vendor/github.com/dsoprea/go-exif/ifd_tag_entry.go | 233 --- vendor/github.com/dsoprea/go-exif/package.go | 4 - vendor/github.com/dsoprea/go-exif/parser.go | 190 --- vendor/github.com/dsoprea/go-exif/tag_type.go | 397 ----- vendor/github.com/dsoprea/go-exif/tags.go | 229 --- vendor/github.com/dsoprea/go-exif/tags_data.go | 951 ----------- .../github.com/dsoprea/go-exif/tags_undefined.go | 417 ----- vendor/github.com/dsoprea/go-exif/type.go | 310 ---- vendor/github.com/dsoprea/go-exif/type_encode.go | 262 --- vendor/github.com/dsoprea/go-exif/utility.go | 222 --- vendor/github.com/dsoprea/go-exif/v2/.MODULE_ROOT | 0 vendor/github.com/dsoprea/go-exif/v2/LICENSE | 9 - vendor/github.com/dsoprea/go-exif/v2/common/ifd.go | 659 -------- .../github.com/dsoprea/go-exif/v2/common/parser.go | 219 --- .../dsoprea/go-exif/v2/common/testing_common.go | 88 -- .../github.com/dsoprea/go-exif/v2/common/type.go | 452 ------ .../dsoprea/go-exif/v2/common/utility.go | 79 - .../dsoprea/go-exif/v2/common/value_context.go | 412 ----- .../dsoprea/go-exif/v2/common/value_encoder.go | 229 --- vendor/github.com/dsoprea/go-exif/v2/error.go | 14 - vendor/github.com/dsoprea/go-exif/v2/exif.go | 258 --- vendor/github.com/dsoprea/go-exif/v2/gps.go | 117 -- vendor/github.com/dsoprea/go-exif/v2/ifd.go | 34 - .../github.com/dsoprea/go-exif/v2/ifd_builder.go | 1199 -------------- .../dsoprea/go-exif/v2/ifd_builder_encode.go | 532 ------- .../github.com/dsoprea/go-exif/v2/ifd_enumerate.go | 1521 ------------------ .../github.com/dsoprea/go-exif/v2/ifd_tag_entry.go | 297 ---- vendor/github.com/dsoprea/go-exif/v2/package.go | 8 - vendor/github.com/dsoprea/go-exif/v2/tags.go | 411 ----- vendor/github.com/dsoprea/go-exif/v2/tags_data.go | 929 ----------- .../dsoprea/go-exif/v2/testing_common.go | 182 --- .../dsoprea/go-exif/v2/undefined/README.md | 4 - .../dsoprea/go-exif/v2/undefined/accessor.go | 62 - .../dsoprea/go-exif/v2/undefined/exif_8828_oecf.go | 148 -- .../go-exif/v2/undefined/exif_9000_exif_version.go | 69 - .../exif_9101_components_configuration.go | 124 -- .../go-exif/v2/undefined/exif_927C_maker_note.go | 114 -- .../go-exif/v2/undefined/exif_9286_user_comment.go | 142 -- .../v2/undefined/exif_A000_flashpix_version.go | 69 - .../exif_A20C_spatial_frequency_response.go | 160 -- .../go-exif/v2/undefined/exif_A300_file_source.go | 79 - .../go-exif/v2/undefined/exif_A301_scene_type.go | 76 - .../go-exif/v2/undefined/exif_A302_cfa_pattern.go | 97 -- .../v2/undefined/exif_iop_0002_interop_version.go | 69 - .../v2/undefined/gps_001B_gps_processing_method.go | 65 - .../v2/undefined/gps_001C_gps_area_information.go | 65 - .../dsoprea/go-exif/v2/undefined/registration.go | 42 - .../dsoprea/go-exif/v2/undefined/type.go | 44 - vendor/github.com/dsoprea/go-exif/v2/utility.go | 233 --- vendor/github.com/dsoprea/go-exif/v3/.MODULE_ROOT | 0 vendor/github.com/dsoprea/go-exif/v3/LICENSE | 9 + vendor/github.com/dsoprea/go-exif/v3/common/ifd.go | 651 ++++++++ .../github.com/dsoprea/go-exif/v3/common/parser.go | 280 ++++ .../dsoprea/go-exif/v3/common/testing_common.go | 88 ++ .../github.com/dsoprea/go-exif/v3/common/type.go | 482 ++++++ .../dsoprea/go-exif/v3/common/utility.go | 148 ++ .../dsoprea/go-exif/v3/common/value_context.go | 464 ++++++ .../dsoprea/go-exif/v3/common/value_encoder.go | 273 ++++ vendor/github.com/dsoprea/go-exif/v3/data_layer.go | 50 + vendor/github.com/dsoprea/go-exif/v3/error.go | 14 + vendor/github.com/dsoprea/go-exif/v3/exif.go | 333 ++++ vendor/github.com/dsoprea/go-exif/v3/gps.go | 117 ++ .../github.com/dsoprea/go-exif/v3/ifd_builder.go | 1199 ++++++++++++++ .../dsoprea/go-exif/v3/ifd_builder_encode.go | 532 +++++++ .../github.com/dsoprea/go-exif/v3/ifd_enumerate.go | 1672 ++++++++++++++++++++ .../github.com/dsoprea/go-exif/v3/ifd_tag_entry.go | 298 ++++ vendor/github.com/dsoprea/go-exif/v3/package.go | 8 + vendor/github.com/dsoprea/go-exif/v3/tags.go | 475 ++++++ vendor/github.com/dsoprea/go-exif/v3/tags_data.go | 968 ++++++++++++ .../dsoprea/go-exif/v3/testing_common.go | 188 +++ .../dsoprea/go-exif/v3/undefined/README.md | 4 + .../dsoprea/go-exif/v3/undefined/accessor.go | 62 + .../dsoprea/go-exif/v3/undefined/exif_8828_oecf.go | 148 ++ .../go-exif/v3/undefined/exif_9000_exif_version.go | 69 + .../exif_9101_components_configuration.go | 124 ++ .../go-exif/v3/undefined/exif_927C_maker_note.go | 114 ++ .../go-exif/v3/undefined/exif_9286_user_comment.go | 142 ++ .../v3/undefined/exif_A000_flashpix_version.go | 69 + .../exif_A20C_spatial_frequency_response.go | 160 ++ .../go-exif/v3/undefined/exif_A300_file_source.go | 79 + .../go-exif/v3/undefined/exif_A301_scene_type.go | 76 + .../go-exif/v3/undefined/exif_A302_cfa_pattern.go | 97 ++ .../v3/undefined/exif_iop_0002_interop_version.go | 69 + .../v3/undefined/gps_001B_gps_processing_method.go | 65 + .../v3/undefined/gps_001C_gps_area_information.go | 65 + .../dsoprea/go-exif/v3/undefined/registration.go | 42 + .../dsoprea/go-exif/v3/undefined/type.go | 44 + vendor/github.com/dsoprea/go-exif/v3/utility.go | 237 +++ vendor/github.com/dsoprea/go-exif/value_context.go | 367 ----- .../dsoprea/go-jpeg-image-structure/.MODULE_ROOT | 0 .../dsoprea/go-jpeg-image-structure/.travis.yml | 21 - .../dsoprea/go-jpeg-image-structure/LICENSE | 9 - .../dsoprea/go-jpeg-image-structure/README.md | 10 - .../dsoprea/go-jpeg-image-structure/markers.go | 212 --- .../go-jpeg-image-structure/media_parser.go | 128 -- .../dsoprea/go-jpeg-image-structure/segment.go | 349 ---- .../go-jpeg-image-structure/segment_list.go | 395 ----- .../dsoprea/go-jpeg-image-structure/splitter.go | 437 ----- .../go-jpeg-image-structure/testing_common.go | 73 - .../dsoprea/go-jpeg-image-structure/utility.go | 110 -- .../go-jpeg-image-structure/v2/.MODULE_ROOT | 0 .../dsoprea/go-jpeg-image-structure/v2/LICENSE | 9 + .../dsoprea/go-jpeg-image-structure/v2/README.md | 10 + .../dsoprea/go-jpeg-image-structure/v2/markers.go | 212 +++ .../go-jpeg-image-structure/v2/media_parser.go | 139 ++ .../dsoprea/go-jpeg-image-structure/v2/segment.go | 352 +++++ .../go-jpeg-image-structure/v2/segment_list.go | 416 +++++ .../dsoprea/go-jpeg-image-structure/v2/splitter.go | 437 +++++ .../go-jpeg-image-structure/v2/testing_common.go | 73 + .../dsoprea/go-jpeg-image-structure/v2/utility.go | 110 ++ .../dsoprea/go-png-image-structure/.MODULE_ROOT | 0 .../dsoprea/go-png-image-structure/.travis.yml | 21 - .../dsoprea/go-png-image-structure/LICENSE | 9 - .../dsoprea/go-png-image-structure/README.md | 8 - .../go-png-image-structure/chunk_decoder.go | 89 -- .../dsoprea/go-png-image-structure/media_parser.go | 106 -- .../dsoprea/go-png-image-structure/png.go | 414 ----- .../go-png-image-structure/testing_common.go | 64 - .../dsoprea/go-png-image-structure/utility.go | 65 - .../dsoprea/go-png-image-structure/v2/.MODULE_ROOT | 0 .../dsoprea/go-png-image-structure/v2/LICENSE | 9 + .../go-png-image-structure/v2/chunk_decoder.go | 87 + .../go-png-image-structure/v2/media_parser.go | 118 ++ .../dsoprea/go-png-image-structure/v2/png.go | 416 +++++ .../go-png-image-structure/v2/testing_common.go | 64 + .../dsoprea/go-png-image-structure/v2/utility.go | 65 + vendor/github.com/dsoprea/go-utility/LICENSE | 7 - .../github.com/dsoprea/go-utility/image/README.md | 9 - .../dsoprea/go-utility/image/media_parser_type.go | 34 - vendor/github.com/dsoprea/go-utility/v2/LICENSE | 7 + .../dsoprea/go-utility/v2/filesystem/README.md | 64 + .../dsoprea/go-utility/v2/filesystem/bounceback.go | 273 ++++ .../v2/filesystem/boundedreadwriteseekcloser.go | 95 ++ .../v2/filesystem/boundedreadwriteseeker.go | 156 ++ .../go-utility/v2/filesystem/calculate_seek.go | 52 + .../dsoprea/go-utility/v2/filesystem/common.go | 15 + .../v2/filesystem/copy_bytes_between_positions.go | 40 + .../dsoprea/go-utility/v2/filesystem/does_exist.go | 19 + .../go-utility/v2/filesystem/graceful_copy.go | 54 + .../dsoprea/go-utility/v2/filesystem/list_files.go | 143 ++ .../go-utility/v2/filesystem/progress_wrapper.go | 93 ++ .../go-utility/v2/filesystem/read_counter.go | 36 + .../v2/filesystem/readseeker_to_readerat.go | 63 + .../v2/filesystem/readwriteseekcloser.go | 29 + .../go-utility/v2/filesystem/seekable_buffer.go | 146 ++ .../go-utility/v2/filesystem/simplefileinfo.go | 69 + .../dsoprea/go-utility/v2/filesystem/utility.go | 17 + .../go-utility/v2/filesystem/write_counter.go | 36 + .../dsoprea/go-utility/v2/image/README.md | 9 + .../go-utility/v2/image/media_parser_type.go | 34 + .../superseriousbusiness/exif-terminator/LICENSE | 661 ++++++++ .../superseriousbusiness/exif-terminator/README.md | 116 ++ .../superseriousbusiness/exif-terminator/jpeg.go | 138 ++ .../superseriousbusiness/exif-terminator/png.go | 93 ++ .../exif-terminator/terminator.go | 116 ++ .../superseriousbusiness/exifremove/LICENSE | 21 - .../exifremove/pkg/exifremove/exifremove.go | 140 -- .../exifremove/pkg/exifremove/png_crc_fix.go | 104 -- vendor/modules.txt | 38 +- 172 files changed, 15042 insertions(+), 19894 deletions(-) delete mode 100644 vendor/github.com/dsoprea/go-exif/.travis.yml delete mode 100644 vendor/github.com/dsoprea/go-exif/LICENSE delete mode 100644 vendor/github.com/dsoprea/go-exif/README.md delete mode 100644 vendor/github.com/dsoprea/go-exif/error.go delete mode 100644 vendor/github.com/dsoprea/go-exif/exif.go delete mode 100644 vendor/github.com/dsoprea/go-exif/gps.go delete mode 100644 vendor/github.com/dsoprea/go-exif/ifd.go delete mode 100644 vendor/github.com/dsoprea/go-exif/ifd_builder.go delete mode 100644 vendor/github.com/dsoprea/go-exif/ifd_builder_encode.go delete mode 100644 vendor/github.com/dsoprea/go-exif/ifd_enumerate.go delete mode 100644 vendor/github.com/dsoprea/go-exif/ifd_tag_entry.go delete mode 100644 vendor/github.com/dsoprea/go-exif/package.go delete mode 100644 vendor/github.com/dsoprea/go-exif/parser.go delete mode 100644 vendor/github.com/dsoprea/go-exif/tag_type.go delete mode 100644 vendor/github.com/dsoprea/go-exif/tags.go delete mode 100644 vendor/github.com/dsoprea/go-exif/tags_data.go delete mode 100644 vendor/github.com/dsoprea/go-exif/tags_undefined.go delete mode 100644 vendor/github.com/dsoprea/go-exif/type.go delete mode 100644 vendor/github.com/dsoprea/go-exif/type_encode.go delete mode 100644 vendor/github.com/dsoprea/go-exif/utility.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/.MODULE_ROOT delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/LICENSE delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/ifd.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/parser.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/testing_common.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/type.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/utility.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/value_context.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/common/value_encoder.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/error.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/exif.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/gps.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/ifd.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/ifd_builder.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/ifd_builder_encode.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/ifd_enumerate.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/ifd_tag_entry.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/package.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/tags.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/tags_data.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/testing_common.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/README.md delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/accessor.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_8828_oecf.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9000_exif_version.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9101_components_configuration.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_927C_maker_note.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9286_user_comment.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A000_flashpix_version.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A20C_spatial_frequency_response.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A300_file_source.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A301_scene_type.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A302_cfa_pattern.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/exif_iop_0002_interop_version.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001B_gps_processing_method.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001C_gps_area_information.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/registration.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/undefined/type.go delete mode 100644 vendor/github.com/dsoprea/go-exif/v2/utility.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/.MODULE_ROOT create mode 100644 vendor/github.com/dsoprea/go-exif/v3/LICENSE create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/ifd.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/parser.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/testing_common.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/type.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/utility.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/value_context.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/common/value_encoder.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/data_layer.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/error.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/exif.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/gps.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/ifd_builder.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/ifd_builder_encode.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/ifd_enumerate.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/ifd_tag_entry.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/package.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/tags.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/tags_data.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/testing_common.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/README.md create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/accessor.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_8828_oecf.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9000_exif_version.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9101_components_configuration.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_927C_maker_note.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9286_user_comment.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A000_flashpix_version.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A20C_spatial_frequency_response.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A300_file_source.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A301_scene_type.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A302_cfa_pattern.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/exif_iop_0002_interop_version.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001B_gps_processing_method.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001C_gps_area_information.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/registration.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/undefined/type.go create mode 100644 vendor/github.com/dsoprea/go-exif/v3/utility.go delete mode 100644 vendor/github.com/dsoprea/go-exif/value_context.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/.MODULE_ROOT delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/.travis.yml delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/LICENSE delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/README.md delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/markers.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/media_parser.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/segment.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/segment_list.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/splitter.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/testing_common.go delete mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/utility.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/.MODULE_ROOT create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/LICENSE create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/README.md create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/markers.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/media_parser.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment_list.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/splitter.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/testing_common.go create mode 100644 vendor/github.com/dsoprea/go-jpeg-image-structure/v2/utility.go delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/.MODULE_ROOT delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/.travis.yml delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/LICENSE delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/README.md delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/chunk_decoder.go delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/media_parser.go delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/png.go delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/testing_common.go delete mode 100644 vendor/github.com/dsoprea/go-png-image-structure/utility.go create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/.MODULE_ROOT create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/LICENSE create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/chunk_decoder.go create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/media_parser.go create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/png.go create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/testing_common.go create mode 100644 vendor/github.com/dsoprea/go-png-image-structure/v2/utility.go delete mode 100644 vendor/github.com/dsoprea/go-utility/LICENSE delete mode 100644 vendor/github.com/dsoprea/go-utility/image/README.md delete mode 100644 vendor/github.com/dsoprea/go-utility/image/media_parser_type.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/LICENSE create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go create mode 100644 vendor/github.com/dsoprea/go-utility/v2/image/README.md create mode 100644 vendor/github.com/dsoprea/go-utility/v2/image/media_parser_type.go create mode 100644 vendor/github.com/superseriousbusiness/exif-terminator/LICENSE create mode 100644 vendor/github.com/superseriousbusiness/exif-terminator/README.md create mode 100644 vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go create mode 100644 vendor/github.com/superseriousbusiness/exif-terminator/png.go create mode 100644 vendor/github.com/superseriousbusiness/exif-terminator/terminator.go delete mode 100644 vendor/github.com/superseriousbusiness/exifremove/LICENSE delete mode 100644 vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/exifremove.go delete mode 100644 vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/png_crc_fix.go diff --git a/README.md b/README.md index 1d5e5fb19..14aa6ecd2 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude - [spf13/pflag](https://github.com/spf13/pflag); command-line flag utilities. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). - [spf13/viper](https://github.com/spf13/viper); configuration management. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). - [stretchr/testify](https://github.com/stretchr/testify); test framework. [MIT License](https://spdx.org/licenses/MIT.html). -- [superseriousbusiness/exifremove](https://github.com/superseriousbusiness/exifremove) forked from [scottleedavis/go-exif-remove](https://github.com/scottleedavis/go-exif-remove); EXIF data removal. [MIT License](https://spdx.org/licenses/MIT.html). +- [superseriousbusiness/exif-terminator](https://github.com/superseriousbusiness/exif-terminator); EXIF data removal. [GNU AGPL v3 LICENSE](https://spdx.org/licenses/AGPL-3.0-or-later.html). - [superseriousbusiness/activity](https://github.com/superseriousbusiness/activity) forked from [go-fed/activity](https://github.com/go-fed/activity); Golang ActivityPub/ActivityStreams library. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). - [superseriousbusiness/oauth2](https://github.com/superseriousbusiness/oauth2) forked from [go-oauth2/oauth2](https://github.com/go-oauth2/oauth2); oauth server framework and token handling. [MIT License](https://spdx.org/licenses/MIT.html). - [go-swagger/go-swagger](https://github.com/go-swagger/go-swagger); Swagger OpenAPI spec generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). diff --git a/go.mod b/go.mod index 75956c7d6..b2ceb76aa 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/viper v1.10.0 github.com/stretchr/testify v1.7.0 github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8 - github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 + github.com/superseriousbusiness/exif-terminator v0.1.0 github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB github.com/tdewolff/minify/v2 v2.9.22 github.com/uptrace/bun v1.0.19 @@ -50,20 +50,18 @@ require ( codeberg.org/gruf/go-fastpath v1.0.2 // indirect codeberg.org/gruf/go-format v1.0.3 // indirect codeberg.org/gruf/go-hashenc v1.0.1 // indirect - codeberg.org/gruf/go-logger v1.3.2 // indirect codeberg.org/gruf/go-mutexes v1.0.1 // indirect codeberg.org/gruf/go-nowish v1.1.0 // indirect codeberg.org/gruf/go-pools v1.0.2 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b // indirect - github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b // indirect + github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b // indirect github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 // indirect - github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836 // indirect + github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 // indirect github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d // indirect - github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d // indirect - github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e // indirect + github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d // indirect + github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-errors/errors v1.4.1 // indirect diff --git a/go.sum b/go.sum index 082158f63..cc773bbfb 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,6 @@ codeberg.org/gruf/go-bytes v1.0.1/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9 codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo= codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg= codeberg.org/gruf/go-cache v1.1.2/go.mod h1:/Dbc+xU72Op3hMn6x2PXF3NE9uIDFeS+sXPF00hN/7o= -codeberg.org/gruf/go-errors v1.0.4 h1:jOJCn/GMb6ELLRVlnmpimGRC2CbTreH5/CBZNWh9GZA= -codeberg.org/gruf/go-errors v1.0.4/go.mod h1:rJ08LdIE79Jg8vZ2TGylz/I+tZ1UuMJkGK5mNambIfQ= codeberg.org/gruf/go-errors v1.0.5 h1:rxV70oQkfasUdggLHxOX2QAoJOMFM7XWxHQR45Zx/Fg= codeberg.org/gruf/go-errors v1.0.5/go.mod h1:n03EpmvcmfzU3/xJKC0XXtleXXJUNFpT2fgISODvZ1Y= codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI= @@ -62,13 +60,9 @@ codeberg.org/gruf/go-format v1.0.3 h1:WoUGzTwZe6SIhILNvtr0qNIA7BOOCgdBlk5bUrfeii codeberg.org/gruf/go-format v1.0.3/go.mod h1:k3TLXp1dqAXdDqxlon0yEM+3FFHdNn0D6BVJTwTy5As= codeberg.org/gruf/go-hashenc v1.0.1 h1:EBvNe2wW8IPMUqT1XihB6/IM6KMJDLMFBxIUvmsy1f8= codeberg.org/gruf/go-hashenc v1.0.1/go.mod h1:IfHhPCVScOiYmJLqdCQT9bYVS1nxNTV4ewMUvFWDPtc= -codeberg.org/gruf/go-logger v1.3.1/go.mod h1:tBduUc+Yb9vqGRxY9/FB0ZlYznSteLy/KmIANo7zFjA= -codeberg.org/gruf/go-logger v1.3.2 h1:/2Cg8Tmu6H10lljq/BvHE+76O2d4tDNUDwitN6YUxxk= -codeberg.org/gruf/go-logger v1.3.2/go.mod h1:q4xmTSdaxPzfndSXVF1X2xcyCVk7Nd/PIWCDs/4biMg= codeberg.org/gruf/go-mutexes v1.0.1 h1:X9bZW74YSEplWWdCrVXAvue5ztw3w5hh+INdXTENu88= codeberg.org/gruf/go-mutexes v1.0.1/go.mod h1:y2hbGLkWVHhNyxBOIVsA3/y2QMm6RSrYsC3sLVZ4EXM= codeberg.org/gruf/go-nowish v1.0.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= -codeberg.org/gruf/go-nowish v1.0.2/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= codeberg.org/gruf/go-nowish v1.1.0 h1:rj1z0AXDhLvnxs/DazWFxYAugs6rv5vhgWJkRCgrESg= codeberg.org/gruf/go-nowish v1.1.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= codeberg.org/gruf/go-pools v1.0.2 h1:B0X6yoCL9FVmnvyoizb1SYRwMYPWwEJBjPnBMM5ILos= @@ -76,8 +70,6 @@ codeberg.org/gruf/go-pools v1.0.2/go.mod h1:MjUV3H6IASyBeBPCyCr7wjPpSNu8E2N87LG4 codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBsH4= codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= -codeberg.org/gruf/go-store v1.1.5 h1:fp28vzGD15OsAF51CCwi7woH+Y3vb0aMl4OFh9JSjA0= -codeberg.org/gruf/go-store v1.1.5/go.mod h1:Q6ev500ddKghDQ8KS4IstL/W9fptDKa2T9oeHP+tXsI= codeberg.org/gruf/go-store v1.2.2 h1:YJPzJpZv/D3t9hQC00/u76eQDScQw4++OWjfobnjHAA= codeberg.org/gruf/go-store v1.2.2/go.mod h1:Xjw1U098th0yXF2CCx6jThQ+9FIPWAX9OGjYslO+UtE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -154,21 +146,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= -github.com/dsoprea/go-exif v0.0.0-20210131231135-d154f10435cc/go.mod h1:lOaOt7+UEppOgyvRy749v3do836U/hw0YVJNjoyPaEs= -github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b h1:hoVHc4m/v8Al8mbWyvKJWr4Z37yM4QUSVh/NY6A5Sbc= -github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b/go.mod h1:lOaOt7+UEppOgyvRy749v3do836U/hw0YVJNjoyPaEs= github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E= -github.com/dsoprea/go-exif/v2 v2.0.0-20200604193436-ca8584a0e1c4/go.mod h1:9EXlPeHfblFFnwu5UOqmP2eoZfJyAZ2Ri/Vki33ajO0= -github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b h1:8lVRnnni9zebcpjkrEXrEyxFpRWG/oTpWc2Y3giKomE= -github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b/go.mod h1:oKrjk2kb3rAR5NbtSTLUMvMSbc+k8ZosI3MaVH47noc= github.com/dsoprea/go-exif/v3 v3.0.0-20200717053412-08f1b6708903/go.mod h1:0nsO1ce0mh5czxGeLo4+OCZ/C6Eo6ZlMWsz7rH/Gxv8= -github.com/dsoprea/go-exif/v3 v3.0.0-20210512043655-120bcdb2a55e/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk= +github.com/dsoprea/go-exif/v3 v3.0.0-20210428042052-dca55bf8ca15/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk= +github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b h1:NgNuLvW/gAFKU30ULWW0gtkCt56JfB7FrZ2zyo0wT8I= +github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk= github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb/go.mod h1:kYIdx9N9NaOyD7U6D+YtExN7QhRm+5kq7//yOsRXQtM= github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 h1:YDRiMEm32T60Kpm35YzOK9ZHgjsS1Qrid+XskNcsdp8= github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413/go.mod h1:kYIdx9N9NaOyD7U6D+YtExN7QhRm+5kq7//yOsRXQtM= -github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210128210355-86b1014917f2/go.mod h1:ZoOP3yUG0HD1T4IUjIFsz/2OAB2yB4YX6NSm4K+uJRg= -github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836 h1:OHRfKIVRz2XrhZ6A7fJKHLoKky1giN+VUgU2npF0BvE= -github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836/go.mod h1:6+tQXZ+I62x13UZ+hemLVoZIuq/usVzvau7bqwUo9P0= +github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 h1:KGCiMMWxODEMmI3+9Ms04l73efoqFVNKKKPbVyOvKrU= +github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836/go.mod h1:WaARaUjQuSuDCDFAiU/GwzfxMTJBulfEhqEA2Tx6B4Y= github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA= github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8= github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg= @@ -176,13 +163,11 @@ github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd/go.mod h1:7I+3P github.com/dsoprea/go-photoshop-info-format v0.0.0-20200609050348-3db9b63b202c/go.mod h1:pqKB+ijp27cEcrHxhXVgUUMlSDRuGJJp1E+20Lj5H0E= github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d h1:dg6UMHa50VI01WuPWXPbNJpO8QSyvIF5T5n2IZiqX3A= github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d/go.mod h1:pqKB+ijp27cEcrHxhXVgUUMlSDRuGJJp1E+20Lj5H0E= -github.com/dsoprea/go-png-image-structure v0.0.0-20200807080309-a98d4e94ac82/go.mod h1:aDYQkL/5gfRNZkoxiLTSWU4Y8/gV/4MVsy/MU9uwTak= -github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d h1:8+qI8ant/vZkNSsbwSjIR6XJfWcDVTg/qx/3pRUUZNA= -github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d/go.mod h1:yTR3tKgyk20phAFg6IE9ulMA5NjEDD2wyx+okRFLVtw= -github.com/dsoprea/go-utility v0.0.0-20200512094054-1abbbc781176/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8= +github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d h1:2zNIgrJTspLxUKoJGl0Ln24+hufPKSjP3cu4++5MeSE= +github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d/go.mod h1:scnx0wQSM7UiCMK66dSdiPZvL2hl6iF5DvpZ7uT59MY= +github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf h1:/w4QxepU4AHh3AuO6/g8y/YIIHH5+aKP3Bj8sg5cqhU= github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8= -github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e h1:ojqYA1mU6LuRm8XzrVOvyfb000y59cbUcu6Wt8sFSAs= -github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e/go.mod h1:KVK+/Hul09ujXAGq+42UBgCTnXkiJZRnLYdURGjQUwo= +github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e h1:IxIbA7VbCNrwumIYjDoMOdf4KOSkMC6NJE4s8oRbE7E= github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -367,7 +352,6 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -666,8 +650,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8 h1:8Bwy6CSsT33/sF5FhjND4vr7jiJCaq4elNTAW4rUzVc= github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8/go.mod h1:ZY9xwFDucvp6zTvM6FQZGl8PSOofPBFIAy6gSc85XkY= -github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 h1:1SWXcTphBQjYGWRRxLFIAR1LVtQEj4eR7xPtyeOVM/c= -github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203/go.mod h1:0Xw5cYMOYpgaWs+OOSx41ugycl2qvKTi9tlMMcZhFyY= +github.com/superseriousbusiness/exif-terminator v0.1.0 h1:ePzfV0vcw+tm/haSOGzKbBTKkHAvyQLbCzfsdVkb3hM= +github.com/superseriousbusiness/exif-terminator v0.1.0/go.mod h1:pmlOKzkFZWmqaucLAtrRbZG0R5F3dbrcLWOcd7gAOLI= github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB h1:PtW2w6budTvRV2J5QAoSvThTHBuvh8t/+BXIZFAaBSc= github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB/go.mod h1:uYC/W92oVRJ49Vh1GcvTqpeFqHi+Ovrl2sMllQWRAEo= github.com/tdewolff/minify/v2 v2.9.22 h1:PlmaAakaJHdMMdTTwjjsuSwIxKqWPTlvjTj6a/g/ILU= @@ -740,9 +724,11 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.1 h1:O+N0Y8Re2XAYjp0adlZDA2juyRguhMfPCgh8YIf7vyE= github.com/zeebo/blake3 v0.2.1/go.mod h1:TSQ0KjMH+pht+bRyvVooJ1rBpvvngSGaPISafq9MxJk= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/vendor/github.com/dsoprea/go-exif/.travis.yml b/vendor/github.com/dsoprea/go-exif/.travis.yml deleted file mode 100644 index 162896e30..000000000 --- a/vendor/github.com/dsoprea/go-exif/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: go -go: - - master - - stable - - "1.14" - - "1.13" - - "1.12" -env: - - GO111MODULE=on -install: - - go get -t ./... -script: -# v1 - - go test -v . - - go test -v ./exif-read-tool -# v2 - - cd v2 - - go test -v ./... - - cd .. -# v3. Coverage reports comes from this. - - cd v3 - - go test -v ./... -coverprofile=coverage.txt -covermode=atomic -after_success: - - curl -s https://codecov.io/bash | bash diff --git a/vendor/github.com/dsoprea/go-exif/LICENSE b/vendor/github.com/dsoprea/go-exif/LICENSE deleted file mode 100644 index 0b9358a3a..000000000 --- a/vendor/github.com/dsoprea/go-exif/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT LICENSE - -Copyright 2019 Dustin Oprea - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-exif/README.md b/vendor/github.com/dsoprea/go-exif/README.md deleted file mode 100644 index 972d58493..000000000 --- a/vendor/github.com/dsoprea/go-exif/README.md +++ /dev/null @@ -1,206 +0,0 @@ -[![Build Status](https://travis-ci.org/dsoprea/go-exif.svg?branch=master)](https://travis-ci.org/dsoprea/go-exif) -[![codecov](https://codecov.io/gh/dsoprea/go-exif/branch/master/graph/badge.svg)](https://codecov.io/gh/dsoprea/go-exif) -[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-exif/v3)](https://goreportcard.com/report/github.com/dsoprea/go-exif/v3) -[![GoDoc](https://godoc.org/github.com/dsoprea/go-exif/v3?status.svg)](https://godoc.org/github.com/dsoprea/go-exif/v3) - -# Overview - -This package provides native Go functionality to parse an existing EXIF block, update an existing EXIF block, or add a new EXIF block. - - -# Getting - -To get the project and dependencies: - -``` -$ go get -t github.com/dsoprea/go-exif/v3 -``` - - -# Scope - -This project is concerned only with parsing and encoding raw EXIF data. It does -not understand specific file-formats. This package assumes you know how to -extract the raw EXIF data from a file, such as a JPEG, and, if you want to -update it, know how to write it back. File-specific formats are not the concern -of *go-exif*, though we provide -[exif.SearchAndExtractExif][search-and-extract-exif] and -[exif.SearchFileAndExtractExif][search-file-and-extract-exif] as brute-force -search mechanisms that will help you explore the EXIF information for newer -formats that you might not yet have any way to parse. - -That said, the author also provides the following projects to support the -efficient processing of the corresponding image formats: - -- [go-jpeg-image-structure](https://github.com/dsoprea/go-jpeg-image-structure) -- [go-png-image-structure](https://github.com/dsoprea/go-png-image-structure) -- [go-tiff-image-structure](https://github.com/dsoprea/go-tiff-image-structure) -- [go-heic-exif-extractor](https://github.com/dsoprea/go-heic-exif-extractor) - -See the [SetExif example in go-jpeg-image-structure][jpeg-set-exif] for -practical information on getting started with JPEG files. - - -# Usage - -The package provides a set of [working examples][examples] and is covered by -unit-tests. Please look to these for getting familiar with how to read and write -EXIF. - -Create an instance of the `Exif` type and call `Scan()` with a byte-slice, where -the first byte is the beginning of the raw EXIF data. You may pass a callback -that will be invoked for every tag or `nil` if you do not want one. If no -callback is given, you are effectively just validating the structure or parsing -of the image. - -Obviously, it is most efficient to properly parse the media file and then -provide the specific EXIF data to be parsed, but there is also a heuristic for -finding the EXIF data within the media blob, directly. This means that, at least -for testing or curiosity, **you do not have to parse or even understand the -format of image or audio file in order to find and decode the EXIF information -inside of it.** See the usage of the `SearchAndExtractExif` method in the -example. - -The library often refers to an IFD with an "IFD path" (e.g. IFD/Exif, -IFD/GPSInfo). A "fully-qualified" IFD-path is one that includes an index -describing which specific sibling IFD is being referred to if not the first one -(e.g. IFD1, the IFD where the thumbnail is expressed per the TIFF standard). - -There is an "IFD mapping" and a "tag index" that must be created and passed to -the library from the top. These contain all of the knowledge of the IFD -hierarchies and their tag-IDs (the IFD mapping) and the tags that they are -allowed to host (the tag index). There are convenience functions to load them -with the standard TIFF information, but you, alternatively, may choose -something totally different (to support parsing any kind of EXIF data that does -not follow or is not relevant to TIFF at all). - - -# Standards and Customization - -This project is configuration driven. By default, it has no knowledge of tags -and IDs until you load them prior to using (which is incorporated in the -examples). You are just as easily able to add additional custom IFDs and custom -tags for them. If desired, you could completely ignore the standard information -and load *totally* non-standard IFDs and tags. - -This would be useful for divergent implementations that add non-standard -information to images. It would also be useful if there is some need to just -store a flat list of tags in an image for simplified, proprietary usage. - - -# Reader Tool - -There is a runnable reading/dumping tool included: - -``` -$ go get github.com/dsoprea/go-exif/v3/command/exif-read-tool -$ exif-read-tool --filepath "" -``` - -Example output: - -``` -IFD-PATH=[IFD] ID=(0x010f) NAME=[Make] COUNT=(6) TYPE=[ASCII] VALUE=[Canon] -IFD-PATH=[IFD] ID=(0x0110) NAME=[Model] COUNT=(22) TYPE=[ASCII] VALUE=[Canon EOS 5D Mark III] -IFD-PATH=[IFD] ID=(0x0112) NAME=[Orientation] COUNT=(1) TYPE=[SHORT] VALUE=[1] -IFD-PATH=[IFD] ID=(0x011a) NAME=[XResolution] COUNT=(1) TYPE=[RATIONAL] VALUE=[72/1] -IFD-PATH=[IFD] ID=(0x011b) NAME=[YResolution] COUNT=(1) TYPE=[RATIONAL] VALUE=[72/1] -IFD-PATH=[IFD] ID=(0x0128) NAME=[ResolutionUnit] COUNT=(1) TYPE=[SHORT] VALUE=[2] -IFD-PATH=[IFD] ID=(0x0132) NAME=[DateTime] COUNT=(20) TYPE=[ASCII] VALUE=[2017:12:02 08:18:50] -... -``` - -You can also print the raw, parsed data as JSON: - -``` -$ exif-read-tool --filepath "" -json -``` - -Example output: - -``` -[ - { - "ifd_path": "IFD", - "fq_ifd_path": "IFD", - "ifd_index": 0, - "tag_id": 271, - "tag_name": "Make", - "tag_type_id": 2, - "tag_type_name": "ASCII", - "unit_count": 6, - "value": "Canon", - "value_string": "Canon" - }, - { - "ifd_path": "IFD", -... -``` - - -# Testing - -The traditional method: - -``` -$ go test github.com/dsoprea/go-exif/v3/... -``` - - -# Release Notes - -## v3 Release - -This release primarily introduces an interchangeable data-layer, where any -`io.ReadSeeker` can be used to read EXIF data rather than necessarily loading -the EXIF blob into memory first. - -Several backwards-incompatible clean-ups were also included in this release. See -[releases][releases] for more information. - -## v2 Release - -Features a heavily reflowed interface that makes usage much simpler. The -undefined-type tag-processing (which affects most photographic images) has also -been overhauled and streamlined. It is now complete and stable. Adoption is -strongly encouraged. - - -# *Contributing* - -EXIF has an excellently-documented structure but there are a lot of devices and -manufacturers out there. There are only so many files that we can personally -find to test against, and most of these are images that have been generated only -in the past few years. JPEG, being the largest implementor of EXIF, has been -around for even longer (but not much). Therefore, there is a lot of -compatibility to test for. - -**If you are able to help by running the included reader-tool against all of the -EXIF-compatible files you have, it would be deeply appreciated. This is mostly -going to be JPEG files (but not all variations). If you are able to test a large -number of files (thousands or millions) then please post an issue mentioning how -many files you have processed. If you had failures, then please share them and -try to support efforts to understand them.** - -If you are able to test 100K+ files, I will give you credit on the project. The -further back in time your images reach, the higher in the list your name/company -will go. - - -# Contributors/Testing - -Thank you to the following users for solving non-trivial issues, supporting the -project with solving edge-case problems in specific images, or otherwise -providing their non-trivial time or image corpus to test go-exif: - -- [philip-firstorder](https://github.com/philip-firstorder) (200K images) -- [matchstick](https://github.com/matchstick) (102K images) - -In addition to these, it has been tested on my own collection, north of 478K -images. - -[search-and-extract-exif]: https://godoc.org/github.com/dsoprea/go-exif/v3#SearchAndExtractExif -[search-file-and-extract-exif]: https://godoc.org/github.com/dsoprea/go-exif/v3#SearchFileAndExtractExif -[jpeg-set-exif]: https://godoc.org/github.com/dsoprea/go-jpeg-image-structure#example-SegmentList-SetExif -[examples]: https://godoc.org/github.com/dsoprea/go-exif/v3#pkg-examples -[releases]: https://github.com/dsoprea/go-exif/releases diff --git a/vendor/github.com/dsoprea/go-exif/error.go b/vendor/github.com/dsoprea/go-exif/error.go deleted file mode 100644 index 0e6e138cb..000000000 --- a/vendor/github.com/dsoprea/go-exif/error.go +++ /dev/null @@ -1,10 +0,0 @@ -package exif - -import ( - "errors" -) - -var ( - ErrTagNotFound = errors.New("tag not found") - ErrTagNotStandard = errors.New("tag not a standard tag") -) diff --git a/vendor/github.com/dsoprea/go-exif/exif.go b/vendor/github.com/dsoprea/go-exif/exif.go deleted file mode 100644 index 8d1b848f0..000000000 --- a/vendor/github.com/dsoprea/go-exif/exif.go +++ /dev/null @@ -1,247 +0,0 @@ -package exif - -import ( - "bytes" - "errors" - "fmt" - "os" - - "encoding/binary" - "io/ioutil" - - "github.com/dsoprea/go-logging" -) - -const ( - // ExifAddressableAreaStart is the absolute offset in the file that all - // offsets are relative to. - ExifAddressableAreaStart = uint32(0x0) - - // ExifDefaultFirstIfdOffset is essentially the number of bytes in addition - // to `ExifAddressableAreaStart` that you have to move in order to escape - // the rest of the header and get to the earliest point where we can put - // stuff (which has to be the first IFD). This is the size of the header - // sequence containing the two-character byte-order, two-character fixed- - // bytes, and the four bytes describing the first-IFD offset. - ExifDefaultFirstIfdOffset = uint32(2 + 2 + 4) -) - -var ( - exifLogger = log.NewLogger("exif.exif") - - // EncodeDefaultByteOrder is the default byte-order for encoding operations. - EncodeDefaultByteOrder = binary.BigEndian - - // Default byte order for tests. - TestDefaultByteOrder = binary.BigEndian - - BigEndianBoBytes = [2]byte{'M', 'M'} - LittleEndianBoBytes = [2]byte{'I', 'I'} - - ByteOrderLookup = map[[2]byte]binary.ByteOrder{ - BigEndianBoBytes: binary.BigEndian, - LittleEndianBoBytes: binary.LittleEndian, - } - - ByteOrderLookupR = map[binary.ByteOrder][2]byte{ - binary.BigEndian: BigEndianBoBytes, - binary.LittleEndian: LittleEndianBoBytes, - } - - ExifFixedBytesLookup = map[binary.ByteOrder][2]byte{ - binary.LittleEndian: {0x2a, 0x00}, - binary.BigEndian: {0x00, 0x2a}, - } -) - -var ( - ErrNoExif = errors.New("no exif data") - ErrExifHeaderError = errors.New("exif header error") -) - -// SearchAndExtractExif returns a slice from the beginning of the EXIF data to -// end of the file (it's not practical to try and calculate where the data -// actually ends; it needs to be formally parsed). -func SearchAndExtractExif(data []byte) (rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - // Search for the beginning of the EXIF information. The EXIF is near the - // beginning of our/most JPEGs, so this has a very low cost. - - foundAt := -1 - for i := 0; i < len(data); i++ { - if _, err := ParseExifHeader(data[i:]); err == nil { - foundAt = i - break - } else if log.Is(err, ErrNoExif) == false { - return nil, err - } - } - - if foundAt == -1 { - return nil, ErrNoExif - } - - return data[foundAt:], nil -} - -// SearchFileAndExtractExif returns a slice from the beginning of the EXIF data -// to the end of the file (it's not practical to try and calculate where the -// data actually ends). -func SearchFileAndExtractExif(filepath string) (rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - // Open the file. - - f, err := os.Open(filepath) - log.PanicIf(err) - - defer f.Close() - - data, err := ioutil.ReadAll(f) - log.PanicIf(err) - - rawExif, err = SearchAndExtractExif(data) - log.PanicIf(err) - - return rawExif, nil -} - -type ExifHeader struct { - ByteOrder binary.ByteOrder - FirstIfdOffset uint32 -} - -func (eh ExifHeader) String() string { - return fmt.Sprintf("ExifHeader", eh.ByteOrder, eh.FirstIfdOffset) -} - -// ParseExifHeader parses the bytes at the very top of the header. -// -// This will panic with ErrNoExif on any data errors so that we can double as -// an EXIF-detection routine. -func ParseExifHeader(data []byte) (eh ExifHeader, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Good reference: - // - // CIPA DC-008-2016; JEITA CP-3451D - // -> http://www.cipa.jp/std/documents/e/DC-008-Translation-2016-E.pdf - - if len(data) < 2 { - exifLogger.Warningf(nil, "Not enough data for EXIF header (1): (%d)", len(data)) - return eh, ErrNoExif - } - - byteOrderBytes := [2]byte{data[0], data[1]} - - byteOrder, found := ByteOrderLookup[byteOrderBytes] - if found == false { - // exifLogger.Warningf(nil, "EXIF byte-order not recognized: [%v]", byteOrderBytes) - return eh, ErrNoExif - } - - if len(data) < 4 { - exifLogger.Warningf(nil, "Not enough data for EXIF header (2): (%d)", len(data)) - return eh, ErrNoExif - } - - fixedBytes := [2]byte{data[2], data[3]} - expectedFixedBytes := ExifFixedBytesLookup[byteOrder] - if fixedBytes != expectedFixedBytes { - // exifLogger.Warningf(nil, "EXIF header fixed-bytes should be [%v] but are: [%v]", expectedFixedBytes, fixedBytes) - return eh, ErrNoExif - } - - if len(data) < 2 { - exifLogger.Warningf(nil, "Not enough data for EXIF header (3): (%d)", len(data)) - return eh, ErrNoExif - } - - firstIfdOffset := byteOrder.Uint32(data[4:8]) - - eh = ExifHeader{ - ByteOrder: byteOrder, - FirstIfdOffset: firstIfdOffset, - } - - return eh, nil -} - -// Visit recursively invokes a callback for every tag. -func Visit(rootIfdName string, ifdMapping *IfdMapping, tagIndex *TagIndex, exifData []byte, visitor RawTagVisitor) (eh ExifHeader, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - eh, err = ParseExifHeader(exifData) - log.PanicIf(err) - - ie := NewIfdEnumerate(ifdMapping, tagIndex, exifData, eh.ByteOrder) - - err = ie.Scan(rootIfdName, eh.FirstIfdOffset, visitor, true) - log.PanicIf(err) - - return eh, nil -} - -// Collect recursively builds a static structure of all IFDs and tags. -func Collect(ifdMapping *IfdMapping, tagIndex *TagIndex, exifData []byte) (eh ExifHeader, index IfdIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - eh, err = ParseExifHeader(exifData) - log.PanicIf(err) - - ie := NewIfdEnumerate(ifdMapping, tagIndex, exifData, eh.ByteOrder) - - index, err = ie.Collect(eh.FirstIfdOffset, true) - log.PanicIf(err) - - return eh, index, nil -} - -// BuildExifHeader constructs the bytes that go in the very beginning. -func BuildExifHeader(byteOrder binary.ByteOrder, firstIfdOffset uint32) (headerBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - b := new(bytes.Buffer) - - // This is the point in the data that all offsets are relative to. - boBytes := ByteOrderLookupR[byteOrder] - _, err = b.WriteString(string(boBytes[:])) - log.PanicIf(err) - - fixedBytes := ExifFixedBytesLookup[byteOrder] - - _, err = b.Write(fixedBytes[:]) - log.PanicIf(err) - - err = binary.Write(b, byteOrder, firstIfdOffset) - log.PanicIf(err) - - return b.Bytes(), nil -} diff --git a/vendor/github.com/dsoprea/go-exif/gps.go b/vendor/github.com/dsoprea/go-exif/gps.go deleted file mode 100644 index 7d74f22d3..000000000 --- a/vendor/github.com/dsoprea/go-exif/gps.go +++ /dev/null @@ -1,56 +0,0 @@ -package exif - -import ( - "errors" - "fmt" - "time" - - "github.com/golang/geo/s2" -) - -var ( - ErrGpsCoordinatesNotValid = errors.New("GPS coordinates not valid") -) - -type GpsDegrees struct { - Orientation byte - Degrees, Minutes, Seconds float64 -} - -func (d GpsDegrees) String() string { - return fmt.Sprintf("Degrees", string([]byte{d.Orientation}), d.Degrees, d.Minutes, d.Seconds) -} - -func (d GpsDegrees) Decimal() float64 { - decimal := float64(d.Degrees) + float64(d.Minutes)/60.0 + float64(d.Seconds)/3600.0 - - if d.Orientation == 'S' || d.Orientation == 'W' { - return -decimal - } else { - return decimal - } -} - -type GpsInfo struct { - Latitude, Longitude GpsDegrees - Altitude int - Timestamp time.Time -} - -func (gi *GpsInfo) String() string { - return fmt.Sprintf("GpsInfo", gi.Latitude.Decimal(), gi.Longitude.Decimal(), gi.Altitude, gi.Timestamp) -} - -func (gi *GpsInfo) S2CellId() s2.CellID { - latitude := gi.Latitude.Decimal() - longitude := gi.Longitude.Decimal() - - ll := s2.LatLngFromDegrees(latitude, longitude) - cellId := s2.CellIDFromLatLng(ll) - - if cellId.IsValid() == false { - panic(ErrGpsCoordinatesNotValid) - } - - return cellId -} diff --git a/vendor/github.com/dsoprea/go-exif/ifd.go b/vendor/github.com/dsoprea/go-exif/ifd.go deleted file mode 100644 index e75404ddc..000000000 --- a/vendor/github.com/dsoprea/go-exif/ifd.go +++ /dev/null @@ -1,407 +0,0 @@ -package exif - -import ( - "errors" - "fmt" - "strings" - - "github.com/dsoprea/go-logging" -) - -const ( - // IFD names. The paths that we referred to the IFDs with are comprised of - // these. - - IfdStandard = "IFD" - IfdExif = "Exif" - IfdGps = "GPSInfo" - IfdIop = "Iop" - - // Tag IDs for child IFDs. - - IfdExifId = 0x8769 - IfdGpsId = 0x8825 - IfdIopId = 0xA005 - - // Just a placeholder. - - IfdRootId = 0x0000 - - // The paths of the standard IFDs expressed in the standard IFD-mappings - // and as the group-names in the tag data. - - IfdPathStandard = "IFD" - IfdPathStandardExif = "IFD/Exif" - IfdPathStandardExifIop = "IFD/Exif/Iop" - IfdPathStandardGps = "IFD/GPSInfo" -) - -var ( - ifdLogger = log.NewLogger("exif.ifd") -) - -var ( - ErrChildIfdNotMapped = errors.New("no child-IFD for that tag-ID under parent") -) - -// type IfdIdentity struct { -// ParentIfdName string -// IfdName string -// } - -// func (ii IfdIdentity) String() string { -// return fmt.Sprintf("IfdIdentity", ii.ParentIfdName, ii.IfdName) -// } - -type MappedIfd struct { - ParentTagId uint16 - Placement []uint16 - Path []string - - Name string - TagId uint16 - Children map[uint16]*MappedIfd -} - -func (mi *MappedIfd) String() string { - pathPhrase := mi.PathPhrase() - return fmt.Sprintf("MappedIfd<(0x%04X) [%s] PATH=[%s]>", mi.TagId, mi.Name, pathPhrase) -} - -func (mi *MappedIfd) PathPhrase() string { - return strings.Join(mi.Path, "/") -} - -// IfdMapping describes all of the IFDs that we currently recognize. -type IfdMapping struct { - rootNode *MappedIfd -} - -func NewIfdMapping() (ifdMapping *IfdMapping) { - rootNode := &MappedIfd{ - Path: make([]string, 0), - Children: make(map[uint16]*MappedIfd), - } - - return &IfdMapping{ - rootNode: rootNode, - } -} - -func NewIfdMappingWithStandard() (ifdMapping *IfdMapping) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - im := NewIfdMapping() - - err := LoadStandardIfds(im) - log.PanicIf(err) - - return im -} - -func (im *IfdMapping) Get(parentPlacement []uint16) (childIfd *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ptr := im.rootNode - for _, tagId := range parentPlacement { - if descendantPtr, found := ptr.Children[tagId]; found == false { - log.Panicf("ifd child with tag-ID (%04x) not registered: [%s]", tagId, ptr.PathPhrase()) - } else { - ptr = descendantPtr - } - } - - return ptr, nil -} - -func (im *IfdMapping) GetWithPath(pathPhrase string) (mi *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if pathPhrase == "" { - log.Panicf("path-phrase is empty") - } - - path := strings.Split(pathPhrase, "/") - ptr := im.rootNode - - for _, name := range path { - var hit *MappedIfd - for _, mi := range ptr.Children { - if mi.Name == name { - hit = mi - break - } - } - - if hit == nil { - log.Panicf("ifd child with name [%s] not registered: [%s]", name, ptr.PathPhrase()) - } - - ptr = hit - } - - return ptr, nil -} - -// GetChild is a convenience function to get the child path for a given parent -// placement and child tag-ID. -func (im *IfdMapping) GetChild(parentPathPhrase string, tagId uint16) (mi *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - mi, err = im.GetWithPath(parentPathPhrase) - log.PanicIf(err) - - for _, childMi := range mi.Children { - if childMi.TagId == tagId { - return childMi, nil - } - } - - // Whether or not an IFD is defined in data, such an IFD is not registered - // and would be unknown. - log.Panic(ErrChildIfdNotMapped) - return nil, nil -} - -type IfdTagIdAndIndex struct { - Name string - TagId uint16 - Index int -} - -func (itii IfdTagIdAndIndex) String() string { - return fmt.Sprintf("IfdTagIdAndIndex", itii.Name, itii.TagId, itii.Index) -} - -// ResolvePath takes a list of names, which can also be suffixed with indices -// (to identify the second, third, etc.. sibling IFD) and returns a list of -// tag-IDs and those indices. -// -// Example: -// -// - IFD/Exif/Iop -// - IFD0/Exif/Iop -// -// This is the only call that supports adding the numeric indices. -func (im *IfdMapping) ResolvePath(pathPhrase string) (lineage []IfdTagIdAndIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - pathPhrase = strings.TrimSpace(pathPhrase) - - if pathPhrase == "" { - log.Panicf("can not resolve empty path-phrase") - } - - path := strings.Split(pathPhrase, "/") - lineage = make([]IfdTagIdAndIndex, len(path)) - - ptr := im.rootNode - empty := IfdTagIdAndIndex{} - for i, name := range path { - indexByte := name[len(name)-1] - index := 0 - if indexByte >= '0' && indexByte <= '9' { - index = int(indexByte - '0') - name = name[:len(name)-1] - } - - itii := IfdTagIdAndIndex{} - for _, mi := range ptr.Children { - if mi.Name != name { - continue - } - - itii.Name = name - itii.TagId = mi.TagId - itii.Index = index - - ptr = mi - - break - } - - if itii == empty { - log.Panicf("ifd child with name [%s] not registered: [%s]", name, pathPhrase) - } - - lineage[i] = itii - } - - return lineage, nil -} - -func (im *IfdMapping) FqPathPhraseFromLineage(lineage []IfdTagIdAndIndex) (fqPathPhrase string) { - fqPathParts := make([]string, len(lineage)) - for i, itii := range lineage { - if itii.Index > 0 { - fqPathParts[i] = fmt.Sprintf("%s%d", itii.Name, itii.Index) - } else { - fqPathParts[i] = itii.Name - } - } - - return strings.Join(fqPathParts, "/") -} - -func (im *IfdMapping) PathPhraseFromLineage(lineage []IfdTagIdAndIndex) (pathPhrase string) { - pathParts := make([]string, len(lineage)) - for i, itii := range lineage { - pathParts[i] = itii.Name - } - - return strings.Join(pathParts, "/") -} - -// StripPathPhraseIndices returns a non-fully-qualified path-phrase (no -// indices). -func (im *IfdMapping) StripPathPhraseIndices(pathPhrase string) (strippedPathPhrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - lineage, err := im.ResolvePath(pathPhrase) - log.PanicIf(err) - - strippedPathPhrase = im.PathPhraseFromLineage(lineage) - return strippedPathPhrase, nil -} - -// Add puts the given IFD at the given position of the tree. The position of the -// tree is referred to as the placement and is represented by a set of tag-IDs, -// where the leftmost is the root tag and the tags going to the right are -// progressive descendants. -func (im *IfdMapping) Add(parentPlacement []uint16, tagId uint16, name string) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! It would be nicer to provide a list of names in the placement rather than tag-IDs. - - ptr, err := im.Get(parentPlacement) - log.PanicIf(err) - - path := make([]string, len(parentPlacement)+1) - if len(parentPlacement) > 0 { - copy(path, ptr.Path) - } - - path[len(path)-1] = name - - placement := make([]uint16, len(parentPlacement)+1) - if len(placement) > 0 { - copy(placement, ptr.Placement) - } - - placement[len(placement)-1] = tagId - - childIfd := &MappedIfd{ - ParentTagId: ptr.TagId, - Path: path, - Placement: placement, - Name: name, - TagId: tagId, - Children: make(map[uint16]*MappedIfd), - } - - if _, found := ptr.Children[tagId]; found == true { - log.Panicf("child IFD with tag-ID (%04x) already registered under IFD [%s] with tag-ID (%04x)", tagId, ptr.Name, ptr.TagId) - } - - ptr.Children[tagId] = childIfd - - return nil -} - -func (im *IfdMapping) dumpLineages(stack []*MappedIfd, input []string) (output []string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - currentIfd := stack[len(stack)-1] - - output = input - for _, childIfd := range currentIfd.Children { - stackCopy := make([]*MappedIfd, len(stack)+1) - - copy(stackCopy, stack) - stackCopy[len(stack)] = childIfd - - // Add to output, but don't include the obligatory root node. - parts := make([]string, len(stackCopy)-1) - for i, mi := range stackCopy[1:] { - parts[i] = mi.Name - } - - output = append(output, strings.Join(parts, "/")) - - output, err = im.dumpLineages(stackCopy, output) - log.PanicIf(err) - } - - return output, nil -} - -func (im *IfdMapping) DumpLineages() (output []string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - stack := []*MappedIfd{im.rootNode} - output = make([]string, 0) - - output, err = im.dumpLineages(stack, output) - log.PanicIf(err) - - return output, nil -} - -func LoadStandardIfds(im *IfdMapping) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = im.Add([]uint16{}, IfdRootId, IfdStandard) - log.PanicIf(err) - - err = im.Add([]uint16{IfdRootId}, IfdExifId, IfdExif) - log.PanicIf(err) - - err = im.Add([]uint16{IfdRootId, IfdExifId}, IfdIopId, IfdIop) - log.PanicIf(err) - - err = im.Add([]uint16{IfdRootId}, IfdGpsId, IfdGps) - log.PanicIf(err) - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/ifd_builder.go b/vendor/github.com/dsoprea/go-exif/ifd_builder.go deleted file mode 100644 index 40ef4dc4f..000000000 --- a/vendor/github.com/dsoprea/go-exif/ifd_builder.go +++ /dev/null @@ -1,1265 +0,0 @@ -package exif - -// NOTES: -// -// The thumbnail offset and length tags shouldn't be set directly. Use the -// (*IfdBuilder).SetThumbnail() method instead. - -import ( - "errors" - "fmt" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - ifdBuilderLogger = log.NewLogger("exif.ifd_builder") -) - -var ( - ErrTagEntryNotFound = errors.New("tag entry not found") - ErrChildIbNotFound = errors.New("child IB not found") -) - -type IfdBuilderTagValue struct { - valueBytes []byte - ib *IfdBuilder -} - -func (ibtv IfdBuilderTagValue) String() string { - if ibtv.IsBytes() == true { - var valuePhrase string - if len(ibtv.valueBytes) <= 8 { - valuePhrase = fmt.Sprintf("%v", ibtv.valueBytes) - } else { - valuePhrase = fmt.Sprintf("%v...", ibtv.valueBytes[:8]) - } - - return fmt.Sprintf("IfdBuilderTagValue", valuePhrase, len(ibtv.valueBytes)) - } else if ibtv.IsIb() == true { - return fmt.Sprintf("IfdBuilderTagValue", ibtv.ib) - } else { - log.Panicf("IBTV state undefined") - return "" - } -} - -func NewIfdBuilderTagValueFromBytes(valueBytes []byte) *IfdBuilderTagValue { - return &IfdBuilderTagValue{ - valueBytes: valueBytes, - } -} - -func NewIfdBuilderTagValueFromIfdBuilder(ib *IfdBuilder) *IfdBuilderTagValue { - return &IfdBuilderTagValue{ - ib: ib, - } -} - -// IsBytes returns true if the bytes are populated. This is always the case -// when we're loaded from a tag in an existing IFD. -func (ibtv IfdBuilderTagValue) IsBytes() bool { - return ibtv.valueBytes != nil -} - -func (ibtv IfdBuilderTagValue) Bytes() []byte { - if ibtv.IsBytes() == false { - log.Panicf("this tag is not a byte-slice value") - } else if ibtv.IsIb() == true { - log.Panicf("this tag is an IFD-builder value not a byte-slice") - } - - return ibtv.valueBytes -} - -func (ibtv IfdBuilderTagValue) IsIb() bool { - return ibtv.ib != nil -} - -func (ibtv IfdBuilderTagValue) Ib() *IfdBuilder { - if ibtv.IsIb() == false { - log.Panicf("this tag is not an IFD-builder value") - } else if ibtv.IsBytes() == true { - log.Panicf("this tag is a byte-slice, not a IFD-builder") - } - - return ibtv.ib -} - -type BuilderTag struct { - // ifdPath is the path of the IFD that hosts this tag. - ifdPath string - - tagId uint16 - typeId TagTypePrimitive - - // value is either a value that can be encoded, an IfdBuilder instance (for - // child IFDs), or an IfdTagEntry instance representing an existing, - // previously-stored tag. - value *IfdBuilderTagValue - - // byteOrder is the byte order. It's chiefly/originally here to support - // printing the value. - byteOrder binary.ByteOrder -} - -func NewBuilderTag(ifdPath string, tagId uint16, typeId TagTypePrimitive, value *IfdBuilderTagValue, byteOrder binary.ByteOrder) *BuilderTag { - return &BuilderTag{ - ifdPath: ifdPath, - tagId: tagId, - typeId: typeId, - value: value, - byteOrder: byteOrder, - } -} - -func NewChildIfdBuilderTag(ifdPath string, tagId uint16, value *IfdBuilderTagValue) *BuilderTag { - return &BuilderTag{ - ifdPath: ifdPath, - tagId: tagId, - typeId: TypeLong, - value: value, - } -} - -func (bt *BuilderTag) Value() (value *IfdBuilderTagValue) { - return bt.value -} - -func (bt *BuilderTag) String() string { - var valueString string - - if bt.value.IsBytes() == true { - var err error - - valueString, err = Format(bt.value.Bytes(), bt.typeId, false, bt.byteOrder) - log.PanicIf(err) - } else { - valueString = fmt.Sprintf("%v", bt.value) - } - - return fmt.Sprintf("BuilderTag", bt.ifdPath, bt.tagId, TypeNames[bt.typeId], valueString) -} - -func (bt *BuilderTag) SetValue(byteOrder binary.ByteOrder, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - tt := NewTagType(bt.typeId, byteOrder) - ve := NewValueEncoder(byteOrder) - - var ed EncodedData - if bt.typeId == TypeUndefined { - var err error - - ed, err = EncodeUndefined(bt.ifdPath, bt.tagId, value) - log.PanicIf(err) - } else { - var err error - - ed, err = ve.EncodeWithType(tt, value) - log.PanicIf(err) - } - - bt.value = NewIfdBuilderTagValueFromBytes(ed.Encoded) - - return nil -} - -// NewStandardBuilderTag constructs a `BuilderTag` instance. The type is looked -// up. `ii` is the type of IFD that owns this tag. -func NewStandardBuilderTag(ifdPath string, it *IndexedTag, byteOrder binary.ByteOrder, value interface{}) *BuilderTag { - typeId := it.Type - tt := NewTagType(typeId, byteOrder) - - ve := NewValueEncoder(byteOrder) - - var ed EncodedData - if it.Type == TypeUndefined { - var err error - - ed, err = EncodeUndefined(ifdPath, it.Id, value) - log.PanicIf(err) - } else { - var err error - - ed, err = ve.EncodeWithType(tt, value) - log.PanicIf(err) - } - - tagValue := NewIfdBuilderTagValueFromBytes(ed.Encoded) - - return NewBuilderTag( - ifdPath, - it.Id, - typeId, - tagValue, - byteOrder) -} - -type IfdBuilder struct { - // ifdName is the name of the IFD represented by this instance. - name string - - // ifdPath is the path of the IFD represented by this instance. - ifdPath string - - // fqIfdPath is the fully-qualified path of the IFD represented by this - // instance. - fqIfdPath string - - // ifdTagId will be non-zero if we're a child IFD. - ifdTagId uint16 - - byteOrder binary.ByteOrder - - // Includes both normal tags and IFD tags (which point to child IFDs). - // TODO(dustin): Keep a separate list of children like with `Ifd`. - // TODO(dustin): Either rename this or `Entries` in `Ifd` to be the same thing. - tags []*BuilderTag - - // existingOffset will be the offset that this IFD is currently found at if - // it represents an IFD that has previously been stored (or 0 if not). - existingOffset uint32 - - // nextIb represents the next link if we're chaining to another. - nextIb *IfdBuilder - - // thumbnailData is populated with thumbnail data if there was thumbnail - // data. Otherwise, it's nil. - thumbnailData []byte - - ifdMapping *IfdMapping - tagIndex *TagIndex -} - -func NewIfdBuilder(ifdMapping *IfdMapping, tagIndex *TagIndex, fqIfdPath string, byteOrder binary.ByteOrder) (ib *IfdBuilder) { - ifdPath, err := ifdMapping.StripPathPhraseIndices(fqIfdPath) - log.PanicIf(err) - - var ifdTagId uint16 - - mi, err := ifdMapping.GetWithPath(ifdPath) - if err == nil { - ifdTagId = mi.TagId - } else if log.Is(err, ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - ib = &IfdBuilder{ - // The right-most part of the IFD-path. - name: mi.Name, - - // ifdPath describes the current IFD placement within the IFD tree. - ifdPath: ifdPath, - - // fqIfdPath describes the current IFD placement within the IFD tree as - // well as being qualified with non-zero indices. - fqIfdPath: fqIfdPath, - - // ifdTagId is empty unless it's a child-IFD. - ifdTagId: ifdTagId, - - byteOrder: byteOrder, - tags: make([]*BuilderTag, 0), - - ifdMapping: ifdMapping, - tagIndex: tagIndex, - } - - return ib -} - -// NewIfdBuilderWithExistingIfd creates a new IB using the same header type -// information as the given IFD. -func NewIfdBuilderWithExistingIfd(ifd *Ifd) (ib *IfdBuilder) { - name := ifd.Name - ifdPath := ifd.IfdPath - fqIfdPath := ifd.FqIfdPath - - var ifdTagId uint16 - - // There is no tag-ID for the root IFD. It will never be a child IFD. - if ifdPath != IfdPathStandard { - mi, err := ifd.ifdMapping.GetWithPath(ifdPath) - log.PanicIf(err) - - ifdTagId = mi.TagId - } - - ib = &IfdBuilder{ - name: name, - ifdPath: ifdPath, - fqIfdPath: fqIfdPath, - ifdTagId: ifdTagId, - byteOrder: ifd.ByteOrder, - existingOffset: ifd.Offset, - ifdMapping: ifd.ifdMapping, - tagIndex: ifd.tagIndex, - } - - return ib -} - -// NewIfdBuilderFromExistingChain creates a chain of IB instances from an -// IFD chain generated from real data. -func NewIfdBuilderFromExistingChain(rootIfd *Ifd, itevr *IfdTagEntryValueResolver) (firstIb *IfdBuilder) { - // OBSOLETE(dustin): Support for `itevr` is now obsolete. This parameter will be removed in the future. - - var lastIb *IfdBuilder - i := 0 - for thisExistingIfd := rootIfd; thisExistingIfd != nil; thisExistingIfd = thisExistingIfd.NextIfd { - newIb := NewIfdBuilder(rootIfd.ifdMapping, rootIfd.tagIndex, rootIfd.FqIfdPath, thisExistingIfd.ByteOrder) - if firstIb == nil { - firstIb = newIb - } else { - lastIb.SetNextIb(newIb) - } - - err := newIb.AddTagsFromExisting(thisExistingIfd, nil, nil, nil) - log.PanicIf(err) - - lastIb = newIb - i++ - } - - return firstIb -} - -func (ib *IfdBuilder) NextIb() (nextIb *IfdBuilder, err error) { - return ib.nextIb, nil -} - -func (ib *IfdBuilder) ChildWithTagId(childIfdTagId uint16) (childIb *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for _, bt := range ib.tags { - if bt.value.IsIb() == false { - continue - } - - childIbThis := bt.value.Ib() - - if childIbThis.ifdTagId == childIfdTagId { - return childIbThis, nil - } - } - - log.Panic(ErrChildIbNotFound) - - // Never reached. - return nil, nil -} - -func getOrCreateIbFromRootIbInner(rootIb *IfdBuilder, parentIb *IfdBuilder, currentLineage []IfdTagIdAndIndex) (ib *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - thisIb := rootIb - - // Since we're calling ourselves recursively with incrementally different - // paths, the FQ IFD-path of the parent that called us needs to be passed - // in, in order for us to know it. - var parentLineage []IfdTagIdAndIndex - if parentIb != nil { - var err error - - parentLineage, err = thisIb.ifdMapping.ResolvePath(parentIb.fqIfdPath) - log.PanicIf(err) - } - - // Process the current path part. - currentItii := currentLineage[0] - - // Make sure the leftmost part of the FQ IFD-path agrees with the IB we - // were given. - - expectedFqRootIfdPath := "" - if parentLineage != nil { - expectedLineage := append(parentLineage, currentItii) - expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(expectedLineage) - } else { - expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(currentLineage[:1]) - } - - if expectedFqRootIfdPath != thisIb.fqIfdPath { - log.Panicf("the FQ IFD-path [%s] we were given does not match the builder's FQ IFD-path [%s]", expectedFqRootIfdPath, thisIb.fqIfdPath) - } - - // If we actually wanted a sibling (currentItii.Index > 0) then seek to it, - // appending new siblings, as required, until we get there. - for i := 0; i < currentItii.Index; i++ { - if thisIb.nextIb == nil { - // Generate an FQ IFD-path for the sibling. It'll use the same - // non-FQ IFD-path as the current IB. - - siblingFqIfdPath := "" - if parentLineage != nil { - siblingFqIfdPath = fmt.Sprintf("%s/%s%d", parentIb.fqIfdPath, currentItii.Name, i+1) - } else { - siblingFqIfdPath = fmt.Sprintf("%s%d", currentItii.Name, i+1) - } - - thisIb.nextIb = NewIfdBuilder(thisIb.ifdMapping, thisIb.tagIndex, siblingFqIfdPath, thisIb.byteOrder) - } - - thisIb = thisIb.nextIb - } - - // There is no child IFD to process. We're done. - if len(currentLineage) == 1 { - return thisIb, nil - } - - // Establish the next child to be processed. - - childItii := currentLineage[1] - - var foundChild *IfdBuilder - for _, bt := range thisIb.tags { - if bt.value.IsIb() == false { - continue - } - - childIb := bt.value.Ib() - - if childIb.ifdTagId == childItii.TagId { - foundChild = childIb - break - } - } - - // If we didn't find the child, add it. - if foundChild == nil { - thisIbLineage, err := thisIb.ifdMapping.ResolvePath(thisIb.fqIfdPath) - log.PanicIf(err) - - childLineage := make([]IfdTagIdAndIndex, len(thisIbLineage)+1) - copy(childLineage, thisIbLineage) - - childLineage[len(childLineage)-1] = childItii - - fqIfdChildPath := thisIb.ifdMapping.FqPathPhraseFromLineage(childLineage) - - foundChild = NewIfdBuilder(thisIb.ifdMapping, thisIb.tagIndex, fqIfdChildPath, thisIb.byteOrder) - - err = thisIb.AddChildIb(foundChild) - log.PanicIf(err) - } - - finalIb, err := getOrCreateIbFromRootIbInner(foundChild, thisIb, currentLineage[1:]) - log.PanicIf(err) - - return finalIb, nil -} - -// GetOrCreateIbFromRootIb returns an IB representing the requested IFD, even if -// an IB doesn't already exist for it. This function may call itself -// recursively. -func GetOrCreateIbFromRootIb(rootIb *IfdBuilder, fqIfdPath string) (ib *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // lineage is a necessity of our recursion process. It doesn't include any - // parent IFDs on its left-side; it starts with the current IB only. - lineage, err := rootIb.ifdMapping.ResolvePath(fqIfdPath) - log.PanicIf(err) - - ib, err = getOrCreateIbFromRootIbInner(rootIb, nil, lineage) - log.PanicIf(err) - - return ib, nil -} - -func (ib *IfdBuilder) String() string { - nextIfdPhrase := "" - if ib.nextIb != nil { - // TODO(dustin): We were setting this to ii.String(), but we were getting hex-data when printing this after building from an existing chain. - nextIfdPhrase = ib.nextIb.ifdPath - } - - return fmt.Sprintf("IfdBuilder", ib.ifdPath, ib.ifdTagId, len(ib.tags), ib.existingOffset, nextIfdPhrase) -} - -func (ib *IfdBuilder) Tags() (tags []*BuilderTag) { - return ib.tags -} - -// SetThumbnail sets thumbnail data. -// -// NOTES: -// -// - We don't manage any facet of the thumbnail data. This is the -// responsibility of the user/developer. -// - This method will fail unless the thumbnail is set on a the root IFD. -// However, in order to be valid, it must be set on the second one, linked to -// by the first, as per the EXIF/TIFF specification. -// - We set the offset to (0) now but will allocate the data and properly assign -// the offset when the IB is encoded (later). -func (ib *IfdBuilder) SetThumbnail(data []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if ib.ifdPath != IfdPathStandard { - log.Panicf("thumbnails can only go into a root Ifd (and only the second one)") - } - - // TODO(dustin): !! Add a test for this function. - - if data == nil || len(data) == 0 { - log.Panic("thumbnail is empty") - } - - ib.thumbnailData = data - - ibtvfb := NewIfdBuilderTagValueFromBytes(ib.thumbnailData) - offsetBt := - NewBuilderTag( - ib.ifdPath, - ThumbnailOffsetTagId, - TypeLong, - ibtvfb, - ib.byteOrder) - - err = ib.Set(offsetBt) - log.PanicIf(err) - - thumbnailSizeIt, err := ib.tagIndex.Get(ib.ifdPath, ThumbnailSizeTagId) - log.PanicIf(err) - - sizeBt := NewStandardBuilderTag(ib.ifdPath, thumbnailSizeIt, ib.byteOrder, []uint32{uint32(len(ib.thumbnailData))}) - - err = ib.Set(sizeBt) - log.PanicIf(err) - - return nil -} - -func (ib *IfdBuilder) Thumbnail() []byte { - return ib.thumbnailData -} - -func (ib *IfdBuilder) printTagTree(levels int) { - indent := strings.Repeat(" ", levels*2) - - i := 0 - for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { - prefix := " " - if i > 0 { - prefix = ">" - } - - if levels == 0 { - fmt.Printf("%s%sIFD: %s INDEX=(%d)\n", indent, prefix, currentIb, i) - } else { - fmt.Printf("%s%sChild IFD: %s\n", indent, prefix, currentIb) - } - - if len(currentIb.tags) > 0 { - fmt.Printf("\n") - - for i, tag := range currentIb.tags { - isChildIb := false - _, err := ib.ifdMapping.GetChild(currentIb.ifdPath, tag.tagId) - if err == nil { - isChildIb = true - } else if log.Is(err, ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - tagName := "" - - // If a normal tag (not a child IFD) get the name. - if isChildIb == true { - tagName = "" - } else { - it, err := ib.tagIndex.Get(tag.ifdPath, tag.tagId) - if log.Is(err, ErrTagNotFound) == true { - tagName = "" - } else if err != nil { - log.Panic(err) - } else { - tagName = it.Name - } - } - - value := tag.Value() - - if value.IsIb() == true { - fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, value.Ib()) - } else { - fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, tag) - } - - if isChildIb == true { - if tag.value.IsIb() == false { - log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) - } - - fmt.Printf("\n") - - childIb := tag.value.Ib() - childIb.printTagTree(levels + 1) - } - } - - fmt.Printf("\n") - } - - i++ - } -} - -func (ib *IfdBuilder) PrintTagTree() { - ib.printTagTree(0) -} - -func (ib *IfdBuilder) printIfdTree(levels int) { - indent := strings.Repeat(" ", levels*2) - - i := 0 - for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { - prefix := " " - if i > 0 { - prefix = ">" - } - - fmt.Printf("%s%s%s\n", indent, prefix, currentIb) - - if len(currentIb.tags) > 0 { - for _, tag := range currentIb.tags { - isChildIb := false - _, err := ib.ifdMapping.GetChild(currentIb.ifdPath, tag.tagId) - if err == nil { - isChildIb = true - } else if log.Is(err, ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - if isChildIb == true { - if tag.value.IsIb() == false { - log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) - } - - childIb := tag.value.Ib() - childIb.printIfdTree(levels + 1) - } - } - } - - i++ - } -} - -func (ib *IfdBuilder) PrintIfdTree() { - ib.printIfdTree(0) -} - -func (ib *IfdBuilder) dumpToStrings(thisIb *IfdBuilder, prefix string, tagId uint16, lines []string) (linesOutput []string) { - if lines == nil { - linesOutput = make([]string, 0) - } else { - linesOutput = lines - } - - siblingIfdIndex := 0 - for ; thisIb != nil; thisIb = thisIb.nextIb { - line := fmt.Sprintf("IFD", prefix, thisIb.fqIfdPath, siblingIfdIndex, thisIb.ifdTagId, tagId) - linesOutput = append(linesOutput, line) - - for i, tag := range thisIb.tags { - var childIb *IfdBuilder - childIfdName := "" - if tag.value.IsIb() == true { - childIb = tag.value.Ib() - childIfdName = childIb.ifdPath - } - - line := fmt.Sprintf("TAG", prefix, thisIb.fqIfdPath, thisIb.ifdTagId, childIfdName, i, tag.tagId) - linesOutput = append(linesOutput, line) - - if childIb == nil { - continue - } - - childPrefix := "" - if prefix == "" { - childPrefix = fmt.Sprintf("%s", thisIb.ifdPath) - } else { - childPrefix = fmt.Sprintf("%s->%s", prefix, thisIb.ifdPath) - } - - linesOutput = thisIb.dumpToStrings(childIb, childPrefix, tag.tagId, linesOutput) - } - - siblingIfdIndex++ - } - - return linesOutput -} - -func (ib *IfdBuilder) DumpToStrings() (lines []string) { - return ib.dumpToStrings(ib, "", 0, lines) -} - -func (ib *IfdBuilder) SetNextIb(nextIb *IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ib.nextIb = nextIb - - return nil -} - -func (ib *IfdBuilder) DeleteN(tagId uint16, n int) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if n < 1 { - log.Panicf("N must be at least 1: (%d)", n) - } - - for n > 0 { - j := -1 - for i, bt := range ib.tags { - if bt.tagId == tagId { - j = i - break - } - } - - if j == -1 { - log.Panic(ErrTagEntryNotFound) - } - - ib.tags = append(ib.tags[:j], ib.tags[j+1:]...) - n-- - } - - return nil -} - -func (ib *IfdBuilder) DeleteFirst(tagId uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = ib.DeleteN(tagId, 1) - log.PanicIf(err) - - return nil -} - -func (ib *IfdBuilder) DeleteAll(tagId uint16) (n int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for { - err = ib.DeleteN(tagId, 1) - if log.Is(err, ErrTagEntryNotFound) == true { - break - } else if err != nil { - log.Panic(err) - } - - n++ - } - - return n, nil -} - -func (ib *IfdBuilder) ReplaceAt(position int, bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if position < 0 { - log.Panicf("replacement position must be 0 or greater") - } else if position >= len(ib.tags) { - log.Panicf("replacement position does not exist") - } - - ib.tags[position] = bt - - return nil -} - -func (ib *IfdBuilder) Replace(tagId uint16, bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - position, err := ib.Find(tagId) - log.PanicIf(err) - - ib.tags[position] = bt - - return nil -} - -// Set will add a new entry or update an existing entry. -func (ib *IfdBuilder) Set(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - position, err := ib.Find(bt.tagId) - if err == nil { - ib.tags[position] = bt - } else if log.Is(err, ErrTagEntryNotFound) == true { - err = ib.add(bt) - log.PanicIf(err) - } else { - log.Panic(err) - } - - return nil -} - -func (ib *IfdBuilder) FindN(tagId uint16, maxFound int) (found []int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found = make([]int, 0) - - for i, bt := range ib.tags { - if bt.tagId == tagId { - found = append(found, i) - if maxFound == 0 || len(found) >= maxFound { - break - } - } - } - - return found, nil -} - -func (ib *IfdBuilder) Find(tagId uint16) (position int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found, err := ib.FindN(tagId, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - return found[0], nil -} - -func (ib *IfdBuilder) FindTag(tagId uint16) (bt *BuilderTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found, err := ib.FindN(tagId, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - position := found[0] - - return ib.tags[position], nil -} - -func (ib *IfdBuilder) FindTagWithName(tagName string) (bt *BuilderTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.GetWithName(ib.ifdPath, tagName) - log.PanicIf(err) - - found, err := ib.FindN(it.Id, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - position := found[0] - - return ib.tags[position], nil -} - -func (ib *IfdBuilder) add(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if bt.ifdPath == "" { - log.Panicf("BuilderTag ifdPath is not set: %s", bt) - } else if bt.typeId == 0x0 { - log.Panicf("BuilderTag type-ID is not set: %s", bt) - } else if bt.value == nil { - log.Panicf("BuilderTag value is not set: %s", bt) - } - - ib.tags = append(ib.tags, bt) - return nil -} - -func (ib *IfdBuilder) Add(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if bt.value.IsIb() == true { - log.Panicf("child IfdBuilders must be added via AddChildIb() or AddTagsFromExisting(), not Add()") - } - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// AddChildIb adds a tag that branches to a new IFD. -func (ib *IfdBuilder) AddChildIb(childIb *IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if childIb.ifdTagId == 0 { - log.Panicf("IFD can not be used as a child IFD (not associated with a tag-ID): %v", childIb) - } else if childIb.byteOrder != ib.byteOrder { - log.Panicf("Child IFD does not have the same byte-order: [%s] != [%s]", childIb.byteOrder, ib.byteOrder) - } - - // Since no standard IFDs supports occuring more than once, check that a - // tag of this type has not been previously added. Note that we just search - // the current IFD and *not every* IFD. - for _, bt := range childIb.tags { - if bt.tagId == childIb.ifdTagId { - log.Panicf("child-IFD already added: %v", childIb.ifdPath) - } - } - - bt := ib.NewBuilderTagFromBuilder(childIb) - ib.tags = append(ib.tags, bt) - - return nil -} - -func (ib *IfdBuilder) NewBuilderTagFromBuilder(childIb *IfdBuilder) (bt *BuilderTag) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - value := NewIfdBuilderTagValueFromIfdBuilder(childIb) - - bt = NewChildIfdBuilderTag( - ib.ifdPath, - childIb.ifdTagId, - value) - - return bt -} - -// AddTagsFromExisting does a verbatim copy of the entries in `ifd` to this -// builder. It excludes child IFDs. These must be added explicitly via -// `AddChildIb()`. -func (ib *IfdBuilder) AddTagsFromExisting(ifd *Ifd, itevr *IfdTagEntryValueResolver, includeTagIds []uint16, excludeTagIds []uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // OBSOLETE(dustin): Support for `itevr` is now obsolete. This parameter will be removed in the future. - - thumbnailData, err := ifd.Thumbnail() - if err == nil { - err = ib.SetThumbnail(thumbnailData) - log.PanicIf(err) - } else if log.Is(err, ErrNoThumbnail) == false { - log.Panic(err) - } - - for i, ite := range ifd.Entries { - if ite.TagId == ThumbnailOffsetTagId || ite.TagId == ThumbnailSizeTagId { - // These will be added on-the-fly when we encode. - continue - } - - if excludeTagIds != nil && len(excludeTagIds) > 0 { - found := false - for _, excludedTagId := range excludeTagIds { - if excludedTagId == ite.TagId { - found = true - } - } - - if found == true { - continue - } - } - - if includeTagIds != nil && len(includeTagIds) > 0 { - // Whether or not there was a list of excludes, if there is a list - // of includes than the current tag has to be in it. - - found := false - for _, includedTagId := range includeTagIds { - if includedTagId == ite.TagId { - found = true - break - } - } - - if found == false { - continue - } - } - - var bt *BuilderTag - - if ite.ChildIfdPath != "" { - // If we want to add an IFD tag, we'll have to build it first and - // *then* add it via a different method. - - // Figure out which of the child-IFDs that are associated with - // this IFD represents this specific child IFD. - - var childIfd *Ifd - for _, thisChildIfd := range ifd.Children { - if thisChildIfd.ParentTagIndex != i { - continue - } else if thisChildIfd.TagId != 0xffff && thisChildIfd.TagId != ite.TagId { - log.Panicf("child-IFD tag is not correct: TAG-POSITION=(%d) ITE=%s CHILD-IFD=%s", thisChildIfd.ParentTagIndex, ite, thisChildIfd) - } - - childIfd = thisChildIfd - break - } - - if childIfd == nil { - childTagIds := make([]string, len(ifd.Children)) - for j, childIfd := range ifd.Children { - childTagIds[j] = fmt.Sprintf("0x%04x (parent tag-position %d)", childIfd.TagId, childIfd.ParentTagIndex) - } - - log.Panicf("could not find child IFD for child ITE: IFD-PATH=[%s] TAG-ID=(0x%04x) CURRENT-TAG-POSITION=(%d) CHILDREN=%v", ite.IfdPath, ite.TagId, i, childTagIds) - } - - childIb := NewIfdBuilderFromExistingChain(childIfd, nil) - bt = ib.NewBuilderTagFromBuilder(childIb) - } else { - // Non-IFD tag. - - valueContext := ifd.GetValueContext(ite) - - var rawBytes []byte - - if ite.TagType == TypeUndefined { - // It's an undefined-type value. Try to process, or skip if - // we don't know how to. - - undefinedInterface, err := valueContext.Undefined() - if err != nil { - if err == ErrUnhandledUnknownTypedTag { - // It's an undefined-type tag that we don't handle. If - // we don't know how to handle it, we can't know how - // many bytes it is and we must skip it. - continue - } - - log.Panic(err) - } - - undefined, ok := undefinedInterface.(UnknownTagValue) - if ok != true { - log.Panicf("unexpected value returned from undefined-value processor") - } - - rawBytes, err = undefined.ValueBytes() - log.PanicIf(err) - } else { - // It's a value with a standard type. - - var err error - - rawBytes, err = valueContext.readRawEncoded() - log.PanicIf(err) - } - - value := NewIfdBuilderTagValueFromBytes(rawBytes) - - bt = NewBuilderTag( - ifd.IfdPath, - ite.TagId, - ite.TagType, - value, - ib.byteOrder) - } - - err := ib.add(bt) - log.PanicIf(err) - } - - return nil -} - -// AddStandard quickly and easily composes and adds the tag using the -// information already known about a tag. Only works with standard tags. -func (ib *IfdBuilder) AddStandard(tagId uint16, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.Get(ib.ifdPath, tagId) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.ifdPath, it, ib.byteOrder, value) - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// AddStandardWithName quickly and easily composes and adds the tag using the -// information already known about a tag (using the name). Only works with -// standard tags. -func (ib *IfdBuilder) AddStandardWithName(tagName string, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.GetWithName(ib.ifdPath, tagName) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.ifdPath, it, ib.byteOrder, value) - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// SetStandard quickly and easily composes and adds or replaces the tag using -// the information already known about a tag. Only works with standard tags. -func (ib *IfdBuilder) SetStandard(tagId uint16, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test for this function. - - it, err := ib.tagIndex.Get(ib.ifdPath, tagId) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.ifdPath, it, ib.byteOrder, value) - - i, err := ib.Find(tagId) - if err != nil { - if log.Is(err, ErrTagEntryNotFound) == false { - log.Panic(err) - } - - ib.tags = append(ib.tags, bt) - } else { - ib.tags[i] = bt - } - - return nil -} - -// SetStandardWithName quickly and easily composes and adds or replaces the -// tag using the information already known about a tag (using the name). Only -// works with standard tags. -func (ib *IfdBuilder) SetStandardWithName(tagName string, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test for this function. - - it, err := ib.tagIndex.GetWithName(ib.ifdPath, tagName) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.ifdPath, it, ib.byteOrder, value) - - i, err := ib.Find(bt.tagId) - if err != nil { - if log.Is(err, ErrTagEntryNotFound) == false { - log.Panic(err) - } - - ib.tags = append(ib.tags, bt) - } else { - ib.tags[i] = bt - } - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/ifd_builder_encode.go b/vendor/github.com/dsoprea/go-exif/ifd_builder_encode.go deleted file mode 100644 index 90fb2ddbf..000000000 --- a/vendor/github.com/dsoprea/go-exif/ifd_builder_encode.go +++ /dev/null @@ -1,530 +0,0 @@ -package exif - -import ( - "bytes" - "fmt" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -const ( - // Tag-ID + Tag-Type + Unit-Count + Value/Offset. - IfdTagEntrySize = uint32(2 + 2 + 4 + 4) -) - -type ByteWriter struct { - b *bytes.Buffer - byteOrder binary.ByteOrder -} - -func NewByteWriter(b *bytes.Buffer, byteOrder binary.ByteOrder) (bw *ByteWriter) { - return &ByteWriter{ - b: b, - byteOrder: byteOrder, - } -} - -func (bw ByteWriter) writeAsBytes(value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = binary.Write(bw.b, bw.byteOrder, value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteUint32(value uint32) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = bw.writeAsBytes(value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteUint16(value uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = bw.writeAsBytes(value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteFourBytes(value []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - len_ := len(value) - if len_ != 4 { - log.Panicf("value is not four-bytes: (%d)", len_) - } - - _, err = bw.b.Write(value) - log.PanicIf(err) - - return nil -} - -// ifdOffsetIterator keeps track of where the next IFD should be written by -// keeping track of where the offsets start, the data that has been added, and -// bumping the offset *when* the data is added. -type ifdDataAllocator struct { - offset uint32 - b bytes.Buffer -} - -func newIfdDataAllocator(ifdDataAddressableOffset uint32) *ifdDataAllocator { - return &ifdDataAllocator{ - offset: ifdDataAddressableOffset, - } -} - -func (ida *ifdDataAllocator) Allocate(value []byte) (offset uint32, err error) { - _, err = ida.b.Write(value) - log.PanicIf(err) - - offset = ida.offset - ida.offset += uint32(len(value)) - - return offset, nil -} - -func (ida *ifdDataAllocator) NextOffset() uint32 { - return ida.offset -} - -func (ida *ifdDataAllocator) Bytes() []byte { - return ida.b.Bytes() -} - -// IfdByteEncoder converts an IB to raw bytes (for writing) while also figuring -// out all of the allocations and indirection that is required for extended -// data. -type IfdByteEncoder struct { - // journal holds a list of actions taken while encoding. - journal [][3]string -} - -func NewIfdByteEncoder() (ibe *IfdByteEncoder) { - return &IfdByteEncoder{ - journal: make([][3]string, 0), - } -} - -func (ibe *IfdByteEncoder) Journal() [][3]string { - return ibe.journal -} - -func (ibe *IfdByteEncoder) TableSize(entryCount int) uint32 { - // Tag-Count + (Entry-Size * Entry-Count) + Next-IFD-Offset. - return uint32(2) + (IfdTagEntrySize * uint32(entryCount)) + uint32(4) -} - -func (ibe *IfdByteEncoder) pushToJournal(where, direction, format string, args ...interface{}) { - event := [3]string{ - direction, - where, - fmt.Sprintf(format, args...), - } - - ibe.journal = append(ibe.journal, event) -} - -// PrintJournal prints a hierarchical representation of the steps taken during -// encoding. -func (ibe *IfdByteEncoder) PrintJournal() { - maxWhereLength := 0 - for _, event := range ibe.journal { - where := event[1] - - len_ := len(where) - if len_ > maxWhereLength { - maxWhereLength = len_ - } - } - - level := 0 - for i, event := range ibe.journal { - direction := event[0] - where := event[1] - message := event[2] - - if direction != ">" && direction != "<" && direction != "-" { - log.Panicf("journal operation not valid: [%s]", direction) - } - - if direction == "<" { - if level <= 0 { - log.Panicf("journal operations unbalanced (too many closes)") - } - - level-- - } - - indent := strings.Repeat(" ", level) - - fmt.Printf("%3d %s%s %s: %s\n", i, indent, direction, where, message) - - if direction == ">" { - level++ - } - } - - if level != 0 { - log.Panicf("journal operations unbalanced (too many opens)") - } -} - -// encodeTagToBytes encodes the given tag to a byte stream. If -// `nextIfdOffsetToWrite` is more than (0), recurse into child IFDs -// (`nextIfdOffsetToWrite` is required in order for them to know where the its -// IFD data will be written, in order for them to know the offset of where -// their allocated-data block will start, which follows right behind). -func (ibe *IfdByteEncoder) encodeTagToBytes(ib *IfdBuilder, bt *BuilderTag, bw *ByteWriter, ida *ifdDataAllocator, nextIfdOffsetToWrite uint32) (childIfdBlock []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Write tag-ID. - err = bw.WriteUint16(bt.tagId) - log.PanicIf(err) - - // Works for both values and child IFDs (which have an official size of - // LONG). - err = bw.WriteUint16(uint16(bt.typeId)) - log.PanicIf(err) - - // Write unit-count. - - if bt.value.IsBytes() == true { - effectiveType := bt.typeId - if bt.typeId == TypeUndefined { - effectiveType = TypeByte - } - - // It's a non-unknown value.Calculate the count of values of - // the type that we're writing and the raw bytes for the whole list. - - typeSize := uint32(effectiveType.Size()) - - valueBytes := bt.value.Bytes() - - len_ := len(valueBytes) - unitCount := uint32(len_) / typeSize - - if _, found := tagsWithoutAlignment[bt.tagId]; found == false { - remainder := uint32(len_) % typeSize - - if remainder > 0 { - log.Panicf("tag (0x%04x) value of (%d) bytes not evenly divisible by type-size (%d)", bt.tagId, len_, typeSize) - } - } - - err = bw.WriteUint32(unitCount) - log.PanicIf(err) - - // Write four-byte value/offset. - - if len_ > 4 { - offset, err := ida.Allocate(valueBytes) - log.PanicIf(err) - - err = bw.WriteUint32(offset) - log.PanicIf(err) - } else { - fourBytes := make([]byte, 4) - copy(fourBytes, valueBytes) - - err = bw.WriteFourBytes(fourBytes) - log.PanicIf(err) - } - } else { - if bt.value.IsIb() == false { - log.Panicf("tag value is not a byte-slice but also not a child IB: %v", bt) - } - - // Write unit-count (one LONG representing one offset). - err = bw.WriteUint32(1) - log.PanicIf(err) - - if nextIfdOffsetToWrite > 0 { - var err error - - ibe.pushToJournal("encodeTagToBytes", ">", "[%s]->[%s]", ib.ifdPath, bt.value.Ib().ifdPath) - - // Create the block of IFD data and everything it requires. - childIfdBlock, err = ibe.encodeAndAttachIfd(bt.value.Ib(), nextIfdOffsetToWrite) - log.PanicIf(err) - - ibe.pushToJournal("encodeTagToBytes", "<", "[%s]->[%s]", bt.value.Ib().ifdPath, ib.ifdPath) - - // Use the next-IFD offset for it. The IFD will actually get - // attached after we return. - err = bw.WriteUint32(nextIfdOffsetToWrite) - log.PanicIf(err) - - } else { - // No child-IFDs are to be allocated. Finish the entry with a NULL - // pointer. - - ibe.pushToJournal("encodeTagToBytes", "-", "*Not* descending to child: [%s]", bt.value.Ib().ifdPath) - - err = bw.WriteUint32(0) - log.PanicIf(err) - } - } - - return childIfdBlock, nil -} - -// encodeIfdToBytes encodes the given IB to a byte-slice. We are given the -// offset at which this IFD will be written. This method is used called both to -// pre-determine how big the table is going to be (so that we can calculate the -// address to allocate data at) as well as to write the final table. -// -// It is necessary to fully realize the table in order to predetermine its size -// because it is not enough to know the size of the table: If there are child -// IFDs, we will not be able to allocate them without first knowing how much -// data we need to allocate for the current IFD. -func (ibe *IfdByteEncoder) encodeIfdToBytes(ib *IfdBuilder, ifdAddressableOffset uint32, nextIfdOffsetToWrite uint32, setNextIb bool) (data []byte, tableSize uint32, dataSize uint32, childIfdSizes []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ibe.pushToJournal("encodeIfdToBytes", ">", "%s", ib) - - tableSize = ibe.TableSize(len(ib.tags)) - - b := new(bytes.Buffer) - bw := NewByteWriter(b, ib.byteOrder) - - // Write tag count. - err = bw.WriteUint16(uint16(len(ib.tags))) - log.PanicIf(err) - - ida := newIfdDataAllocator(ifdAddressableOffset) - - childIfdBlocks := make([][]byte, 0) - - // Write raw bytes for each tag entry. Allocate larger data to be referred - // to in the follow-up data-block as required. Any "unknown"-byte tags that - // we can't parse will not be present here (using AddTagsFromExisting(), at - // least). - for _, bt := range ib.tags { - childIfdBlock, err := ibe.encodeTagToBytes(ib, bt, bw, ida, nextIfdOffsetToWrite) - log.PanicIf(err) - - if childIfdBlock != nil { - // We aren't allowed to have non-nil child IFDs if we're just - // sizing things up. - if nextIfdOffsetToWrite == 0 { - log.Panicf("no IFD offset provided for child-IFDs; no new child-IFDs permitted") - } - - nextIfdOffsetToWrite += uint32(len(childIfdBlock)) - childIfdBlocks = append(childIfdBlocks, childIfdBlock) - } - } - - dataBytes := ida.Bytes() - dataSize = uint32(len(dataBytes)) - - childIfdSizes = make([]uint32, len(childIfdBlocks)) - childIfdsTotalSize := uint32(0) - for i, childIfdBlock := range childIfdBlocks { - len_ := uint32(len(childIfdBlock)) - childIfdSizes[i] = len_ - childIfdsTotalSize += len_ - } - - // N the link from this IFD to the next IFD that will be written in the - // next cycle. - if setNextIb == true { - // Write address of next IFD in chain. This will be the original - // allocation offset plus the size of everything we have allocated for - // this IFD and its child-IFDs. - // - // It is critical that this number is stepped properly. We experienced - // an issue whereby it first looked like we were duplicating the IFD and - // then that we were duplicating the tags in the wrong IFD, and then - // finally we determined that the next-IFD offset for the first IFD was - // accidentally pointing back to the EXIF IFD, so we were visiting it - // twice when visiting through the tags after decoding. It was an - // expensive bug to find. - - ibe.pushToJournal("encodeIfdToBytes", "-", "Setting 'next' IFD to (0x%08x).", nextIfdOffsetToWrite) - - err := bw.WriteUint32(nextIfdOffsetToWrite) - log.PanicIf(err) - } else { - err := bw.WriteUint32(0) - log.PanicIf(err) - } - - _, err = b.Write(dataBytes) - log.PanicIf(err) - - // Append any child IFD blocks after our table and data blocks. These IFDs - // were equipped with the appropriate offset information so it's expected - // that all offsets referred to by these will be correct. - // - // Note that child-IFDs are append after the current IFD and before the - // next IFD, as opposed to the root IFDs, which are chained together but - // will be interrupted by these child-IFDs (which is expected, per the - // standard). - - for _, childIfdBlock := range childIfdBlocks { - _, err = b.Write(childIfdBlock) - log.PanicIf(err) - } - - ibe.pushToJournal("encodeIfdToBytes", "<", "%s", ib) - - return b.Bytes(), tableSize, dataSize, childIfdSizes, nil -} - -// encodeAndAttachIfd is a reentrant function that processes the IFD chain. -func (ibe *IfdByteEncoder) encodeAndAttachIfd(ib *IfdBuilder, ifdAddressableOffset uint32) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ibe.pushToJournal("encodeAndAttachIfd", ">", "%s", ib) - - b := new(bytes.Buffer) - - i := 0 - - for thisIb := ib; thisIb != nil; thisIb = thisIb.nextIb { - - // Do a dry-run in order to pre-determine its size requirement. - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Beginning encoding process: (%d) [%s]", i, thisIb.ifdPath) - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Calculating size: (%d) [%s]", i, thisIb.ifdPath) - - _, tableSize, allocatedDataSize, _, err := ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, 0, false) - log.PanicIf(err) - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Finished calculating size: (%d) [%s]", i, thisIb.ifdPath) - - ifdAddressableOffset += tableSize - nextIfdOffsetToWrite := ifdAddressableOffset + allocatedDataSize - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Next IFD will be written at offset (0x%08x)", nextIfdOffsetToWrite) - - // Write our IFD as well as any child-IFDs (now that we know the offset - // where new IFDs and their data will be allocated). - - setNextIb := thisIb.nextIb != nil - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Encoding starting: (%d) [%s] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, thisIb.ifdPath, nextIfdOffsetToWrite) - - tableAndAllocated, effectiveTableSize, effectiveAllocatedDataSize, childIfdSizes, err := - ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, nextIfdOffsetToWrite, setNextIb) - - log.PanicIf(err) - - if effectiveTableSize != tableSize { - log.Panicf("written table size does not match the pre-calculated table size: (%d) != (%d) %s", effectiveTableSize, tableSize, ib) - } else if effectiveAllocatedDataSize != allocatedDataSize { - log.Panicf("written allocated-data size does not match the pre-calculated allocated-data size: (%d) != (%d) %s", effectiveAllocatedDataSize, allocatedDataSize, ib) - } - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Encoding done: (%d) [%s]", i, thisIb.ifdPath) - - totalChildIfdSize := uint32(0) - for _, childIfdSize := range childIfdSizes { - totalChildIfdSize += childIfdSize - } - - if len(tableAndAllocated) != int(tableSize+allocatedDataSize+totalChildIfdSize) { - log.Panicf("IFD table and data is not a consistent size: (%d) != (%d)", len(tableAndAllocated), tableSize+allocatedDataSize+totalChildIfdSize) - } - - // TODO(dustin): We might want to verify the original tableAndAllocated length, too. - - _, err = b.Write(tableAndAllocated) - log.PanicIf(err) - - // Advance past what we've allocated, thus far. - - ifdAddressableOffset += allocatedDataSize + totalChildIfdSize - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Finishing encoding process: (%d) [%s] [FINAL:] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, ib.ifdPath, nextIfdOffsetToWrite) - - i++ - } - - ibe.pushToJournal("encodeAndAttachIfd", "<", "%s", ib) - - return b.Bytes(), nil -} - -// EncodeToExifPayload is the base encoding step that transcribes the entire IB -// structure to its on-disk layout. -func (ibe *IfdByteEncoder) EncodeToExifPayload(ib *IfdBuilder) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - data, err = ibe.encodeAndAttachIfd(ib, ExifDefaultFirstIfdOffset) - log.PanicIf(err) - - return data, nil -} - -// EncodeToExif calls EncodeToExifPayload and then packages the result into a -// complete EXIF block. -func (ibe *IfdByteEncoder) EncodeToExif(ib *IfdBuilder) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - encodedIfds, err := ibe.EncodeToExifPayload(ib) - log.PanicIf(err) - - // Wrap the IFD in a formal EXIF block. - - b := new(bytes.Buffer) - - headerBytes, err := BuildExifHeader(ib.byteOrder, ExifDefaultFirstIfdOffset) - log.PanicIf(err) - - _, err = b.Write(headerBytes) - log.PanicIf(err) - - _, err = b.Write(encodedIfds) - log.PanicIf(err) - - return b.Bytes(), nil -} diff --git a/vendor/github.com/dsoprea/go-exif/ifd_enumerate.go b/vendor/github.com/dsoprea/go-exif/ifd_enumerate.go deleted file mode 100644 index 317e847a9..000000000 --- a/vendor/github.com/dsoprea/go-exif/ifd_enumerate.go +++ /dev/null @@ -1,1356 +0,0 @@ -package exif - -import ( - "bytes" - "errors" - "fmt" - "reflect" - "strconv" - "strings" - "time" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - ifdEnumerateLogger = log.NewLogger("exifjpeg.ifd") -) - -var ( - ErrNoThumbnail = errors.New("no thumbnail") - ErrNoGpsTags = errors.New("no gps tags") - ErrTagTypeNotValid = errors.New("tag type invalid") -) - -var ( - ValidGpsVersions = [][4]byte{ - {2, 2, 0, 0}, - - // Suddenly appeared at the default in 2.31: https://home.jeita.or.jp/tsc/std-pdf/CP-3451D.pdf - // - // Note that the presence of 2.3.0.0 doesn't seem to guarantee - // coordinates. In some cases, we seen just the following: - // - // GPS Tag Version |2.3.0.0 - // GPS Receiver Status |V - // Geodetic Survey Data|WGS-84 - // GPS Differential Cor|0 - // - {2, 3, 0, 0}, - } -) - -// IfdTagEnumerator knows how to decode an IFD and all of the tags it -// describes. -// -// The IFDs and the actual values can float throughout the EXIF block, but the -// IFD itself is just a minor header followed by a set of repeating, -// statically-sized records. So, the tags (though notnecessarily their values) -// are fairly simple to enumerate. -type IfdTagEnumerator struct { - byteOrder binary.ByteOrder - addressableData []byte - ifdOffset uint32 - buffer *bytes.Buffer -} - -func NewIfdTagEnumerator(addressableData []byte, byteOrder binary.ByteOrder, ifdOffset uint32) (ite *IfdTagEnumerator) { - ite = &IfdTagEnumerator{ - addressableData: addressableData, - byteOrder: byteOrder, - buffer: bytes.NewBuffer(addressableData[ifdOffset:]), - } - - return ite -} - -// getUint16 reads a uint16 and advances both our current and our current -// accumulator (which allows us to know how far to seek to the beginning of the -// next IFD when it's time to jump). -func (ife *IfdTagEnumerator) getUint16() (value uint16, raw []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - needBytes := 2 - offset := 0 - raw = make([]byte, needBytes) - - for offset < needBytes { - n, err := ife.buffer.Read(raw[offset:]) - log.PanicIf(err) - - offset += n - } - - value = ife.byteOrder.Uint16(raw) - - return value, raw, nil -} - -// getUint32 reads a uint32 and advances both our current and our current -// accumulator (which allows us to know how far to seek to the beginning of the -// next IFD when it's time to jump). -func (ife *IfdTagEnumerator) getUint32() (value uint32, raw []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - needBytes := 4 - offset := 0 - raw = make([]byte, needBytes) - - for offset < needBytes { - n, err := ife.buffer.Read(raw[offset:]) - log.PanicIf(err) - - offset += n - } - - value = ife.byteOrder.Uint32(raw) - - return value, raw, nil -} - -type IfdEnumerate struct { - exifData []byte - buffer *bytes.Buffer - byteOrder binary.ByteOrder - currentOffset uint32 - tagIndex *TagIndex - ifdMapping *IfdMapping -} - -func NewIfdEnumerate(ifdMapping *IfdMapping, tagIndex *TagIndex, exifData []byte, byteOrder binary.ByteOrder) *IfdEnumerate { - return &IfdEnumerate{ - exifData: exifData, - buffer: bytes.NewBuffer(exifData), - byteOrder: byteOrder, - ifdMapping: ifdMapping, - tagIndex: tagIndex, - } -} - -func (ie *IfdEnumerate) getTagEnumerator(ifdOffset uint32) (ite *IfdTagEnumerator) { - ite = NewIfdTagEnumerator( - ie.exifData[ExifAddressableAreaStart:], - ie.byteOrder, - ifdOffset) - - return ite -} - -func (ie *IfdEnumerate) parseTag(fqIfdPath string, tagPosition int, ite *IfdTagEnumerator, resolveValue bool) (tag *IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tagId, _, err := ite.getUint16() - log.PanicIf(err) - - tagTypeRaw, _, err := ite.getUint16() - log.PanicIf(err) - - tagType := TagTypePrimitive(tagTypeRaw) - - unitCount, _, err := ite.getUint32() - log.PanicIf(err) - - valueOffset, rawValueOffset, err := ite.getUint32() - log.PanicIf(err) - - if _, found := TypeNames[tagType]; found == false { - log.Panic(ErrTagTypeNotValid) - } - - ifdPath, err := ie.ifdMapping.StripPathPhraseIndices(fqIfdPath) - log.PanicIf(err) - - tag = &IfdTagEntry{ - IfdPath: ifdPath, - TagId: tagId, - TagIndex: tagPosition, - TagType: tagType, - UnitCount: unitCount, - ValueOffset: valueOffset, - RawValueOffset: rawValueOffset, - } - - if resolveValue == true { - value, isUnhandledUnknown, err := ie.resolveTagValue(tag) - log.PanicIf(err) - - tag.value = value - tag.isUnhandledUnknown = isUnhandledUnknown - } - - // If it's an IFD but not a standard one, it'll just be seen as a LONG - // (the standard IFD tag type), later, unless we skip it because it's - // [likely] not even in the standard list of known tags. - mi, err := ie.ifdMapping.GetChild(ifdPath, tagId) - if err == nil { - tag.ChildIfdName = mi.Name - tag.ChildIfdPath = mi.PathPhrase() - tag.ChildFqIfdPath = fmt.Sprintf("%s/%s", fqIfdPath, mi.Name) - - // We also need to set `tag.ChildFqIfdPath` but can't do it here - // because we don't have the IFD index. - } else if log.Is(err, ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - return tag, nil -} - -func (ie *IfdEnumerate) GetValueContext(ite *IfdTagEntry) *ValueContext { - - // TODO(dustin): Add test - - addressableData := ie.exifData[ExifAddressableAreaStart:] - - return newValueContextFromTag( - ite, - addressableData, - ie.byteOrder) -} - -func (ie *IfdEnumerate) resolveTagValue(ite *IfdTagEntry) (valueBytes []byte, isUnhandledUnknown bool, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - addressableData := ie.exifData[ExifAddressableAreaStart:] - - // Return the exact bytes of the unknown-type value. Returning a string - // (`ValueString`) is easy because we can just pass everything to - // `Sprintf()`. Returning the raw, typed value (`Value`) is easy - // (obviously). However, here, in order to produce the list of bytes, we - // need to coerce whatever `Undefined()` returns. - if ite.TagType == TypeUndefined { - valueContext := ie.GetValueContext(ite) - - value, err := valueContext.Undefined() - if err != nil { - if err == ErrUnhandledUnknownTypedTag { - valueBytes = []byte(UnparseableUnknownTagValuePlaceholder) - return valueBytes, true, nil - } - - log.Panic(err) - } else { - switch value.(type) { - case []byte: - return value.([]byte), false, nil - case TagUnknownType_UnknownValue: - b := []byte(value.(TagUnknownType_UnknownValue)) - return b, false, nil - case string: - return []byte(value.(string)), false, nil - case UnknownTagValue: - valueBytes, err := value.(UnknownTagValue).ValueBytes() - log.PanicIf(err) - - return valueBytes, false, nil - default: - // TODO(dustin): !! Finish translating the rest of the types (make reusable and replace into other similar implementations?) - log.Panicf("can not produce bytes for unknown-type tag (0x%04x) (1): [%s]", ite.TagId, reflect.TypeOf(value)) - } - } - } else { - originalType := NewTagType(ite.TagType, ie.byteOrder) - byteCount := uint32(originalType.Type().Size()) * ite.UnitCount - - tt := NewTagType(TypeByte, ie.byteOrder) - - if tt.valueIsEmbedded(byteCount) == true { - iteLogger.Debugf(nil, "Reading BYTE value (ITE; embedded).") - - // In this case, the bytes normally used for the offset are actually - // data. - valueBytes, err = tt.ParseBytes(ite.RawValueOffset, byteCount) - log.PanicIf(err) - } else { - iteLogger.Debugf(nil, "Reading BYTE value (ITE; at offset).") - - valueBytes, err = tt.ParseBytes(addressableData[ite.ValueOffset:], byteCount) - log.PanicIf(err) - } - } - - return valueBytes, false, nil -} - -// RawTagVisitorPtr is an optional callback that can get hit for every tag we parse -// through. `addressableData` is the byte array startign after the EXIF header -// (where the offsets of all IFDs and values are calculated from). -// -// This was reimplemented as an interface to allow for simpler change management -// in the future. -type RawTagWalk interface { - Visit(fqIfdPath string, ifdIndex int, tagId uint16, tagType TagType, valueContext *ValueContext) (err error) -} - -type RawTagWalkLegacyWrapper struct { - legacyVisitor RawTagVisitor -} - -func (rtwlw RawTagWalkLegacyWrapper) Visit(fqIfdPath string, ifdIndex int, tagId uint16, tagType TagType, valueContext *ValueContext) (err error) { - return rtwlw.legacyVisitor(fqIfdPath, ifdIndex, tagId, tagType, *valueContext) -} - -// RawTagVisitor is an optional callback that can get hit for every tag we parse -// through. `addressableData` is the byte array startign after the EXIF header -// (where the offsets of all IFDs and values are calculated from). -// -// DEPRECATED(dustin): Use a RawTagWalk instead. -type RawTagVisitor func(fqIfdPath string, ifdIndex int, tagId uint16, tagType TagType, valueContext ValueContext) (err error) - -// ParseIfd decodes the IFD block that we're currently sitting on the first -// byte of. -func (ie *IfdEnumerate) ParseIfd(fqIfdPath string, ifdIndex int, ite *IfdTagEnumerator, visitor interface{}, doDescend bool, resolveValues bool) (nextIfdOffset uint32, entries []*IfdTagEntry, thumbnailData []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - var visitorWrapper RawTagWalk - - if visitor != nil { - var ok bool - - visitorWrapper, ok = visitor.(RawTagWalk) - if ok == false { - // Legacy usage. - - // `ok` can be `true` but `legacyVisitor` can still be `nil` (when - // passed as nil). - if legacyVisitor, ok := visitor.(RawTagVisitor); ok == true && legacyVisitor != nil { - visitorWrapper = RawTagWalkLegacyWrapper{ - legacyVisitor: legacyVisitor, - } - } - } - } - - tagCount, _, err := ite.getUint16() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Current IFD tag-count: (%d)", tagCount) - - entries = make([]*IfdTagEntry, 0) - - var iteThumbnailOffset *IfdTagEntry - var iteThumbnailSize *IfdTagEntry - - for i := 0; i < int(tagCount); i++ { - tag, err := ie.parseTag(fqIfdPath, i, ite, resolveValues) - if err != nil { - if log.Is(err, ErrTagTypeNotValid) == true { - ifdEnumerateLogger.Warningf(nil, "Tag in IFD [%s] at position (%d) has invalid type and will be skipped.", fqIfdPath, i) - continue - } - - log.Panic(err) - } - - if tag.TagId == ThumbnailOffsetTagId { - iteThumbnailOffset = tag - - continue - } else if tag.TagId == ThumbnailSizeTagId { - iteThumbnailSize = tag - continue - } - - if visitorWrapper != nil { - tt := NewTagType(tag.TagType, ie.byteOrder) - - valueContext := ie.GetValueContext(tag) - - err := visitorWrapper.Visit(fqIfdPath, ifdIndex, tag.TagId, tt, valueContext) - log.PanicIf(err) - } - - // If it's an IFD but not a standard one, it'll just be seen as a LONG - // (the standard IFD tag type), later, unless we skip it because it's - // [likely] not even in the standard list of known tags. - if tag.ChildIfdPath != "" { - if doDescend == true { - ifdEnumerateLogger.Debugf(nil, "Descending to IFD [%s].", tag.ChildIfdPath) - - err := ie.scan(tag.ChildFqIfdPath, tag.ValueOffset, visitor, resolveValues) - log.PanicIf(err) - } - } - - entries = append(entries, tag) - } - - if iteThumbnailOffset != nil && iteThumbnailSize != nil { - thumbnailData, err = ie.parseThumbnail(iteThumbnailOffset, iteThumbnailSize) - log.PanicIf(err) - } - - nextIfdOffset, _, err = ite.getUint32() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Next IFD at offset: (%08x)", nextIfdOffset) - - return nextIfdOffset, entries, thumbnailData, nil -} - -func (ie *IfdEnumerate) parseThumbnail(offsetIte, lengthIte *IfdTagEntry) (thumbnailData []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - addressableData := ie.exifData[ExifAddressableAreaStart:] - - vRaw, err := lengthIte.Value(addressableData, ie.byteOrder) - log.PanicIf(err) - - vList := vRaw.([]uint32) - if len(vList) != 1 { - log.Panicf("not exactly one long: (%d)", len(vList)) - } - - length := vList[0] - - // The tag is official a LONG type, but it's actually an offset to a blob of bytes. - offsetIte.TagType = TypeByte - offsetIte.UnitCount = length - - thumbnailData, err = offsetIte.ValueBytes(addressableData, ie.byteOrder) - log.PanicIf(err) - - return thumbnailData, nil -} - -// Scan enumerates the different EXIF's IFD blocks. -func (ie *IfdEnumerate) scan(fqIfdName string, ifdOffset uint32, visitor interface{}, resolveValues bool) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for ifdIndex := 0; ; ifdIndex++ { - ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] (%d) at offset (%04x).", fqIfdName, ifdIndex, ifdOffset) - ite := ie.getTagEnumerator(ifdOffset) - - nextIfdOffset, _, _, err := ie.ParseIfd(fqIfdName, ifdIndex, ite, visitor, true, resolveValues) - log.PanicIf(err) - - if nextIfdOffset == 0 { - break - } - - ifdOffset = nextIfdOffset - } - - return nil -} - -// Scan enumerates the different EXIF blocks (called IFDs). `rootIfdName` will -// be "IFD" in the TIFF standard. -func (ie *IfdEnumerate) Scan(rootIfdName string, ifdOffset uint32, visitor RawTagVisitor, resolveValue bool) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = ie.scan(rootIfdName, ifdOffset, visitor, resolveValue) - log.PanicIf(err) - - return nil -} - -// Ifd represents a single parsed IFD. -type Ifd struct { - - // TODO(dustin): !! Why are all of these public? Privatize them and then add NextIfd(). - - // This is just for convenience, just so that we can easily get the values - // and not involve other projects in semantics that they won't otherwise - // need to know. - addressableData []byte - - ByteOrder binary.ByteOrder - - // Name is the name of the IFD (the rightmost name in the path, sans any - // indices). - Name string - - // IfdPath is a simple IFD path (e.g. IFD/GPSInfo). No indices. - IfdPath string - - // FqIfdPath is a fully-qualified IFD path (e.g. IFD0/GPSInfo0). With - // indices. - FqIfdPath string - - TagId uint16 - - Id int - - ParentIfd *Ifd - - // ParentTagIndex is our tag position in the parent IFD, if we had a parent - // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling - // instead of as a child). - ParentTagIndex int - - // Name string - Index int - Offset uint32 - - Entries []*IfdTagEntry - EntriesByTagId map[uint16][]*IfdTagEntry - - Children []*Ifd - - ChildIfdIndex map[string]*Ifd - - NextIfdOffset uint32 - NextIfd *Ifd - - thumbnailData []byte - - ifdMapping *IfdMapping - tagIndex *TagIndex -} - -func (ifd *Ifd) ChildWithIfdPath(ifdPath string) (childIfd *Ifd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for _, childIfd := range ifd.Children { - if childIfd.IfdPath == ifdPath { - return childIfd, nil - } - } - - log.Panic(ErrTagNotFound) - return nil, nil -} - -func (ifd *Ifd) TagValue(ite *IfdTagEntry) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - value, err = ite.Value(ifd.addressableData, ifd.ByteOrder) - log.PanicIf(err) - - return value, nil -} - -func (ifd *Ifd) TagValueBytes(ite *IfdTagEntry) (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - value, err = ite.ValueBytes(ifd.addressableData, ifd.ByteOrder) - log.PanicIf(err) - - return value, nil -} - -// FindTagWithId returns a list of tags (usually just zero or one) that match -// the given tag ID. This is efficient. -func (ifd *Ifd) FindTagWithId(tagId uint16) (results []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - results, found := ifd.EntriesByTagId[tagId] - if found != true { - log.Panic(ErrTagNotFound) - } - - return results, nil -} - -// FindTagWithName returns a list of tags (usually just zero or one) that match -// the given tag name. This is not efficient (though the labor is trivial). -func (ifd *Ifd) FindTagWithName(tagName string) (results []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ifd.tagIndex.GetWithName(ifd.IfdPath, tagName) - if log.Is(err, ErrTagNotFound) == true { - log.Panic(ErrTagNotStandard) - } else if err != nil { - log.Panic(err) - } - - results = make([]*IfdTagEntry, 0) - for _, ite := range ifd.Entries { - if ite.TagId == it.Id { - results = append(results, ite) - } - } - - if len(results) == 0 { - log.Panic(ErrTagNotFound) - } - - return results, nil -} - -func (ifd Ifd) String() string { - parentOffset := uint32(0) - if ifd.ParentIfd != nil { - parentOffset = ifd.ParentIfd.Offset - } - - return fmt.Sprintf("Ifd", ifd.Id, ifd.IfdPath, ifd.Index, len(ifd.Entries), ifd.Offset, len(ifd.Children), parentOffset, ifd.NextIfdOffset) -} - -func (ifd *Ifd) Thumbnail() (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if ifd.thumbnailData == nil { - log.Panic(ErrNoThumbnail) - } - - return ifd.thumbnailData, nil -} - -func (ifd *Ifd) dumpTags(tags []*IfdTagEntry) []*IfdTagEntry { - if tags == nil { - tags = make([]*IfdTagEntry, 0) - } - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, tag := range ifd.Entries { - tags = append(tags, tag) - - if tag.ChildIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[tag.ChildIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", tag.ChildIfdPath) - } - - tags = childIfd.dumpTags(tags) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - tags = ifd.NextIfd.dumpTags(tags) - } - - return tags -} - -// DumpTags prints the IFD hierarchy. -func (ifd *Ifd) DumpTags() []*IfdTagEntry { - return ifd.dumpTags(nil) -} - -func (ifd *Ifd) printTagTree(populateValues bool, index, level int, nextLink bool) { - indent := strings.Repeat(" ", level*2) - - prefix := " " - if nextLink { - prefix = ">" - } - - fmt.Printf("%s%sIFD: %s\n", indent, prefix, ifd) - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, tag := range ifd.Entries { - if tag.ChildIfdPath != "" { - fmt.Printf("%s - TAG: %s\n", indent, tag) - } else { - it, err := ifd.tagIndex.Get(ifd.IfdPath, tag.TagId) - - tagName := "" - if err == nil { - tagName = it.Name - } - - var value interface{} - if populateValues == true { - var err error - - value, err = ifd.TagValue(tag) - if err != nil { - if err == ErrUnhandledUnknownTypedTag { - value = UnparseableUnknownTagValuePlaceholder - } else { - log.Panic(err) - } - } - } - - fmt.Printf("%s - TAG: %s NAME=[%s] VALUE=[%v]\n", indent, tag, tagName, value) - } - - if tag.ChildIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[tag.ChildIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", tag.ChildIfdPath) - } - - childIfd.printTagTree(populateValues, 0, level+1, false) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - ifd.NextIfd.printTagTree(populateValues, index+1, level, true) - } -} - -// PrintTagTree prints the IFD hierarchy. -func (ifd *Ifd) PrintTagTree(populateValues bool) { - ifd.printTagTree(populateValues, 0, 0, false) -} - -func (ifd *Ifd) printIfdTree(level int, nextLink bool) { - indent := strings.Repeat(" ", level*2) - - prefix := " " - if nextLink { - prefix = ">" - } - - fmt.Printf("%s%s%s\n", indent, prefix, ifd) - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, tag := range ifd.Entries { - if tag.ChildIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[tag.ChildIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", tag.ChildIfdPath) - } - - childIfd.printIfdTree(level+1, false) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - ifd.NextIfd.printIfdTree(level, true) - } -} - -// PrintIfdTree prints the IFD hierarchy. -func (ifd *Ifd) PrintIfdTree() { - ifd.printIfdTree(0, false) -} - -func (ifd *Ifd) dumpTree(tagsDump []string, level int) []string { - if tagsDump == nil { - tagsDump = make([]string, 0) - } - - indent := strings.Repeat(" ", level*2) - - var ifdPhrase string - if ifd.ParentIfd != nil { - ifdPhrase = fmt.Sprintf("[%s]->[%s]:(%d)", ifd.ParentIfd.IfdPath, ifd.IfdPath, ifd.Index) - } else { - ifdPhrase = fmt.Sprintf("[ROOT]->[%s]:(%d)", ifd.IfdPath, ifd.Index) - } - - startBlurb := fmt.Sprintf("%s> IFD %s TOP", indent, ifdPhrase) - tagsDump = append(tagsDump, startBlurb) - - ifdsFoundCount := 0 - for _, tag := range ifd.Entries { - tagsDump = append(tagsDump, fmt.Sprintf("%s - (0x%04x)", indent, tag.TagId)) - - if tag.ChildIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[tag.ChildIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", tag.ChildIfdPath) - } - - tagsDump = childIfd.dumpTree(tagsDump, level+1) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - finishBlurb := fmt.Sprintf("%s< IFD %s BOTTOM", indent, ifdPhrase) - tagsDump = append(tagsDump, finishBlurb) - - if ifd.NextIfd != nil { - siblingBlurb := fmt.Sprintf("%s* LINKING TO SIBLING IFD [%s]:(%d)", indent, ifd.NextIfd.IfdPath, ifd.NextIfd.Index) - tagsDump = append(tagsDump, siblingBlurb) - - tagsDump = ifd.NextIfd.dumpTree(tagsDump, level) - } - - return tagsDump -} - -// DumpTree returns a list of strings describing the IFD hierarchy. -func (ifd *Ifd) DumpTree() []string { - return ifd.dumpTree(nil, 0) -} - -// GpsInfo parses and consolidates the GPS info. This can only be called on the -// GPS IFD. -func (ifd *Ifd) GpsInfo() (gi *GpsInfo, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Also add functionality to update the GPS info. - - gi = new(GpsInfo) - - if ifd.IfdPath != IfdPathStandardGps { - log.Panicf("GPS can only be read on GPS IFD: [%s] != [%s]", ifd.IfdPath, IfdPathStandardGps) - } - - if tags, found := ifd.EntriesByTagId[TagVersionId]; found == false { - // We've seen this. We'll just have to default to assuming we're in a - // 2.2.0.0 format. - ifdEnumerateLogger.Warningf(nil, "No GPS version tag (0x%04x) found.", TagVersionId) - } else { - hit := false - for _, acceptedGpsVersion := range ValidGpsVersions { - if bytes.Compare(tags[0].value, acceptedGpsVersion[:]) == 0 { - hit = true - break - } - } - - if hit != true { - ifdEnumerateLogger.Warningf(nil, "GPS version not supported: %v", tags[0].value) - log.Panic(ErrNoGpsTags) - } - } - - tags, found := ifd.EntriesByTagId[TagLatitudeId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "latitude not found") - log.Panic(ErrNoGpsTags) - } - - latitudeValue, err := ifd.TagValue(tags[0]) - log.PanicIf(err) - - // Look for whether North or South. - tags, found = ifd.EntriesByTagId[TagLatitudeRefId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "latitude-ref not found") - log.Panic(ErrNoGpsTags) - } - - latitudeRefValue, err := ifd.TagValue(tags[0]) - log.PanicIf(err) - - tags, found = ifd.EntriesByTagId[TagLongitudeId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "longitude not found") - log.Panic(ErrNoGpsTags) - } - - longitudeValue, err := ifd.TagValue(tags[0]) - log.PanicIf(err) - - // Look for whether West or East. - tags, found = ifd.EntriesByTagId[TagLongitudeRefId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "longitude-ref not found") - log.Panic(ErrNoGpsTags) - } - - longitudeRefValue, err := ifd.TagValue(tags[0]) - log.PanicIf(err) - - // Parse location. - - latitudeRaw := latitudeValue.([]Rational) - - gi.Latitude = GpsDegrees{ - Orientation: latitudeRefValue.(string)[0], - Degrees: float64(latitudeRaw[0].Numerator) / float64(latitudeRaw[0].Denominator), - Minutes: float64(latitudeRaw[1].Numerator) / float64(latitudeRaw[1].Denominator), - Seconds: float64(latitudeRaw[2].Numerator) / float64(latitudeRaw[2].Denominator), - } - - longitudeRaw := longitudeValue.([]Rational) - - gi.Longitude = GpsDegrees{ - Orientation: longitudeRefValue.(string)[0], - Degrees: float64(longitudeRaw[0].Numerator) / float64(longitudeRaw[0].Denominator), - Minutes: float64(longitudeRaw[1].Numerator) / float64(longitudeRaw[1].Denominator), - Seconds: float64(longitudeRaw[2].Numerator) / float64(longitudeRaw[2].Denominator), - } - - // Parse altitude. - - altitudeTags, foundAltitude := ifd.EntriesByTagId[TagAltitudeId] - altitudeRefTags, foundAltitudeRef := ifd.EntriesByTagId[TagAltitudeRefId] - - if foundAltitude == true && foundAltitudeRef == true { - altitudeValue, err := ifd.TagValue(altitudeTags[0]) - log.PanicIf(err) - - altitudeRefValue, err := ifd.TagValue(altitudeRefTags[0]) - log.PanicIf(err) - - altitudeRaw := altitudeValue.([]Rational) - altitude := int(altitudeRaw[0].Numerator / altitudeRaw[0].Denominator) - if altitudeRefValue.([]byte)[0] == 1 { - altitude *= -1 - } - - gi.Altitude = altitude - } - - // Parse time. - - timestampTags, foundTimestamp := ifd.EntriesByTagId[TagTimestampId] - datestampTags, foundDatestamp := ifd.EntriesByTagId[TagDatestampId] - - if foundTimestamp == true && foundDatestamp == true { - datestampValue, err := ifd.TagValue(datestampTags[0]) - log.PanicIf(err) - - dateParts := strings.Split(datestampValue.(string), ":") - - year, err1 := strconv.ParseUint(dateParts[0], 10, 16) - month, err2 := strconv.ParseUint(dateParts[1], 10, 8) - day, err3 := strconv.ParseUint(dateParts[2], 10, 8) - - if err1 == nil && err2 == nil && err3 == nil { - timestampValue, err := ifd.TagValue(timestampTags[0]) - log.PanicIf(err) - - timestampRaw := timestampValue.([]Rational) - - hour := int(timestampRaw[0].Numerator / timestampRaw[0].Denominator) - minute := int(timestampRaw[1].Numerator / timestampRaw[1].Denominator) - second := int(timestampRaw[2].Numerator / timestampRaw[2].Denominator) - - gi.Timestamp = time.Date(int(year), time.Month(month), int(day), hour, minute, second, 0, time.UTC) - } - } - - return gi, nil -} - -type ParsedTagVisitor func(*Ifd, *IfdTagEntry) error - -func (ifd *Ifd) EnumerateTagsRecursively(visitor ParsedTagVisitor) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for ptr := ifd; ptr != nil; ptr = ptr.NextIfd { - for _, ite := range ifd.Entries { - if ite.ChildIfdPath != "" { - childIfd := ifd.ChildIfdIndex[ite.ChildIfdPath] - - err := childIfd.EnumerateTagsRecursively(visitor) - log.PanicIf(err) - } else { - err := visitor(ifd, ite) - log.PanicIf(err) - } - } - } - - return nil -} - -func (ifd *Ifd) GetValueContext(ite *IfdTagEntry) *ValueContext { - return newValueContextFromTag( - ite, - ifd.addressableData, - ifd.ByteOrder) -} - -type QueuedIfd struct { - Name string - IfdPath string - FqIfdPath string - - TagId uint16 - - Index int - Offset uint32 - Parent *Ifd - - // ParentTagIndex is our tag position in the parent IFD, if we had a parent - // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling - // instead of as a child). - ParentTagIndex int -} - -type IfdIndex struct { - RootIfd *Ifd - Ifds []*Ifd - Tree map[int]*Ifd - Lookup map[string][]*Ifd -} - -// Scan enumerates the different EXIF blocks (called IFDs). -func (ie *IfdEnumerate) Collect(rootIfdOffset uint32, resolveValues bool) (index IfdIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tree := make(map[int]*Ifd) - ifds := make([]*Ifd, 0) - lookup := make(map[string][]*Ifd) - - queue := []QueuedIfd{ - { - Name: IfdStandard, - IfdPath: IfdStandard, - FqIfdPath: IfdStandard, - - TagId: 0xffff, - - Index: 0, - Offset: rootIfdOffset, - }, - } - - edges := make(map[uint32]*Ifd) - - for { - if len(queue) == 0 { - break - } - - qi := queue[0] - - name := qi.Name - ifdPath := qi.IfdPath - fqIfdPath := qi.FqIfdPath - - index := qi.Index - offset := qi.Offset - parentIfd := qi.Parent - - queue = queue[1:] - - ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] (%d) at offset (%04x).", ifdPath, index, offset) - ite := ie.getTagEnumerator(offset) - - nextIfdOffset, entries, thumbnailData, err := ie.ParseIfd(fqIfdPath, index, ite, nil, false, resolveValues) - log.PanicIf(err) - - id := len(ifds) - - entriesByTagId := make(map[uint16][]*IfdTagEntry) - for _, tag := range entries { - tags, found := entriesByTagId[tag.TagId] - if found == false { - tags = make([]*IfdTagEntry, 0) - } - - entriesByTagId[tag.TagId] = append(tags, tag) - } - - ifd := &Ifd{ - addressableData: ie.exifData[ExifAddressableAreaStart:], - - ByteOrder: ie.byteOrder, - - Name: name, - IfdPath: ifdPath, - FqIfdPath: fqIfdPath, - - TagId: qi.TagId, - - Id: id, - - ParentIfd: parentIfd, - ParentTagIndex: qi.ParentTagIndex, - - Index: index, - Offset: offset, - Entries: entries, - EntriesByTagId: entriesByTagId, - - // This is populated as each child is processed. - Children: make([]*Ifd, 0), - - NextIfdOffset: nextIfdOffset, - thumbnailData: thumbnailData, - - ifdMapping: ie.ifdMapping, - tagIndex: ie.tagIndex, - } - - // Add ourselves to a big list of IFDs. - ifds = append(ifds, ifd) - - // Install ourselves into a by-id lookup table (keys are unique). - tree[id] = ifd - - // Install into by-name buckets. - - if list_, found := lookup[ifdPath]; found == true { - lookup[ifdPath] = append(list_, ifd) - } else { - list_ = make([]*Ifd, 1) - list_[0] = ifd - - lookup[ifdPath] = list_ - } - - // Add a link from the previous IFD in the chain to us. - if previousIfd, found := edges[offset]; found == true { - previousIfd.NextIfd = ifd - } - - // Attach as a child to our parent (where we appeared as a tag in - // that IFD). - if parentIfd != nil { - parentIfd.Children = append(parentIfd.Children, ifd) - } - - // Determine if any of our entries is a child IFD and queue it. - for i, entry := range entries { - if entry.ChildIfdPath == "" { - continue - } - - qi := QueuedIfd{ - Name: entry.ChildIfdName, - IfdPath: entry.ChildIfdPath, - FqIfdPath: entry.ChildFqIfdPath, - TagId: entry.TagId, - - Index: 0, - Offset: entry.ValueOffset, - Parent: ifd, - ParentTagIndex: i, - } - - queue = append(queue, qi) - } - - // If there's another IFD in the chain. - if nextIfdOffset != 0 { - // Allow the next link to know what the previous link was. - edges[nextIfdOffset] = ifd - - siblingIndex := index + 1 - - var fqIfdPath string - if parentIfd != nil { - fqIfdPath = fmt.Sprintf("%s/%s%d", parentIfd.FqIfdPath, name, siblingIndex) - } else { - fqIfdPath = fmt.Sprintf("%s%d", name, siblingIndex) - } - - qi := QueuedIfd{ - Name: name, - IfdPath: ifdPath, - FqIfdPath: fqIfdPath, - TagId: 0xffff, - Index: siblingIndex, - Offset: nextIfdOffset, - } - - queue = append(queue, qi) - } - } - - index.RootIfd = tree[0] - index.Ifds = ifds - index.Tree = tree - index.Lookup = lookup - - err = ie.setChildrenIndex(index.RootIfd) - log.PanicIf(err) - - return index, nil -} - -func (ie *IfdEnumerate) setChildrenIndex(ifd *Ifd) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - childIfdIndex := make(map[string]*Ifd) - for _, childIfd := range ifd.Children { - childIfdIndex[childIfd.IfdPath] = childIfd - } - - ifd.ChildIfdIndex = childIfdIndex - - for _, childIfd := range ifd.Children { - err := ie.setChildrenIndex(childIfd) - log.PanicIf(err) - } - - return nil -} - -// ParseOneIfd is a hack to use an IE to parse a raw IFD block. Can be used for -// testing. -func ParseOneIfd(ifdMapping *IfdMapping, tagIndex *TagIndex, fqIfdPath, ifdPath string, byteOrder binary.ByteOrder, ifdBlock []byte, visitor RawTagVisitor, resolveValues bool) (nextIfdOffset uint32, entries []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ie := NewIfdEnumerate(ifdMapping, tagIndex, make([]byte, 0), byteOrder) - ite := NewIfdTagEnumerator(ifdBlock, byteOrder, 0) - - nextIfdOffset, entries, _, err = ie.ParseIfd(fqIfdPath, 0, ite, visitor, true, resolveValues) - log.PanicIf(err) - - return nextIfdOffset, entries, nil -} - -// ParseOneTag is a hack to use an IE to parse a raw tag block. -func ParseOneTag(ifdMapping *IfdMapping, tagIndex *TagIndex, fqIfdPath, ifdPath string, byteOrder binary.ByteOrder, tagBlock []byte, resolveValue bool) (tag *IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ie := NewIfdEnumerate(ifdMapping, tagIndex, make([]byte, 0), byteOrder) - ite := NewIfdTagEnumerator(tagBlock, byteOrder, 0) - - tag, err = ie.parseTag(fqIfdPath, 0, ite, resolveValue) - log.PanicIf(err) - - return tag, nil -} - -func FindIfdFromRootIfd(rootIfd *Ifd, ifdPath string) (ifd *Ifd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - lineage, err := rootIfd.ifdMapping.ResolvePath(ifdPath) - log.PanicIf(err) - - // Confirm the first IFD is our root IFD type, and then prune it because - // from then on we'll be searching down through our children. - - if len(lineage) == 0 { - log.Panicf("IFD path must be non-empty.") - } else if lineage[0].Name != IfdStandard { - log.Panicf("First IFD path item must be [%s].", IfdStandard) - } - - desiredRootIndex := lineage[0].Index - lineage = lineage[1:] - - // TODO(dustin): !! This is a poorly conceived fix that just doubles the work we already have to do below, which then interacts badly with the indices not being properly represented in the IFD-phrase. - // TODO(dustin): !! <-- However, we're not sure whether we shouldn't store a secondary IFD-path with the indices. Some IFDs may not necessarily restrict which IFD indices they can be a child of (only the IFD itself matters). Validation should be delegated to the caller. - thisIfd := rootIfd - for currentRootIndex := 0; currentRootIndex < desiredRootIndex; currentRootIndex++ { - if thisIfd.NextIfd == nil { - log.Panicf("Root-IFD index (%d) does not exist in the data.", currentRootIndex) - } - - thisIfd = thisIfd.NextIfd - } - - for i, itii := range lineage { - var hit *Ifd - for _, childIfd := range thisIfd.Children { - if childIfd.TagId == itii.TagId { - hit = childIfd - break - } - } - - // If we didn't find the child, add it. - if hit == nil { - log.Panicf("IFD [%s] in [%s] not found: %s", itii.Name, ifdPath, thisIfd.Children) - } - - thisIfd = hit - - // If we didn't find the sibling, add it. - for i = 0; i < itii.Index; i++ { - if thisIfd.NextIfd == nil { - log.Panicf("IFD [%s] does not have (%d) occurrences/siblings\n", thisIfd.IfdPath, itii.Index) - } - - thisIfd = thisIfd.NextIfd - } - } - - return thisIfd, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/ifd_tag_entry.go b/vendor/github.com/dsoprea/go-exif/ifd_tag_entry.go deleted file mode 100644 index 59e79ccf7..000000000 --- a/vendor/github.com/dsoprea/go-exif/ifd_tag_entry.go +++ /dev/null @@ -1,233 +0,0 @@ -package exif - -import ( - "fmt" - "reflect" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - iteLogger = log.NewLogger("exif.ifd_tag_entry") -) - -type IfdTagEntry struct { - TagId uint16 - TagIndex int - TagType TagTypePrimitive - UnitCount uint32 - ValueOffset uint32 - RawValueOffset []byte - - // ChildIfdName is the right most atom in the IFD-path. We need this to - // construct the fully-qualified IFD-path. - ChildIfdName string - - // ChildIfdPath is the IFD-path of the child if this tag represents a child - // IFD. - ChildIfdPath string - - // ChildFqIfdPath is the IFD-path of the child if this tag represents a - // child IFD. Includes indices. - ChildFqIfdPath string - - // TODO(dustin): !! IB's host the child-IBs directly in the tag, but that's not the case here. Refactor to accomodate it for a consistent experience. - - // IfdPath is the IFD that this tag belongs to. - IfdPath string - - // TODO(dustin): !! We now parse and read the value immediately. Update the rest of the logic to use this and get rid of all of the staggered and different resolution mechanisms. - value []byte - isUnhandledUnknown bool -} - -func (ite *IfdTagEntry) String() string { - return fmt.Sprintf("IfdTagEntry", ite.IfdPath, ite.TagId, TypeNames[ite.TagType], ite.UnitCount) -} - -// TODO(dustin): TODO(dustin): Stop exporting IfdPath and TagId. -// -// func (ite *IfdTagEntry) IfdPath() string { -// return ite.IfdPath -// } - -// TODO(dustin): TODO(dustin): Stop exporting IfdPath and TagId. -// -// func (ite *IfdTagEntry) TagId() uint16 { -// return ite.TagId -// } - -// ValueString renders a string from whatever the value in this tag is. -func (ite *IfdTagEntry) ValueString(addressableData []byte, byteOrder binary.ByteOrder) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext := - newValueContextFromTag( - ite, - addressableData, - byteOrder) - - if ite.TagType == TypeUndefined { - valueRaw, err := valueContext.Undefined() - log.PanicIf(err) - - value = fmt.Sprintf("%v", valueRaw) - } else { - value, err = valueContext.Format() - log.PanicIf(err) - } - - return value, nil -} - -// ValueBytes renders a specific list of bytes from the value in this tag. -func (ite *IfdTagEntry) ValueBytes(addressableData []byte, byteOrder binary.ByteOrder) (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Return the exact bytes of the unknown-type value. Returning a string - // (`ValueString`) is easy because we can just pass everything to - // `Sprintf()`. Returning the raw, typed value (`Value`) is easy - // (obviously). However, here, in order to produce the list of bytes, we - // need to coerce whatever `Undefined()` returns. - if ite.TagType == TypeUndefined { - valueContext := - newValueContextFromTag( - ite, - addressableData, - byteOrder) - - value, err := valueContext.Undefined() - log.PanicIf(err) - - switch value.(type) { - case []byte: - return value.([]byte), nil - case TagUnknownType_UnknownValue: - b := []byte(value.(TagUnknownType_UnknownValue)) - return b, nil - case string: - return []byte(value.(string)), nil - case UnknownTagValue: - valueBytes, err := value.(UnknownTagValue).ValueBytes() - log.PanicIf(err) - - return valueBytes, nil - default: - // TODO(dustin): !! Finish translating the rest of the types (make reusable and replace into other similar implementations?) - log.Panicf("can not produce bytes for unknown-type tag (0x%04x) (2): [%s]", ite.TagId, reflect.TypeOf(value)) - } - } - - originalType := NewTagType(ite.TagType, byteOrder) - byteCount := uint32(originalType.Type().Size()) * ite.UnitCount - - tt := NewTagType(TypeByte, byteOrder) - - if tt.valueIsEmbedded(byteCount) == true { - iteLogger.Debugf(nil, "Reading BYTE value (ITE; embedded).") - - // In this case, the bytes normally used for the offset are actually - // data. - value, err = tt.ParseBytes(ite.RawValueOffset, byteCount) - log.PanicIf(err) - } else { - iteLogger.Debugf(nil, "Reading BYTE value (ITE; at offset).") - - value, err = tt.ParseBytes(addressableData[ite.ValueOffset:], byteCount) - log.PanicIf(err) - } - - return value, nil -} - -// Value returns the specific, parsed, typed value from the tag. -func (ite *IfdTagEntry) Value(addressableData []byte, byteOrder binary.ByteOrder) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext := - newValueContextFromTag( - ite, - addressableData, - byteOrder) - - if ite.TagType == TypeUndefined { - value, err = valueContext.Undefined() - log.PanicIf(err) - } else { - tt := NewTagType(ite.TagType, byteOrder) - - value, err = tt.Resolve(valueContext) - log.PanicIf(err) - } - - return value, nil -} - -// IfdTagEntryValueResolver instances know how to resolve the values for any -// tag for a particular EXIF block. -type IfdTagEntryValueResolver struct { - addressableData []byte - byteOrder binary.ByteOrder -} - -func NewIfdTagEntryValueResolver(exifData []byte, byteOrder binary.ByteOrder) (itevr *IfdTagEntryValueResolver) { - return &IfdTagEntryValueResolver{ - addressableData: exifData[ExifAddressableAreaStart:], - byteOrder: byteOrder, - } -} - -// ValueBytes will resolve embedded or allocated data from the tag and return the raw bytes. -func (itevr *IfdTagEntryValueResolver) ValueBytes(ite *IfdTagEntry) (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // OBSOLETE(dustin): This is now redundant. Use `(*ValueContext).readRawEncoded()` instead of this method. - - valueContext := newValueContextFromTag( - ite, - itevr.addressableData, - itevr.byteOrder) - - rawBytes, err := valueContext.readRawEncoded() - log.PanicIf(err) - - return rawBytes, nil -} - -func (itevr *IfdTagEntryValueResolver) Value(ite *IfdTagEntry) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // OBSOLETE(dustin): This is now redundant. Use `(*ValueContext).Values()` instead of this method. - - valueContext := newValueContextFromTag( - ite, - itevr.addressableData, - itevr.byteOrder) - - values, err := valueContext.Values() - log.PanicIf(err) - - return values, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/package.go b/vendor/github.com/dsoprea/go-exif/package.go deleted file mode 100644 index 2eb6b3056..000000000 --- a/vendor/github.com/dsoprea/go-exif/package.go +++ /dev/null @@ -1,4 +0,0 @@ -// exif parses raw EXIF information given a block of raw EXIF data. -// -// v1 of go-exif is now deprecated. Please use v2. -package exif diff --git a/vendor/github.com/dsoprea/go-exif/parser.go b/vendor/github.com/dsoprea/go-exif/parser.go deleted file mode 100644 index 4702db2f8..000000000 --- a/vendor/github.com/dsoprea/go-exif/parser.go +++ /dev/null @@ -1,190 +0,0 @@ -package exif - -import ( - "bytes" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -type Parser struct { -} - -func (p *Parser) ParseBytes(data []byte, unitCount uint32) (value []uint8, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeByte.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = []uint8(data[:count]) - - return value, nil -} - -// ParseAscii returns a string and auto-strips the trailing NUL character. -func (p *Parser) ParseAscii(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeAscii.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - if len(data) == 0 || data[count-1] != 0 { - s := string(data[:count]) - typeLogger.Warningf(nil, "ascii not terminated with nul as expected: [%v]", s) - - return s, nil - } else { - // Auto-strip the NUL from the end. It serves no purpose outside of - // encoding semantics. - - return string(data[:count-1]), nil - } -} - -// ParseAsciiNoNul returns a string without any consideration for a trailing NUL -// character. -func (p *Parser) ParseAsciiNoNul(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeAscii.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - return string(data[:count]), nil -} - -func (p *Parser) ParseShorts(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeShort.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]uint16, count) - for i := 0; i < count; i++ { - value[i] = byteOrder.Uint16(data[i*2:]) - } - - return value, nil -} - -func (p *Parser) ParseLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeLong.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]uint32, count) - for i := 0; i < count; i++ { - value[i] = byteOrder.Uint32(data[i*4:]) - } - - return value, nil -} - -func (p *Parser) ParseRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeRational.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]Rational, count) - for i := 0; i < count; i++ { - value[i].Numerator = byteOrder.Uint32(data[i*8:]) - value[i].Denominator = byteOrder.Uint32(data[i*8+4:]) - } - - return value, nil -} - -func (p *Parser) ParseSignedLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeSignedLong.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - b := bytes.NewBuffer(data) - - value = make([]int32, count) - for i := 0; i < count; i++ { - err := binary.Read(b, byteOrder, &value[i]) - log.PanicIf(err) - } - - return value, nil -} - -func (p *Parser) ParseSignedRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - count := int(unitCount) - - if len(data) < (TypeSignedRational.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - b := bytes.NewBuffer(data) - - value = make([]SignedRational, count) - for i := 0; i < count; i++ { - err = binary.Read(b, byteOrder, &value[i].Numerator) - log.PanicIf(err) - - err = binary.Read(b, byteOrder, &value[i].Denominator) - log.PanicIf(err) - } - - return value, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/tag_type.go b/vendor/github.com/dsoprea/go-exif/tag_type.go deleted file mode 100644 index e53b1c498..000000000 --- a/vendor/github.com/dsoprea/go-exif/tag_type.go +++ /dev/null @@ -1,397 +0,0 @@ -package exif - -// NOTE(dustin): Most of this file encapsulates deprecated functionality and awaits being dumped in a future release. - -import ( - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -type TagType struct { - tagType TagTypePrimitive - name string - byteOrder binary.ByteOrder -} - -func NewTagType(tagType TagTypePrimitive, byteOrder binary.ByteOrder) TagType { - name, found := TypeNames[tagType] - if found == false { - log.Panicf("tag-type not valid: 0x%04x", tagType) - } - - return TagType{ - tagType: tagType, - name: name, - byteOrder: byteOrder, - } -} - -func (tt TagType) String() string { - return fmt.Sprintf("TagType", tt.name) -} - -func (tt TagType) Name() string { - return tt.name -} - -func (tt TagType) Type() TagTypePrimitive { - return tt.tagType -} - -func (tt TagType) ByteOrder() binary.ByteOrder { - return tt.byteOrder -} - -func (tt TagType) Size() int { - - // DEPRECATED(dustin): `(TagTypePrimitive).Size()` should be used, directly. - - return tt.Type().Size() -} - -// valueIsEmbedded will return a boolean indicating whether the value should be -// found directly within the IFD entry or an offset to somewhere else. -func (tt TagType) valueIsEmbedded(unitCount uint32) bool { - return (tt.tagType.Size() * int(unitCount)) <= 4 -} - -func (tt TagType) readRawEncoded(valueContext ValueContext) (rawBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - unitSizeRaw := uint32(tt.tagType.Size()) - - if tt.valueIsEmbedded(valueContext.UnitCount()) == true { - byteLength := unitSizeRaw * valueContext.UnitCount() - return valueContext.RawValueOffset()[:byteLength], nil - } else { - return valueContext.AddressableData()[valueContext.ValueOffset() : valueContext.ValueOffset()+valueContext.UnitCount()*unitSizeRaw], nil - } -} - -func (tt TagType) ParseBytes(data []byte, unitCount uint32) (value []uint8, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseBytes()` should be used. - - value, err = parser.ParseBytes(data, unitCount) - log.PanicIf(err) - - return value, nil -} - -// ParseAscii returns a string and auto-strips the trailing NUL character. -func (tt TagType) ParseAscii(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseAscii()` should be used. - - value, err = parser.ParseAscii(data, unitCount) - log.PanicIf(err) - - return value, nil -} - -// ParseAsciiNoNul returns a string without any consideration for a trailing NUL -// character. -func (tt TagType) ParseAsciiNoNul(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseAsciiNoNul()` should be used. - - value, err = parser.ParseAsciiNoNul(data, unitCount) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ParseShorts(data []byte, unitCount uint32) (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseShorts()` should be used. - - value, err = parser.ParseShorts(data, unitCount, tt.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ParseLongs(data []byte, unitCount uint32) (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseLongs()` should be used. - - value, err = parser.ParseLongs(data, unitCount, tt.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ParseRationals(data []byte, unitCount uint32) (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseRationals()` should be used. - - value, err = parser.ParseRationals(data, unitCount, tt.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ParseSignedLongs(data []byte, unitCount uint32) (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseSignedLongs()` should be used. - - value, err = parser.ParseSignedLongs(data, unitCount, tt.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ParseSignedRationals(data []byte, unitCount uint32) (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(*Parser).ParseSignedRationals()` should be used. - - value, err = parser.ParseSignedRationals(data, unitCount, tt.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadByteValues(valueContext ValueContext) (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadBytes()` should be used. - - value, err = valueContext.ReadBytes() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadAsciiValue(valueContext ValueContext) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadAscii()` should be used. - - value, err = valueContext.ReadAscii() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadAsciiNoNulValue(valueContext ValueContext) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadAsciiNoNul()` should be used. - - value, err = valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadShortValues(valueContext ValueContext) (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadShorts()` should be used. - - value, err = valueContext.ReadShorts() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadLongValues(valueContext ValueContext) (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadLongs()` should be used. - - value, err = valueContext.ReadLongs() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadRationalValues(valueContext ValueContext) (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadRationals()` should be used. - - value, err = valueContext.ReadRationals() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadSignedLongValues(valueContext ValueContext) (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadSignedLongs()` should be used. - - value, err = valueContext.ReadSignedLongs() - log.PanicIf(err) - - return value, nil -} - -func (tt TagType) ReadSignedRationalValues(valueContext ValueContext) (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).ReadSignedRationals()` should be used. - - value, err = valueContext.ReadSignedRationals() - log.PanicIf(err) - - return value, nil -} - -// ResolveAsString resolves the given value and returns a flat string. -// -// Where the type is not ASCII, `justFirst` indicates whether to just stringify -// the first item in the slice (or return an empty string if the slice is -// empty). -// -// Since this method lacks the information to process unknown-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (tt TagType) ResolveAsString(valueContext ValueContext, justFirst bool) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if justFirst == true { - value, err = valueContext.FormatFirst() - log.PanicIf(err) - } else { - value, err = valueContext.Format() - log.PanicIf(err) - } - - return value, nil -} - -// Resolve knows how to resolve the given value. -// -// Since this method lacks the information to process unknown-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (tt TagType) Resolve(valueContext *ValueContext) (values interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `(ValueContext).Values()` should be used. - - values, err = valueContext.Values() - log.PanicIf(err) - - return values, nil -} - -// Encode knows how to encode the given value to a byte slice. -func (tt TagType) Encode(value interface{}) (encoded []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ve := NewValueEncoder(tt.byteOrder) - - ed, err := ve.EncodeWithType(tt, value) - log.PanicIf(err) - - return ed.Encoded, err -} - -func (tt TagType) FromString(valueString string) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // DEPRECATED(dustin): `EncodeStringToBytes()` should be used. - - value, err = EncodeStringToBytes(tt.tagType, valueString) - log.PanicIf(err) - - return value, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/tags.go b/vendor/github.com/dsoprea/go-exif/tags.go deleted file mode 100644 index 7f7e51555..000000000 --- a/vendor/github.com/dsoprea/go-exif/tags.go +++ /dev/null @@ -1,229 +0,0 @@ -package exif - -import ( - "fmt" - - "github.com/dsoprea/go-logging" - "gopkg.in/yaml.v2" -) - -const ( - // IFD1 - - ThumbnailOffsetTagId = 0x0201 - ThumbnailSizeTagId = 0x0202 - - // Exif - - TagVersionId = 0x0000 - - TagLatitudeId = 0x0002 - TagLatitudeRefId = 0x0001 - TagLongitudeId = 0x0004 - TagLongitudeRefId = 0x0003 - - TagTimestampId = 0x0007 - TagDatestampId = 0x001d - - TagAltitudeId = 0x0006 - TagAltitudeRefId = 0x0005 -) - -var ( - // tagsWithoutAlignment is a tag-lookup for tags whose value size won't - // necessarily be a multiple of its tag-type. - tagsWithoutAlignment = map[uint16]struct{}{ - // The thumbnail offset is stored as a long, but its data is a binary - // blob (not a slice of longs). - ThumbnailOffsetTagId: {}, - } -) - -var ( - tagsLogger = log.NewLogger("exif.tags") -) - -// File structures. - -type encodedTag struct { - // id is signed, here, because YAML doesn't have enough information to - // support unsigned. - Id int `yaml:"id"` - Name string `yaml:"name"` - TypeName string `yaml:"type_name"` -} - -// Indexing structures. - -type IndexedTag struct { - Id uint16 - Name string - IfdPath string - Type TagTypePrimitive -} - -func (it *IndexedTag) String() string { - return fmt.Sprintf("TAG", it.Id, it.Name, it.IfdPath) -} - -func (it *IndexedTag) IsName(ifdPath, name string) bool { - return it.Name == name && it.IfdPath == ifdPath -} - -func (it *IndexedTag) Is(ifdPath string, id uint16) bool { - return it.Id == id && it.IfdPath == ifdPath -} - -type TagIndex struct { - tagsByIfd map[string]map[uint16]*IndexedTag - tagsByIfdR map[string]map[string]*IndexedTag -} - -func NewTagIndex() *TagIndex { - ti := new(TagIndex) - - ti.tagsByIfd = make(map[string]map[uint16]*IndexedTag) - ti.tagsByIfdR = make(map[string]map[string]*IndexedTag) - - return ti -} - -func (ti *TagIndex) Add(it *IndexedTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Store by ID. - - family, found := ti.tagsByIfd[it.IfdPath] - if found == false { - family = make(map[uint16]*IndexedTag) - ti.tagsByIfd[it.IfdPath] = family - } - - if _, found := family[it.Id]; found == true { - log.Panicf("tag-ID defined more than once for IFD [%s]: (%02x)", it.IfdPath, it.Id) - } - - family[it.Id] = it - - // Store by name. - - familyR, found := ti.tagsByIfdR[it.IfdPath] - if found == false { - familyR = make(map[string]*IndexedTag) - ti.tagsByIfdR[it.IfdPath] = familyR - } - - if _, found := familyR[it.Name]; found == true { - log.Panicf("tag-name defined more than once for IFD [%s]: (%s)", it.IfdPath, it.Name) - } - - familyR[it.Name] = it - - return nil -} - -// Get returns information about the non-IFD tag. -func (ti *TagIndex) Get(ifdPath string, id uint16) (it *IndexedTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(ti.tagsByIfd) == 0 { - err := LoadStandardTags(ti) - log.PanicIf(err) - } - - family, found := ti.tagsByIfd[ifdPath] - if found == false { - log.Panic(ErrTagNotFound) - } - - it, found = family[id] - if found == false { - log.Panic(ErrTagNotFound) - } - - return it, nil -} - -// Get returns information about the non-IFD tag. -func (ti *TagIndex) GetWithName(ifdPath string, name string) (it *IndexedTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(ti.tagsByIfdR) == 0 { - err := LoadStandardTags(ti) - log.PanicIf(err) - } - - it, found := ti.tagsByIfdR[ifdPath][name] - if found != true { - log.Panic(ErrTagNotFound) - } - - return it, nil -} - -// LoadStandardTags registers the tags that all devices/applications should -// support. -func LoadStandardTags(ti *TagIndex) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Read static data. - - encodedIfds := make(map[string][]encodedTag) - - err = yaml.Unmarshal([]byte(tagsYaml), encodedIfds) - log.PanicIf(err) - - // Load structure. - - count := 0 - for ifdPath, tags := range encodedIfds { - for _, tagInfo := range tags { - tagId := uint16(tagInfo.Id) - tagName := tagInfo.Name - tagTypeName := tagInfo.TypeName - - // TODO(dustin): !! Non-standard types, but found in real data. Ignore for right now. - if tagTypeName == "SSHORT" || tagTypeName == "FLOAT" || tagTypeName == "DOUBLE" { - continue - } - - tagTypeId, found := TypeNamesR[tagTypeName] - if found == false { - log.Panicf("type [%s] for [%s] not valid", tagTypeName, tagName) - continue - } - - it := &IndexedTag{ - IfdPath: ifdPath, - Id: tagId, - Name: tagName, - Type: tagTypeId, - } - - err = ti.Add(it) - log.PanicIf(err) - - count++ - } - } - - tagsLogger.Debugf(nil, "(%d) tags loaded.", count) - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/tags_data.go b/vendor/github.com/dsoprea/go-exif/tags_data.go deleted file mode 100644 index 64ec458d3..000000000 --- a/vendor/github.com/dsoprea/go-exif/tags_data.go +++ /dev/null @@ -1,951 +0,0 @@ -package exif - -var ( - // From assets/tags.yaml . Needs to be here so it's embedded in the binary. - tagsYaml = ` -# Notes: -# -# This file was produced from http://www.exiv2.org/tags.html, using the included -# tool, though that document appears to have some duplicates when all IDs are -# supposed to be unique (EXIF information only has IDs, not IFDs; IFDs are -# determined by our pre-existing knowledge of those tags). -# -# The webpage that we've produced this file from appears to indicate that -# ImageWidth is represented by both 0x0100 and 0x0001 depending on whether the -# encoding is RGB or YCbCr. -IFD/Exif: -- id: 0x829a - name: ExposureTime - type_name: RATIONAL -- id: 0x829d - name: FNumber - type_name: RATIONAL -- id: 0x8822 - name: ExposureProgram - type_name: SHORT -- id: 0x8824 - name: SpectralSensitivity - type_name: ASCII -- id: 0x8827 - name: ISOSpeedRatings - type_name: SHORT -- id: 0x8828 - name: OECF - type_name: UNDEFINED -- id: 0x8830 - name: SensitivityType - type_name: SHORT -- id: 0x8831 - name: StandardOutputSensitivity - type_name: LONG -- id: 0x8832 - name: RecommendedExposureIndex - type_name: LONG -- id: 0x8833 - name: ISOSpeed - type_name: LONG -- id: 0x8834 - name: ISOSpeedLatitudeyyy - type_name: LONG -- id: 0x8835 - name: ISOSpeedLatitudezzz - type_name: LONG -- id: 0x9000 - name: ExifVersion - type_name: UNDEFINED -- id: 0x9003 - name: DateTimeOriginal - type_name: ASCII -- id: 0x9004 - name: DateTimeDigitized - type_name: ASCII -- id: 0x9101 - name: ComponentsConfiguration - type_name: UNDEFINED -- id: 0x9102 - name: CompressedBitsPerPixel - type_name: RATIONAL -- id: 0x9201 - name: ShutterSpeedValue - type_name: SRATIONAL -- id: 0x9202 - name: ApertureValue - type_name: RATIONAL -- id: 0x9203 - name: BrightnessValue - type_name: SRATIONAL -- id: 0x9204 - name: ExposureBiasValue - type_name: SRATIONAL -- id: 0x9205 - name: MaxApertureValue - type_name: RATIONAL -- id: 0x9206 - name: SubjectDistance - type_name: RATIONAL -- id: 0x9207 - name: MeteringMode - type_name: SHORT -- id: 0x9208 - name: LightSource - type_name: SHORT -- id: 0x9209 - name: Flash - type_name: SHORT -- id: 0x920a - name: FocalLength - type_name: RATIONAL -- id: 0x9214 - name: SubjectArea - type_name: SHORT -- id: 0x927c - name: MakerNote - type_name: UNDEFINED -- id: 0x9286 - name: UserComment - type_name: UNDEFINED -- id: 0x9290 - name: SubSecTime - type_name: ASCII -- id: 0x9291 - name: SubSecTimeOriginal - type_name: ASCII -- id: 0x9292 - name: SubSecTimeDigitized - type_name: ASCII -- id: 0xa000 - name: FlashpixVersion - type_name: UNDEFINED -- id: 0xa001 - name: ColorSpace - type_name: SHORT -- id: 0xa002 - name: PixelXDimension - type_name: LONG -- id: 0xa003 - name: PixelYDimension - type_name: LONG -- id: 0xa004 - name: RelatedSoundFile - type_name: ASCII -- id: 0xa005 - name: InteroperabilityTag - type_name: LONG -- id: 0xa20b - name: FlashEnergy - type_name: RATIONAL -- id: 0xa20c - name: SpatialFrequencyResponse - type_name: UNDEFINED -- id: 0xa20e - name: FocalPlaneXResolution - type_name: RATIONAL -- id: 0xa20f - name: FocalPlaneYResolution - type_name: RATIONAL -- id: 0xa210 - name: FocalPlaneResolutionUnit - type_name: SHORT -- id: 0xa214 - name: SubjectLocation - type_name: SHORT -- id: 0xa215 - name: ExposureIndex - type_name: RATIONAL -- id: 0xa217 - name: SensingMethod - type_name: SHORT -- id: 0xa300 - name: FileSource - type_name: UNDEFINED -- id: 0xa301 - name: SceneType - type_name: UNDEFINED -- id: 0xa302 - name: CFAPattern - type_name: UNDEFINED -- id: 0xa401 - name: CustomRendered - type_name: SHORT -- id: 0xa402 - name: ExposureMode - type_name: SHORT -- id: 0xa403 - name: WhiteBalance - type_name: SHORT -- id: 0xa404 - name: DigitalZoomRatio - type_name: RATIONAL -- id: 0xa405 - name: FocalLengthIn35mmFilm - type_name: SHORT -- id: 0xa406 - name: SceneCaptureType - type_name: SHORT -- id: 0xa407 - name: GainControl - type_name: SHORT -- id: 0xa408 - name: Contrast - type_name: SHORT -- id: 0xa409 - name: Saturation - type_name: SHORT -- id: 0xa40a - name: Sharpness - type_name: SHORT -- id: 0xa40b - name: DeviceSettingDescription - type_name: UNDEFINED -- id: 0xa40c - name: SubjectDistanceRange - type_name: SHORT -- id: 0xa420 - name: ImageUniqueID - type_name: ASCII -- id: 0xa430 - name: CameraOwnerName - type_name: ASCII -- id: 0xa431 - name: BodySerialNumber - type_name: ASCII -- id: 0xa432 - name: LensSpecification - type_name: RATIONAL -- id: 0xa433 - name: LensMake - type_name: ASCII -- id: 0xa434 - name: LensModel - type_name: ASCII -- id: 0xa435 - name: LensSerialNumber - type_name: ASCII -IFD/GPSInfo: -- id: 0x0000 - name: GPSVersionID - type_name: BYTE -- id: 0x0001 - name: GPSLatitudeRef - type_name: ASCII -- id: 0x0002 - name: GPSLatitude - type_name: RATIONAL -- id: 0x0003 - name: GPSLongitudeRef - type_name: ASCII -- id: 0x0004 - name: GPSLongitude - type_name: RATIONAL -- id: 0x0005 - name: GPSAltitudeRef - type_name: BYTE -- id: 0x0006 - name: GPSAltitude - type_name: RATIONAL -- id: 0x0007 - name: GPSTimeStamp - type_name: RATIONAL -- id: 0x0008 - name: GPSSatellites - type_name: ASCII -- id: 0x0009 - name: GPSStatus - type_name: ASCII -- id: 0x000a - name: GPSMeasureMode - type_name: ASCII -- id: 0x000b - name: GPSDOP - type_name: RATIONAL -- id: 0x000c - name: GPSSpeedRef - type_name: ASCII -- id: 0x000d - name: GPSSpeed - type_name: RATIONAL -- id: 0x000e - name: GPSTrackRef - type_name: ASCII -- id: 0x000f - name: GPSTrack - type_name: RATIONAL -- id: 0x0010 - name: GPSImgDirectionRef - type_name: ASCII -- id: 0x0011 - name: GPSImgDirection - type_name: RATIONAL -- id: 0x0012 - name: GPSMapDatum - type_name: ASCII -- id: 0x0013 - name: GPSDestLatitudeRef - type_name: ASCII -- id: 0x0014 - name: GPSDestLatitude - type_name: RATIONAL -- id: 0x0015 - name: GPSDestLongitudeRef - type_name: ASCII -- id: 0x0016 - name: GPSDestLongitude - type_name: RATIONAL -- id: 0x0017 - name: GPSDestBearingRef - type_name: ASCII -- id: 0x0018 - name: GPSDestBearing - type_name: RATIONAL -- id: 0x0019 - name: GPSDestDistanceRef - type_name: ASCII -- id: 0x001a - name: GPSDestDistance - type_name: RATIONAL -- id: 0x001b - name: GPSProcessingMethod - type_name: UNDEFINED -- id: 0x001c - name: GPSAreaInformation - type_name: UNDEFINED -- id: 0x001d - name: GPSDateStamp - type_name: ASCII -- id: 0x001e - name: GPSDifferential - type_name: SHORT -IFD: -- id: 0x000b - name: ProcessingSoftware - type_name: ASCII -- id: 0x00fe - name: NewSubfileType - type_name: LONG -- id: 0x00ff - name: SubfileType - type_name: SHORT -- id: 0x0100 - name: ImageWidth - type_name: LONG -- id: 0x0101 - name: ImageLength - type_name: LONG -- id: 0x0102 - name: BitsPerSample - type_name: SHORT -- id: 0x0103 - name: Compression - type_name: SHORT -- id: 0x0106 - name: PhotometricInterpretation - type_name: SHORT -- id: 0x0107 - name: Thresholding - type_name: SHORT -- id: 0x0108 - name: CellWidth - type_name: SHORT -- id: 0x0109 - name: CellLength - type_name: SHORT -- id: 0x010a - name: FillOrder - type_name: SHORT -- id: 0x010d - name: DocumentName - type_name: ASCII -- id: 0x010e - name: ImageDescription - type_name: ASCII -- id: 0x010f - name: Make - type_name: ASCII -- id: 0x0110 - name: Model - type_name: ASCII -- id: 0x0111 - name: StripOffsets - type_name: LONG -- id: 0x0112 - name: Orientation - type_name: SHORT -- id: 0x0115 - name: SamplesPerPixel - type_name: SHORT -- id: 0x0116 - name: RowsPerStrip - type_name: LONG -- id: 0x0117 - name: StripByteCounts - type_name: LONG -- id: 0x011a - name: XResolution - type_name: RATIONAL -- id: 0x011b - name: YResolution - type_name: RATIONAL -- id: 0x011c - name: PlanarConfiguration - type_name: SHORT -- id: 0x0122 - name: GrayResponseUnit - type_name: SHORT -- id: 0x0123 - name: GrayResponseCurve - type_name: SHORT -- id: 0x0124 - name: T4Options - type_name: LONG -- id: 0x0125 - name: T6Options - type_name: LONG -- id: 0x0128 - name: ResolutionUnit - type_name: SHORT -- id: 0x0129 - name: PageNumber - type_name: SHORT -- id: 0x012d - name: TransferFunction - type_name: SHORT -- id: 0x0131 - name: Software - type_name: ASCII -- id: 0x0132 - name: DateTime - type_name: ASCII -- id: 0x013b - name: Artist - type_name: ASCII -- id: 0x013c - name: HostComputer - type_name: ASCII -- id: 0x013d - name: Predictor - type_name: SHORT -- id: 0x013e - name: WhitePoint - type_name: RATIONAL -- id: 0x013f - name: PrimaryChromaticities - type_name: RATIONAL -- id: 0x0140 - name: ColorMap - type_name: SHORT -- id: 0x0141 - name: HalftoneHints - type_name: SHORT -- id: 0x0142 - name: TileWidth - type_name: SHORT -- id: 0x0143 - name: TileLength - type_name: SHORT -- id: 0x0144 - name: TileOffsets - type_name: SHORT -- id: 0x0145 - name: TileByteCounts - type_name: SHORT -- id: 0x014a - name: SubIFDs - type_name: LONG -- id: 0x014c - name: InkSet - type_name: SHORT -- id: 0x014d - name: InkNames - type_name: ASCII -- id: 0x014e - name: NumberOfInks - type_name: SHORT -- id: 0x0150 - name: DotRange - type_name: BYTE -- id: 0x0151 - name: TargetPrinter - type_name: ASCII -- id: 0x0152 - name: ExtraSamples - type_name: SHORT -- id: 0x0153 - name: SampleFormat - type_name: SHORT -- id: 0x0154 - name: SMinSampleValue - type_name: SHORT -- id: 0x0155 - name: SMaxSampleValue - type_name: SHORT -- id: 0x0156 - name: TransferRange - type_name: SHORT -- id: 0x0157 - name: ClipPath - type_name: BYTE -- id: 0x0158 - name: XClipPathUnits - type_name: SSHORT -- id: 0x0159 - name: YClipPathUnits - type_name: SSHORT -- id: 0x015a - name: Indexed - type_name: SHORT -- id: 0x015b - name: JPEGTables - type_name: UNDEFINED -- id: 0x015f - name: OPIProxy - type_name: SHORT -- id: 0x0200 - name: JPEGProc - type_name: LONG -- id: 0x0201 - name: JPEGInterchangeFormat - type_name: LONG -- id: 0x0202 - name: JPEGInterchangeFormatLength - type_name: LONG -- id: 0x0203 - name: JPEGRestartInterval - type_name: SHORT -- id: 0x0205 - name: JPEGLosslessPredictors - type_name: SHORT -- id: 0x0206 - name: JPEGPointTransforms - type_name: SHORT -- id: 0x0207 - name: JPEGQTables - type_name: LONG -- id: 0x0208 - name: JPEGDCTables - type_name: LONG -- id: 0x0209 - name: JPEGACTables - type_name: LONG -- id: 0x0211 - name: YCbCrCoefficients - type_name: RATIONAL -- id: 0x0212 - name: YCbCrSubSampling - type_name: SHORT -- id: 0x0213 - name: YCbCrPositioning - type_name: SHORT -- id: 0x0214 - name: ReferenceBlackWhite - type_name: RATIONAL -- id: 0x02bc - name: XMLPacket - type_name: BYTE -- id: 0x4746 - name: Rating - type_name: SHORT -- id: 0x4749 - name: RatingPercent - type_name: SHORT -- id: 0x800d - name: ImageID - type_name: ASCII -- id: 0x828d - name: CFARepeatPatternDim - type_name: SHORT -- id: 0x828e - name: CFAPattern - type_name: BYTE -- id: 0x828f - name: BatteryLevel - type_name: RATIONAL -- id: 0x8298 - name: Copyright - type_name: ASCII -- id: 0x829a - name: ExposureTime - type_name: RATIONAL -- id: 0x829d - name: FNumber - type_name: RATIONAL -- id: 0x83bb - name: IPTCNAA - type_name: LONG -- id: 0x8649 - name: ImageResources - type_name: BYTE -- id: 0x8769 - name: ExifTag - type_name: LONG -- id: 0x8773 - name: InterColorProfile - type_name: UNDEFINED -- id: 0x8822 - name: ExposureProgram - type_name: SHORT -- id: 0x8824 - name: SpectralSensitivity - type_name: ASCII -- id: 0x8825 - name: GPSTag - type_name: LONG -- id: 0x8827 - name: ISOSpeedRatings - type_name: SHORT -- id: 0x8828 - name: OECF - type_name: UNDEFINED -- id: 0x8829 - name: Interlace - type_name: SHORT -- id: 0x882a - name: TimeZoneOffset - type_name: SSHORT -- id: 0x882b - name: SelfTimerMode - type_name: SHORT -- id: 0x9003 - name: DateTimeOriginal - type_name: ASCII -- id: 0x9102 - name: CompressedBitsPerPixel - type_name: RATIONAL -- id: 0x9201 - name: ShutterSpeedValue - type_name: SRATIONAL -- id: 0x9202 - name: ApertureValue - type_name: RATIONAL -- id: 0x9203 - name: BrightnessValue - type_name: SRATIONAL -- id: 0x9204 - name: ExposureBiasValue - type_name: SRATIONAL -- id: 0x9205 - name: MaxApertureValue - type_name: RATIONAL -- id: 0x9206 - name: SubjectDistance - type_name: SRATIONAL -- id: 0x9207 - name: MeteringMode - type_name: SHORT -- id: 0x9208 - name: LightSource - type_name: SHORT -- id: 0x9209 - name: Flash - type_name: SHORT -- id: 0x920a - name: FocalLength - type_name: RATIONAL -- id: 0x920b - name: FlashEnergy - type_name: RATIONAL -- id: 0x920c - name: SpatialFrequencyResponse - type_name: UNDEFINED -- id: 0x920d - name: Noise - type_name: UNDEFINED -- id: 0x920e - name: FocalPlaneXResolution - type_name: RATIONAL -- id: 0x920f - name: FocalPlaneYResolution - type_name: RATIONAL -- id: 0x9210 - name: FocalPlaneResolutionUnit - type_name: SHORT -- id: 0x9211 - name: ImageNumber - type_name: LONG -- id: 0x9212 - name: SecurityClassification - type_name: ASCII -- id: 0x9213 - name: ImageHistory - type_name: ASCII -- id: 0x9214 - name: SubjectLocation - type_name: SHORT -- id: 0x9215 - name: ExposureIndex - type_name: RATIONAL -- id: 0x9216 - name: TIFFEPStandardID - type_name: BYTE -- id: 0x9217 - name: SensingMethod - type_name: SHORT -- id: 0x9c9b - name: XPTitle - type_name: BYTE -- id: 0x9c9c - name: XPComment - type_name: BYTE -- id: 0x9c9d - name: XPAuthor - type_name: BYTE -- id: 0x9c9e - name: XPKeywords - type_name: BYTE -- id: 0x9c9f - name: XPSubject - type_name: BYTE -- id: 0xc4a5 - name: PrintImageMatching - type_name: UNDEFINED -- id: 0xc612 - name: DNGVersion - type_name: BYTE -- id: 0xc613 - name: DNGBackwardVersion - type_name: BYTE -- id: 0xc614 - name: UniqueCameraModel - type_name: ASCII -- id: 0xc615 - name: LocalizedCameraModel - type_name: BYTE -- id: 0xc616 - name: CFAPlaneColor - type_name: BYTE -- id: 0xc617 - name: CFALayout - type_name: SHORT -- id: 0xc618 - name: LinearizationTable - type_name: SHORT -- id: 0xc619 - name: BlackLevelRepeatDim - type_name: SHORT -- id: 0xc61a - name: BlackLevel - type_name: RATIONAL -- id: 0xc61b - name: BlackLevelDeltaH - type_name: SRATIONAL -- id: 0xc61c - name: BlackLevelDeltaV - type_name: SRATIONAL -- id: 0xc61d - name: WhiteLevel - type_name: SHORT -- id: 0xc61e - name: DefaultScale - type_name: RATIONAL -- id: 0xc61f - name: DefaultCropOrigin - type_name: SHORT -- id: 0xc620 - name: DefaultCropSize - type_name: SHORT -- id: 0xc621 - name: ColorMatrix1 - type_name: SRATIONAL -- id: 0xc622 - name: ColorMatrix2 - type_name: SRATIONAL -- id: 0xc623 - name: CameraCalibration1 - type_name: SRATIONAL -- id: 0xc624 - name: CameraCalibration2 - type_name: SRATIONAL -- id: 0xc625 - name: ReductionMatrix1 - type_name: SRATIONAL -- id: 0xc626 - name: ReductionMatrix2 - type_name: SRATIONAL -- id: 0xc627 - name: AnalogBalance - type_name: RATIONAL -- id: 0xc628 - name: AsShotNeutral - type_name: SHORT -- id: 0xc629 - name: AsShotWhiteXY - type_name: RATIONAL -- id: 0xc62a - name: BaselineExposure - type_name: SRATIONAL -- id: 0xc62b - name: BaselineNoise - type_name: RATIONAL -- id: 0xc62c - name: BaselineSharpness - type_name: RATIONAL -- id: 0xc62d - name: BayerGreenSplit - type_name: LONG -- id: 0xc62e - name: LinearResponseLimit - type_name: RATIONAL -- id: 0xc62f - name: CameraSerialNumber - type_name: ASCII -- id: 0xc630 - name: LensInfo - type_name: RATIONAL -- id: 0xc631 - name: ChromaBlurRadius - type_name: RATIONAL -- id: 0xc632 - name: AntiAliasStrength - type_name: RATIONAL -- id: 0xc633 - name: ShadowScale - type_name: SRATIONAL -- id: 0xc634 - name: DNGPrivateData - type_name: BYTE -- id: 0xc635 - name: MakerNoteSafety - type_name: SHORT -- id: 0xc65a - name: CalibrationIlluminant1 - type_name: SHORT -- id: 0xc65b - name: CalibrationIlluminant2 - type_name: SHORT -- id: 0xc65c - name: BestQualityScale - type_name: RATIONAL -- id: 0xc65d - name: RawDataUniqueID - type_name: BYTE -- id: 0xc68b - name: OriginalRawFileName - type_name: BYTE -- id: 0xc68c - name: OriginalRawFileData - type_name: UNDEFINED -- id: 0xc68d - name: ActiveArea - type_name: SHORT -- id: 0xc68e - name: MaskedAreas - type_name: SHORT -- id: 0xc68f - name: AsShotICCProfile - type_name: UNDEFINED -- id: 0xc690 - name: AsShotPreProfileMatrix - type_name: SRATIONAL -- id: 0xc691 - name: CurrentICCProfile - type_name: UNDEFINED -- id: 0xc692 - name: CurrentPreProfileMatrix - type_name: SRATIONAL -- id: 0xc6bf - name: ColorimetricReference - type_name: SHORT -- id: 0xc6f3 - name: CameraCalibrationSignature - type_name: BYTE -- id: 0xc6f4 - name: ProfileCalibrationSignature - type_name: BYTE -- id: 0xc6f6 - name: AsShotProfileName - type_name: BYTE -- id: 0xc6f7 - name: NoiseReductionApplied - type_name: RATIONAL -- id: 0xc6f8 - name: ProfileName - type_name: BYTE -- id: 0xc6f9 - name: ProfileHueSatMapDims - type_name: LONG -- id: 0xc6fa - name: ProfileHueSatMapData1 - type_name: FLOAT -- id: 0xc6fb - name: ProfileHueSatMapData2 - type_name: FLOAT -- id: 0xc6fc - name: ProfileToneCurve - type_name: FLOAT -- id: 0xc6fd - name: ProfileEmbedPolicy - type_name: LONG -- id: 0xc6fe - name: ProfileCopyright - type_name: BYTE -- id: 0xc714 - name: ForwardMatrix1 - type_name: SRATIONAL -- id: 0xc715 - name: ForwardMatrix2 - type_name: SRATIONAL -- id: 0xc716 - name: PreviewApplicationName - type_name: BYTE -- id: 0xc717 - name: PreviewApplicationVersion - type_name: BYTE -- id: 0xc718 - name: PreviewSettingsName - type_name: BYTE -- id: 0xc719 - name: PreviewSettingsDigest - type_name: BYTE -- id: 0xc71a - name: PreviewColorSpace - type_name: LONG -- id: 0xc71b - name: PreviewDateTime - type_name: ASCII -- id: 0xc71c - name: RawImageDigest - type_name: UNDEFINED -- id: 0xc71d - name: OriginalRawFileDigest - type_name: UNDEFINED -- id: 0xc71e - name: SubTileBlockSize - type_name: LONG -- id: 0xc71f - name: RowInterleaveFactor - type_name: LONG -- id: 0xc725 - name: ProfileLookTableDims - type_name: LONG -- id: 0xc726 - name: ProfileLookTableData - type_name: FLOAT -- id: 0xc740 - name: OpcodeList1 - type_name: UNDEFINED -- id: 0xc741 - name: OpcodeList2 - type_name: UNDEFINED -- id: 0xc74e - name: OpcodeList3 - type_name: UNDEFINED -- id: 0xc761 - name: NoiseProfile - type_name: DOUBLE -IFD/Exif/Iop: -- id: 0x0001 - name: InteroperabilityIndex - type_name: ASCII -- id: 0x0002 - name: InteroperabilityVersion - type_name: UNDEFINED -- id: 0x1000 - name: RelatedImageFileFormat - type_name: ASCII -- id: 0x1001 - name: RelatedImageWidth - type_name: LONG -- id: 0x1002 - name: RelatedImageLength - type_name: LONG -` -) diff --git a/vendor/github.com/dsoprea/go-exif/tags_undefined.go b/vendor/github.com/dsoprea/go-exif/tags_undefined.go deleted file mode 100644 index 63ba59323..000000000 --- a/vendor/github.com/dsoprea/go-exif/tags_undefined.go +++ /dev/null @@ -1,417 +0,0 @@ -package exif - -import ( - "bytes" - "fmt" - "strings" - - "crypto/sha1" - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -const ( - UnparseableUnknownTagValuePlaceholder = "!UNKNOWN" -) - -// TODO(dustin): Rename "unknown" in symbol names to "undefined" in the next release. -// -// See https://github.com/dsoprea/go-exif/issues/27 . - -const ( - TagUnknownType_9298_UserComment_Encoding_ASCII = iota - TagUnknownType_9298_UserComment_Encoding_JIS = iota - TagUnknownType_9298_UserComment_Encoding_UNICODE = iota - TagUnknownType_9298_UserComment_Encoding_UNDEFINED = iota -) - -const ( - TagUnknownType_9101_ComponentsConfiguration_Channel_Y = 0x1 - TagUnknownType_9101_ComponentsConfiguration_Channel_Cb = 0x2 - TagUnknownType_9101_ComponentsConfiguration_Channel_Cr = 0x3 - TagUnknownType_9101_ComponentsConfiguration_Channel_R = 0x4 - TagUnknownType_9101_ComponentsConfiguration_Channel_G = 0x5 - TagUnknownType_9101_ComponentsConfiguration_Channel_B = 0x6 -) - -const ( - TagUnknownType_9101_ComponentsConfiguration_OTHER = iota - TagUnknownType_9101_ComponentsConfiguration_RGB = iota - TagUnknownType_9101_ComponentsConfiguration_YCBCR = iota -) - -var ( - TagUnknownType_9298_UserComment_Encoding_Names = map[int]string{ - TagUnknownType_9298_UserComment_Encoding_ASCII: "ASCII", - TagUnknownType_9298_UserComment_Encoding_JIS: "JIS", - TagUnknownType_9298_UserComment_Encoding_UNICODE: "UNICODE", - TagUnknownType_9298_UserComment_Encoding_UNDEFINED: "UNDEFINED", - } - - TagUnknownType_9298_UserComment_Encodings = map[int][]byte{ - TagUnknownType_9298_UserComment_Encoding_ASCII: {'A', 'S', 'C', 'I', 'I', 0, 0, 0}, - TagUnknownType_9298_UserComment_Encoding_JIS: {'J', 'I', 'S', 0, 0, 0, 0, 0}, - TagUnknownType_9298_UserComment_Encoding_UNICODE: {'U', 'n', 'i', 'c', 'o', 'd', 'e', 0}, - TagUnknownType_9298_UserComment_Encoding_UNDEFINED: {0, 0, 0, 0, 0, 0, 0, 0}, - } - - TagUnknownType_9101_ComponentsConfiguration_Names = map[int]string{ - TagUnknownType_9101_ComponentsConfiguration_OTHER: "OTHER", - TagUnknownType_9101_ComponentsConfiguration_RGB: "RGB", - TagUnknownType_9101_ComponentsConfiguration_YCBCR: "YCBCR", - } - - TagUnknownType_9101_ComponentsConfiguration_Configurations = map[int][]byte{ - TagUnknownType_9101_ComponentsConfiguration_RGB: { - TagUnknownType_9101_ComponentsConfiguration_Channel_R, - TagUnknownType_9101_ComponentsConfiguration_Channel_G, - TagUnknownType_9101_ComponentsConfiguration_Channel_B, - 0, - }, - - TagUnknownType_9101_ComponentsConfiguration_YCBCR: { - TagUnknownType_9101_ComponentsConfiguration_Channel_Y, - TagUnknownType_9101_ComponentsConfiguration_Channel_Cb, - TagUnknownType_9101_ComponentsConfiguration_Channel_Cr, - 0, - }, - } -) - -// TODO(dustin): Rename `UnknownTagValue` to `UndefinedTagValue`. - -type UnknownTagValue interface { - ValueBytes() ([]byte, error) -} - -// TODO(dustin): Rename `TagUnknownType_GeneralString` to `TagUnknownType_GeneralString`. - -type TagUnknownType_GeneralString string - -func (gs TagUnknownType_GeneralString) ValueBytes() (value []byte, err error) { - return []byte(gs), nil -} - -// TODO(dustin): Rename `TagUnknownType_9298_UserComment` to `TagUndefinedType_9298_UserComment`. - -type TagUnknownType_9298_UserComment struct { - EncodingType int - EncodingBytes []byte -} - -func (uc TagUnknownType_9298_UserComment) String() string { - var valuePhrase string - - if len(uc.EncodingBytes) <= 8 { - valuePhrase = fmt.Sprintf("%v", uc.EncodingBytes) - } else { - valuePhrase = fmt.Sprintf("%v...", uc.EncodingBytes[:8]) - } - - return fmt.Sprintf("UserComment", len(uc.EncodingBytes), TagUnknownType_9298_UserComment_Encoding_Names[uc.EncodingType], valuePhrase, len(uc.EncodingBytes)) -} - -func (uc TagUnknownType_9298_UserComment) ValueBytes() (value []byte, err error) { - encodingTypeBytes, found := TagUnknownType_9298_UserComment_Encodings[uc.EncodingType] - if found == false { - log.Panicf("encoding-type not valid for unknown-type tag 9298 (UserComment): (%d)", uc.EncodingType) - } - - value = make([]byte, len(uc.EncodingBytes)+8) - - copy(value[:8], encodingTypeBytes) - copy(value[8:], uc.EncodingBytes) - - return value, nil -} - -// TODO(dustin): Rename `TagUnknownType_927C_MakerNote` to `TagUndefinedType_927C_MakerNote`. - -type TagUnknownType_927C_MakerNote struct { - MakerNoteType []byte - MakerNoteBytes []byte -} - -func (mn TagUnknownType_927C_MakerNote) String() string { - parts := make([]string, 20) - for i, c := range mn.MakerNoteType { - parts[i] = fmt.Sprintf("%02x", c) - } - - h := sha1.New() - - _, err := h.Write(mn.MakerNoteBytes) - log.PanicIf(err) - - digest := h.Sum(nil) - - return fmt.Sprintf("MakerNote", strings.Join(parts, " "), len(mn.MakerNoteBytes), digest) -} - -func (uc TagUnknownType_927C_MakerNote) ValueBytes() (value []byte, err error) { - return uc.MakerNoteBytes, nil -} - -// TODO(dustin): Rename `TagUnknownType_9101_ComponentsConfiguration` to `TagUndefinedType_9101_ComponentsConfiguration`. - -type TagUnknownType_9101_ComponentsConfiguration struct { - ConfigurationId int - ConfigurationBytes []byte -} - -func (cc TagUnknownType_9101_ComponentsConfiguration) String() string { - return fmt.Sprintf("ComponentsConfiguration", TagUnknownType_9101_ComponentsConfiguration_Names[cc.ConfigurationId], cc.ConfigurationBytes) -} - -func (uc TagUnknownType_9101_ComponentsConfiguration) ValueBytes() (value []byte, err error) { - return uc.ConfigurationBytes, nil -} - -// TODO(dustin): Rename `EncodeUnknown_9286` to `EncodeUndefined_9286`. - -func EncodeUnknown_9286(uc TagUnknownType_9298_UserComment) (encoded []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - b := new(bytes.Buffer) - - encodingTypeBytes := TagUnknownType_9298_UserComment_Encodings[uc.EncodingType] - - _, err = b.Write(encodingTypeBytes) - log.PanicIf(err) - - _, err = b.Write(uc.EncodingBytes) - log.PanicIf(err) - - return b.Bytes(), nil -} - -type EncodeableUndefinedValue struct { - IfdPath string - TagId uint16 - Parameters interface{} -} - -func EncodeUndefined(ifdPath string, tagId uint16, value interface{}) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Finish implementing these. - if ifdPath == IfdPathStandardExif { - if tagId == 0x9286 { - encoded, err := EncodeUnknown_9286(value.(TagUnknownType_9298_UserComment)) - log.PanicIf(err) - - ed.Type = TypeUndefined - ed.Encoded = encoded - ed.UnitCount = uint32(len(encoded)) - - return ed, nil - } - } - - log.Panicf("undefined value not encodable: %s (0x%02x)", ifdPath, tagId) - - // Never called. - return EncodedData{}, nil -} - -// TODO(dustin): Rename `TagUnknownType_UnknownValue` to `TagUndefinedType_UnknownValue`. - -type TagUnknownType_UnknownValue []byte - -func (tutuv TagUnknownType_UnknownValue) String() string { - parts := make([]string, len(tutuv)) - for i, c := range tutuv { - parts[i] = fmt.Sprintf("%02x", c) - } - - h := sha1.New() - - _, err := h.Write(tutuv) - log.PanicIf(err) - - digest := h.Sum(nil) - - return fmt.Sprintf("Unknown", strings.Join(parts, " "), len(tutuv), digest) -} - -// UndefinedValue knows how to resolve the value for most unknown-type tags. -func UndefinedValue(ifdPath string, tagId uint16, valueContext interface{}, byteOrder binary.ByteOrder) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Stop exporting this. Use `(*ValueContext).Undefined()`. - - var valueContextPtr *ValueContext - - if vc, ok := valueContext.(*ValueContext); ok == true { - // Legacy usage. - - valueContextPtr = vc - } else { - // Standard usage. - - valueContextValue := valueContext.(ValueContext) - valueContextPtr = &valueContextValue - } - - typeLogger.Debugf(nil, "UndefinedValue: IFD-PATH=[%s] TAG-ID=(0x%02x)", ifdPath, tagId) - - if ifdPath == IfdPathStandardExif { - if tagId == 0x9000 { - // ExifVersion - - valueContextPtr.SetUnknownValueType(TypeAsciiNoNul) - - valueString, err := valueContextPtr.ReadAsciiNoNul() - log.PanicIf(err) - - return TagUnknownType_GeneralString(valueString), nil - } else if tagId == 0xa000 { - // FlashpixVersion - - valueContextPtr.SetUnknownValueType(TypeAsciiNoNul) - - valueString, err := valueContextPtr.ReadAsciiNoNul() - log.PanicIf(err) - - return TagUnknownType_GeneralString(valueString), nil - } else if tagId == 0x9286 { - // UserComment - - valueContextPtr.SetUnknownValueType(TypeByte) - - valueBytes, err := valueContextPtr.ReadBytes() - log.PanicIf(err) - - unknownUc := TagUnknownType_9298_UserComment{ - EncodingType: TagUnknownType_9298_UserComment_Encoding_UNDEFINED, - EncodingBytes: []byte{}, - } - - encoding := valueBytes[:8] - for encodingIndex, encodingBytes := range TagUnknownType_9298_UserComment_Encodings { - if bytes.Compare(encoding, encodingBytes) == 0 { - uc := TagUnknownType_9298_UserComment{ - EncodingType: encodingIndex, - EncodingBytes: valueBytes[8:], - } - - return uc, nil - } - } - - typeLogger.Warningf(nil, "User-comment encoding not valid. Returning 'unknown' type (the default).") - return unknownUc, nil - } else if tagId == 0x927c { - // MakerNote - // TODO(dustin): !! This is the Wild Wild West. This very well might be a child IFD, but any and all OEM's define their own formats. If we're going to be writing changes and this is complete EXIF (which may not have the first eight bytes), it might be fine. However, if these are just IFDs they'll be relative to the main EXIF, this will invalidate the MakerNote data for IFDs and any other implementations that use offsets unless we can interpret them all. It be best to return to this later and just exclude this from being written for now, though means a loss of a wealth of image metadata. - // -> We can also just blindly try to interpret as an IFD and just validate that it's looks good (maybe it will even have a 'next ifd' pointer that we can validate is 0x0). - - valueContextPtr.SetUnknownValueType(TypeByte) - - valueBytes, err := valueContextPtr.ReadBytes() - log.PanicIf(err) - - // TODO(dustin): Doesn't work, but here as an example. - // ie := NewIfdEnumerate(valueBytes, byteOrder) - - // // TODO(dustin): !! Validate types (might have proprietary types, but it might be worth splitting the list between valid and not valid; maybe fail if a certain proportion are invalid, or maybe aren't less then a certain small integer)? - // ii, err := ie.Collect(0x0) - - // for _, entry := range ii.RootIfd.Entries { - // fmt.Printf("ENTRY: 0x%02x %d\n", entry.TagId, entry.TagType) - // } - - mn := TagUnknownType_927C_MakerNote{ - MakerNoteType: valueBytes[:20], - - // MakerNoteBytes has the whole length of bytes. There's always - // the chance that the first 20 bytes includes actual data. - MakerNoteBytes: valueBytes, - } - - return mn, nil - } else if tagId == 0x9101 { - // ComponentsConfiguration - - valueContextPtr.SetUnknownValueType(TypeByte) - - valueBytes, err := valueContextPtr.ReadBytes() - log.PanicIf(err) - - for configurationId, configurationBytes := range TagUnknownType_9101_ComponentsConfiguration_Configurations { - if bytes.Compare(valueBytes, configurationBytes) == 0 { - cc := TagUnknownType_9101_ComponentsConfiguration{ - ConfigurationId: configurationId, - ConfigurationBytes: valueBytes, - } - - return cc, nil - } - } - - cc := TagUnknownType_9101_ComponentsConfiguration{ - ConfigurationId: TagUnknownType_9101_ComponentsConfiguration_OTHER, - ConfigurationBytes: valueBytes, - } - - return cc, nil - } - } else if ifdPath == IfdPathStandardGps { - if tagId == 0x001c { - // GPSAreaInformation - - valueContextPtr.SetUnknownValueType(TypeAsciiNoNul) - - valueString, err := valueContextPtr.ReadAsciiNoNul() - log.PanicIf(err) - - return TagUnknownType_GeneralString(valueString), nil - } else if tagId == 0x001b { - // GPSProcessingMethod - - valueContextPtr.SetUnknownValueType(TypeAsciiNoNul) - - valueString, err := valueContextPtr.ReadAsciiNoNul() - log.PanicIf(err) - - return TagUnknownType_GeneralString(valueString), nil - } - } else if ifdPath == IfdPathStandardExifIop { - if tagId == 0x0002 { - // InteropVersion - - valueContextPtr.SetUnknownValueType(TypeAsciiNoNul) - - valueString, err := valueContextPtr.ReadAsciiNoNul() - log.PanicIf(err) - - return TagUnknownType_GeneralString(valueString), nil - } - } - - // TODO(dustin): !! Still need to do: - // - // complex: 0xa302, 0xa20c, 0x8828 - // long: 0xa301, 0xa300 - // - // 0xa40b is device-specific and unhandled. - // - // See https://github.com/dsoprea/go-exif/issues/26. - - // We have no choice but to return the error. We have no way of knowing how - // much data there is without already knowing what data-type this tag is. - return nil, ErrUnhandledUnknownTypedTag -} diff --git a/vendor/github.com/dsoprea/go-exif/type.go b/vendor/github.com/dsoprea/go-exif/type.go deleted file mode 100644 index 2012b6067..000000000 --- a/vendor/github.com/dsoprea/go-exif/type.go +++ /dev/null @@ -1,310 +0,0 @@ -package exif - -import ( - "errors" - "fmt" - "strconv" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -type TagTypePrimitive uint16 - -func (typeType TagTypePrimitive) String() string { - return TypeNames[typeType] -} - -func (tagType TagTypePrimitive) Size() int { - if tagType == TypeByte { - return 1 - } else if tagType == TypeAscii || tagType == TypeAsciiNoNul { - return 1 - } else if tagType == TypeShort { - return 2 - } else if tagType == TypeLong { - return 4 - } else if tagType == TypeRational { - return 8 - } else if tagType == TypeSignedLong { - return 4 - } else if tagType == TypeSignedRational { - return 8 - } else { - log.Panicf("can not determine tag-value size for type (%d): [%s]", tagType, TypeNames[tagType]) - - // Never called. - return 0 - } -} - -const ( - TypeByte TagTypePrimitive = 1 - TypeAscii TagTypePrimitive = 2 - TypeShort TagTypePrimitive = 3 - TypeLong TagTypePrimitive = 4 - TypeRational TagTypePrimitive = 5 - TypeUndefined TagTypePrimitive = 7 - TypeSignedLong TagTypePrimitive = 9 - TypeSignedRational TagTypePrimitive = 10 - - // TypeAsciiNoNul is just a pseudo-type, for our own purposes. - TypeAsciiNoNul TagTypePrimitive = 0xf0 -) - -var ( - typeLogger = log.NewLogger("exif.type") -) - -var ( - // TODO(dustin): Rename TypeNames() to typeNames() and add getter. - TypeNames = map[TagTypePrimitive]string{ - TypeByte: "BYTE", - TypeAscii: "ASCII", - TypeShort: "SHORT", - TypeLong: "LONG", - TypeRational: "RATIONAL", - TypeUndefined: "UNDEFINED", - TypeSignedLong: "SLONG", - TypeSignedRational: "SRATIONAL", - - TypeAsciiNoNul: "_ASCII_NO_NUL", - } - - TypeNamesR = map[string]TagTypePrimitive{} -) - -var ( - // ErrNotEnoughData is used when there isn't enough data to accomodate what - // we're trying to parse (sizeof(type) * unit_count). - ErrNotEnoughData = errors.New("not enough data for type") - - // ErrWrongType is used when we try to parse anything other than the - // current type. - ErrWrongType = errors.New("wrong type, can not parse") - - // ErrUnhandledUnknownTag is used when we try to parse a tag that's - // recorded as an "unknown" type but not a documented tag (therefore - // leaving us not knowning how to read it). - ErrUnhandledUnknownTypedTag = errors.New("not a standard unknown-typed tag") -) - -type Rational struct { - Numerator uint32 - Denominator uint32 -} - -type SignedRational struct { - Numerator int32 - Denominator int32 -} - -func TagTypeSize(tagType TagTypePrimitive) int { - - // DEPRECATED(dustin): `(TagTypePrimitive).Size()` should be used, directly. - - return tagType.Size() -} - -// Format returns a stringified value for the given bytes. Automatically -// calculates count based on type size. -func Format(rawBytes []byte, tagType TagTypePrimitive, justFirst bool, byteOrder binary.ByteOrder) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add tests - - typeSize := tagType.Size() - - if len(rawBytes)%typeSize != 0 { - log.Panicf("byte-count (%d) does not align for [%s] type with a size of (%d) bytes", len(rawBytes), TypeNames[tagType], typeSize) - } - - // unitCount is the calculated unit-count. This should equal the original - // value from the tag (pre-resolution). - unitCount := uint32(len(rawBytes) / typeSize) - - // Truncate the items if it's not bytes or a string and we just want the first. - - valueSuffix := "" - if justFirst == true && unitCount > 1 && tagType != TypeByte && tagType != TypeAscii && tagType != TypeAsciiNoNul { - unitCount = 1 - valueSuffix = "..." - } - - if tagType == TypeByte { - items, err := parser.ParseBytes(rawBytes, unitCount) - log.PanicIf(err) - - return DumpBytesToString(items), nil - } else if tagType == TypeAscii { - phrase, err := parser.ParseAscii(rawBytes, unitCount) - log.PanicIf(err) - - return phrase, nil - } else if tagType == TypeAsciiNoNul { - phrase, err := parser.ParseAsciiNoNul(rawBytes, unitCount) - log.PanicIf(err) - - return phrase, nil - } else if tagType == TypeShort { - items, err := parser.ParseShorts(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - - if len(items) > 0 { - if justFirst == true { - return fmt.Sprintf("%v%s", items[0], valueSuffix), nil - } else { - return fmt.Sprintf("%v", items), nil - } - } else { - return "", nil - } - } else if tagType == TypeLong { - items, err := parser.ParseLongs(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - - if len(items) > 0 { - if justFirst == true { - return fmt.Sprintf("%v%s", items[0], valueSuffix), nil - } else { - return fmt.Sprintf("%v", items), nil - } - } else { - return "", nil - } - } else if tagType == TypeRational { - items, err := parser.ParseRationals(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - - if len(items) > 0 { - parts := make([]string, len(items)) - for i, r := range items { - parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) - } - - if justFirst == true { - return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil - } else { - return fmt.Sprintf("%v", parts), nil - } - } else { - return "", nil - } - } else if tagType == TypeSignedLong { - items, err := parser.ParseSignedLongs(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - - if len(items) > 0 { - if justFirst == true { - return fmt.Sprintf("%v%s", items[0], valueSuffix), nil - } else { - return fmt.Sprintf("%v", items), nil - } - } else { - return "", nil - } - } else if tagType == TypeSignedRational { - items, err := parser.ParseSignedRationals(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - - parts := make([]string, len(items)) - for i, r := range items { - parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) - } - - if len(items) > 0 { - if justFirst == true { - return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil - } else { - return fmt.Sprintf("%v", parts), nil - } - } else { - return "", nil - } - } else { - // Affects only "unknown" values, in general. - log.Panicf("value of type [%s] can not be formatted into string", tagType.String()) - - // Never called. - return "", nil - } -} - -func EncodeStringToBytes(tagType TagTypePrimitive, valueString string) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if tagType == TypeUndefined { - // TODO(dustin): Circle back to this. - log.Panicf("undefined-type values are not supported") - } - - if tagType == TypeByte { - return []byte(valueString), nil - } else if tagType == TypeAscii || tagType == TypeAsciiNoNul { - // Whether or not we're putting an NUL on the end is only relevant for - // byte-level encoding. This function really just supports a user - // interface. - - return valueString, nil - } else if tagType == TypeShort { - n, err := strconv.ParseUint(valueString, 10, 16) - log.PanicIf(err) - - return uint16(n), nil - } else if tagType == TypeLong { - n, err := strconv.ParseUint(valueString, 10, 32) - log.PanicIf(err) - - return uint32(n), nil - } else if tagType == TypeRational { - parts := strings.SplitN(valueString, "/", 2) - - numerator, err := strconv.ParseUint(parts[0], 10, 32) - log.PanicIf(err) - - denominator, err := strconv.ParseUint(parts[1], 10, 32) - log.PanicIf(err) - - return Rational{ - Numerator: uint32(numerator), - Denominator: uint32(denominator), - }, nil - } else if tagType == TypeSignedLong { - n, err := strconv.ParseInt(valueString, 10, 32) - log.PanicIf(err) - - return int32(n), nil - } else if tagType == TypeSignedRational { - parts := strings.SplitN(valueString, "/", 2) - - numerator, err := strconv.ParseInt(parts[0], 10, 32) - log.PanicIf(err) - - denominator, err := strconv.ParseInt(parts[1], 10, 32) - log.PanicIf(err) - - return SignedRational{ - Numerator: int32(numerator), - Denominator: int32(denominator), - }, nil - } - - log.Panicf("from-string encoding for type not supported; this shouldn't happen: [%s]", tagType.String()) - return nil, nil -} - -func init() { - for typeId, typeName := range TypeNames { - TypeNamesR[typeName] = typeId - } -} diff --git a/vendor/github.com/dsoprea/go-exif/type_encode.go b/vendor/github.com/dsoprea/go-exif/type_encode.go deleted file mode 100644 index 9ff754916..000000000 --- a/vendor/github.com/dsoprea/go-exif/type_encode.go +++ /dev/null @@ -1,262 +0,0 @@ -package exif - -import ( - "bytes" - "reflect" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - typeEncodeLogger = log.NewLogger("exif.type_encode") -) - -// EncodedData encapsulates the compound output of an encoding operation. -type EncodedData struct { - Type TagTypePrimitive - Encoded []byte - - // TODO(dustin): Is this really necessary? We might have this just to correlate to the incoming stream format (raw bytes and a unit-count both for incoming and outgoing). - UnitCount uint32 -} - -type ValueEncoder struct { - byteOrder binary.ByteOrder -} - -func NewValueEncoder(byteOrder binary.ByteOrder) *ValueEncoder { - return &ValueEncoder{ - byteOrder: byteOrder, - } -} - -func (ve *ValueEncoder) encodeBytes(value []uint8) (ed EncodedData, err error) { - ed.Type = TypeByte - ed.Encoded = []byte(value) - ed.UnitCount = uint32(len(value)) - - return ed, nil -} - -func (ve *ValueEncoder) encodeAscii(value string) (ed EncodedData, err error) { - ed.Type = TypeAscii - - ed.Encoded = []byte(value) - ed.Encoded = append(ed.Encoded, 0) - - ed.UnitCount = uint32(len(ed.Encoded)) - - return ed, nil -} - -// encodeAsciiNoNul returns a string encoded as a byte-string without a trailing -// NUL byte. -// -// Note that: -// -// 1. This type can not be automatically encoded using `Encode()`. The default -// mode is to encode *with* a trailing NUL byte using `encodeAscii`. Only -// certain undefined-type tags using an unterminated ASCII string and these -// are exceptional in nature. -// -// 2. The presence of this method allows us to completely test the complimentary -// no-nul parser. -// -func (ve *ValueEncoder) encodeAsciiNoNul(value string) (ed EncodedData, err error) { - ed.Type = TypeAsciiNoNul - ed.Encoded = []byte(value) - ed.UnitCount = uint32(len(ed.Encoded)) - - return ed, nil -} - -func (ve *ValueEncoder) encodeShorts(value []uint16) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*2) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint16(ed.Encoded[i*2:(i+1)*2], value[i]) - } - - ed.Type = TypeShort - - return ed, nil -} - -func (ve *ValueEncoder) encodeLongs(value []uint32) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*4) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint32(ed.Encoded[i*4:(i+1)*4], value[i]) - } - - ed.Type = TypeLong - - return ed, nil -} - -func (ve *ValueEncoder) encodeRationals(value []Rational) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*8) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint32(ed.Encoded[i*8+0:i*8+4], value[i].Numerator) - ve.byteOrder.PutUint32(ed.Encoded[i*8+4:i*8+8], value[i].Denominator) - } - - ed.Type = TypeRational - - return ed, nil -} - -func (ve *ValueEncoder) encodeSignedLongs(value []int32) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - - b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) - - for i := uint32(0); i < ed.UnitCount; i++ { - err := binary.Write(b, ve.byteOrder, value[i]) - log.PanicIf(err) - } - - ed.Type = TypeSignedLong - ed.Encoded = b.Bytes() - - return ed, nil -} - -func (ve *ValueEncoder) encodeSignedRationals(value []SignedRational) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - - b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) - - for i := uint32(0); i < ed.UnitCount; i++ { - err := binary.Write(b, ve.byteOrder, value[i].Numerator) - log.PanicIf(err) - - err = binary.Write(b, ve.byteOrder, value[i].Denominator) - log.PanicIf(err) - } - - ed.Type = TypeSignedRational - ed.Encoded = b.Bytes() - - return ed, nil -} - -// Encode returns bytes for the given value, infering type from the actual -// value. This does not support `TypeAsciiNoNull` (all strings are encoded as -// `TypeAscii`). -func (ve *ValueEncoder) Encode(value interface{}) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): This is redundant with EncodeWithType. Refactor one to use the other. - - switch value.(type) { - case []byte: - ed, err = ve.encodeBytes(value.([]byte)) - log.PanicIf(err) - case string: - ed, err = ve.encodeAscii(value.(string)) - log.PanicIf(err) - case []uint16: - ed, err = ve.encodeShorts(value.([]uint16)) - log.PanicIf(err) - case []uint32: - ed, err = ve.encodeLongs(value.([]uint32)) - log.PanicIf(err) - case []Rational: - ed, err = ve.encodeRationals(value.([]Rational)) - log.PanicIf(err) - case []int32: - ed, err = ve.encodeSignedLongs(value.([]int32)) - log.PanicIf(err) - case []SignedRational: - ed, err = ve.encodeSignedRationals(value.([]SignedRational)) - log.PanicIf(err) - default: - log.Panicf("value not encodable: [%s] [%v]", reflect.TypeOf(value), value) - } - - return ed, nil -} - -// EncodeWithType returns bytes for the given value, using the given `TagType` -// value to determine how to encode. This supports `TypeAsciiNoNul`. -func (ve *ValueEncoder) EncodeWithType(tt TagType, value interface{}) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): This is redundant with Encode. Refactor one to use the other. - - switch tt.Type() { - case TypeByte: - ed, err = ve.encodeBytes(value.([]byte)) - log.PanicIf(err) - case TypeAscii: - ed, err = ve.encodeAscii(value.(string)) - log.PanicIf(err) - case TypeAsciiNoNul: - ed, err = ve.encodeAsciiNoNul(value.(string)) - log.PanicIf(err) - case TypeShort: - ed, err = ve.encodeShorts(value.([]uint16)) - log.PanicIf(err) - case TypeLong: - ed, err = ve.encodeLongs(value.([]uint32)) - log.PanicIf(err) - case TypeRational: - ed, err = ve.encodeRationals(value.([]Rational)) - log.PanicIf(err) - case TypeSignedLong: - ed, err = ve.encodeSignedLongs(value.([]int32)) - log.PanicIf(err) - case TypeSignedRational: - ed, err = ve.encodeSignedRationals(value.([]SignedRational)) - log.PanicIf(err) - default: - log.Panicf("value not encodable (with type): %v [%v]", tt, value) - } - - return ed, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/utility.go b/vendor/github.com/dsoprea/go-exif/utility.go deleted file mode 100644 index 3d1ea2489..000000000 --- a/vendor/github.com/dsoprea/go-exif/utility.go +++ /dev/null @@ -1,222 +0,0 @@ -package exif - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "time" - - "github.com/dsoprea/go-logging" -) - -func DumpBytes(data []byte) { - fmt.Printf("DUMP: ") - for _, x := range data { - fmt.Printf("%02x ", x) - } - - fmt.Printf("\n") -} - -func DumpBytesClause(data []byte) { - fmt.Printf("DUMP: ") - - fmt.Printf("[]byte { ") - - for i, x := range data { - fmt.Printf("0x%02x", x) - - if i < len(data)-1 { - fmt.Printf(", ") - } - } - - fmt.Printf(" }\n") -} - -func DumpBytesToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteRune(' ') - log.PanicIf(err) - } - } - - return b.String() -} - -func DumpBytesClauseToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteString(", ") - log.PanicIf(err) - } - } - - return b.String() -} - -// ParseExifFullTimestamp parses dates like "2018:11:30 13:01:49" into a UTC -// `time.Time` struct. -func ParseExifFullTimestamp(fullTimestampPhrase string) (timestamp time.Time, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - parts := strings.Split(fullTimestampPhrase, " ") - datestampValue, timestampValue := parts[0], parts[1] - - dateParts := strings.Split(datestampValue, ":") - - year, err := strconv.ParseUint(dateParts[0], 10, 16) - if err != nil { - log.Panicf("could not parse year") - } - - month, err := strconv.ParseUint(dateParts[1], 10, 8) - if err != nil { - log.Panicf("could not parse month") - } - - day, err := strconv.ParseUint(dateParts[2], 10, 8) - if err != nil { - log.Panicf("could not parse day") - } - - timeParts := strings.Split(timestampValue, ":") - - hour, err := strconv.ParseUint(timeParts[0], 10, 8) - if err != nil { - log.Panicf("could not parse hour") - } - - minute, err := strconv.ParseUint(timeParts[1], 10, 8) - if err != nil { - log.Panicf("could not parse minute") - } - - second, err := strconv.ParseUint(timeParts[2], 10, 8) - if err != nil { - log.Panicf("could not parse second") - } - - timestamp = time.Date(int(year), time.Month(month), int(day), int(hour), int(minute), int(second), 0, time.UTC) - return timestamp, nil -} - -// ExifFullTimestampString produces a string like "2018:11:30 13:01:49" from a -// `time.Time` struct. It will attempt to convert to UTC first. -func ExifFullTimestampString(t time.Time) (fullTimestampPhrase string) { - t = t.UTC() - - return fmt.Sprintf("%04d:%02d:%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) -} - -// ExifTag is one simple representation of a tag in a flat list of all of them. -type ExifTag struct { - IfdPath string `json:"ifd_path"` - - TagId uint16 `json:"id"` - TagName string `json:"name"` - - TagTypeId TagTypePrimitive `json:"type_id"` - TagTypeName string `json:"type_name"` - Value interface{} `json:"value"` - ValueBytes []byte `json:"value_bytes"` - - ChildIfdPath string `json:"child_ifd_path"` -} - -// String returns a string representation. -func (et ExifTag) String() string { - return fmt.Sprintf("ExifTag 0 { - var ifd *Ifd - ifd, q = q[0], q[1:] - - ti := NewTagIndex() - for _, ite := range ifd.Entries { - tagName := "" - - it, err := ti.Get(ifd.IfdPath, ite.TagId) - if err != nil { - // If it's a non-standard tag, just leave the name blank. - if log.Is(err, ErrTagNotFound) != true { - log.PanicIf(err) - } - } else { - tagName = it.Name - } - - value, err := ifd.TagValue(ite) - if err != nil { - if err == ErrUnhandledUnknownTypedTag { - value = UnparseableUnknownTagValuePlaceholder - } else { - log.Panic(err) - } - } - - valueBytes, err := ifd.TagValueBytes(ite) - if err != nil && err != ErrUnhandledUnknownTypedTag { - log.Panic(err) - } - - et := ExifTag{ - IfdPath: ifd.IfdPath, - TagId: ite.TagId, - TagName: tagName, - TagTypeId: ite.TagType, - TagTypeName: TypeNames[ite.TagType], - Value: value, - ValueBytes: valueBytes, - ChildIfdPath: ite.ChildIfdPath, - } - - exifTags = append(exifTags, et) - } - - for _, childIfd := range ifd.Children { - q = append(q, childIfd) - } - - if ifd.NextIfd != nil { - q = append(q, ifd.NextIfd) - } - } - - return exifTags, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/.MODULE_ROOT b/vendor/github.com/dsoprea/go-exif/v2/.MODULE_ROOT deleted file mode 100644 index e69de29bb..000000000 diff --git a/vendor/github.com/dsoprea/go-exif/v2/LICENSE b/vendor/github.com/dsoprea/go-exif/v2/LICENSE deleted file mode 100644 index 0b9358a3a..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT LICENSE - -Copyright 2019 Dustin Oprea - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/ifd.go b/vendor/github.com/dsoprea/go-exif/v2/common/ifd.go deleted file mode 100644 index 9b93f04d9..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/ifd.go +++ /dev/null @@ -1,659 +0,0 @@ -package exifcommon - -import ( - "errors" - "fmt" - "strings" - - "github.com/dsoprea/go-logging" -) - -var ( - ifdLogger = log.NewLogger("exifcommon.ifd") -) - -var ( - ErrChildIfdNotMapped = errors.New("no child-IFD for that tag-ID under parent") -) - -// MappedIfd is one node in the IFD-mapping. -type MappedIfd struct { - ParentTagId uint16 - Placement []uint16 - Path []string - - Name string - TagId uint16 - Children map[uint16]*MappedIfd -} - -// String returns a descriptive string. -func (mi *MappedIfd) String() string { - pathPhrase := mi.PathPhrase() - return fmt.Sprintf("MappedIfd<(0x%04X) [%s] PATH=[%s]>", mi.TagId, mi.Name, pathPhrase) -} - -// PathPhrase returns a non-fully-qualified IFD path. -func (mi *MappedIfd) PathPhrase() string { - return strings.Join(mi.Path, "/") -} - -// TODO(dustin): Refactor this to use IfdIdentity structs. - -// IfdMapping describes all of the IFDs that we currently recognize. -type IfdMapping struct { - rootNode *MappedIfd -} - -// NewIfdMapping returns a new IfdMapping struct. -func NewIfdMapping() (ifdMapping *IfdMapping) { - rootNode := &MappedIfd{ - Path: make([]string, 0), - Children: make(map[uint16]*MappedIfd), - } - - return &IfdMapping{ - rootNode: rootNode, - } -} - -// NewIfdMappingWithStandard retruns a new IfdMapping struct preloaded with the -// standard IFDs. -func NewIfdMappingWithStandard() (ifdMapping *IfdMapping) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - im := NewIfdMapping() - - err := LoadStandardIfds(im) - log.PanicIf(err) - - return im -} - -// Get returns the node given the path slice. -func (im *IfdMapping) Get(parentPlacement []uint16) (childIfd *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ptr := im.rootNode - for _, tagId := range parentPlacement { - if descendantPtr, found := ptr.Children[tagId]; found == false { - log.Panicf("ifd child with tag-ID (%04x) not registered: [%s]", tagId, ptr.PathPhrase()) - } else { - ptr = descendantPtr - } - } - - return ptr, nil -} - -// GetWithPath returns the node given the path string. -func (im *IfdMapping) GetWithPath(pathPhrase string) (mi *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if pathPhrase == "" { - log.Panicf("path-phrase is empty") - } - - path := strings.Split(pathPhrase, "/") - ptr := im.rootNode - - for _, name := range path { - var hit *MappedIfd - for _, mi := range ptr.Children { - if mi.Name == name { - hit = mi - break - } - } - - if hit == nil { - log.Panicf("ifd child with name [%s] not registered: [%s]", name, ptr.PathPhrase()) - } - - ptr = hit - } - - return ptr, nil -} - -// GetChild is a convenience function to get the child path for a given parent -// placement and child tag-ID. -func (im *IfdMapping) GetChild(parentPathPhrase string, tagId uint16) (mi *MappedIfd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - mi, err = im.GetWithPath(parentPathPhrase) - log.PanicIf(err) - - for _, childMi := range mi.Children { - if childMi.TagId == tagId { - return childMi, nil - } - } - - // Whether or not an IFD is defined in data, such an IFD is not registered - // and would be unknown. - log.Panic(ErrChildIfdNotMapped) - return nil, nil -} - -// IfdTagIdAndIndex represents a specific part of the IFD path. -// -// This is a legacy type. -type IfdTagIdAndIndex struct { - Name string - TagId uint16 - Index int -} - -// String returns a descriptive string. -func (itii IfdTagIdAndIndex) String() string { - return fmt.Sprintf("IfdTagIdAndIndex", itii.Name, itii.TagId, itii.Index) -} - -// ResolvePath takes a list of names, which can also be suffixed with indices -// (to identify the second, third, etc.. sibling IFD) and returns a list of -// tag-IDs and those indices. -// -// Example: -// -// - IFD/Exif/Iop -// - IFD0/Exif/Iop -// -// This is the only call that supports adding the numeric indices. -func (im *IfdMapping) ResolvePath(pathPhrase string) (lineage []IfdTagIdAndIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - pathPhrase = strings.TrimSpace(pathPhrase) - - if pathPhrase == "" { - log.Panicf("can not resolve empty path-phrase") - } - - path := strings.Split(pathPhrase, "/") - lineage = make([]IfdTagIdAndIndex, len(path)) - - ptr := im.rootNode - empty := IfdTagIdAndIndex{} - for i, name := range path { - indexByte := name[len(name)-1] - index := 0 - if indexByte >= '0' && indexByte <= '9' { - index = int(indexByte - '0') - name = name[:len(name)-1] - } - - itii := IfdTagIdAndIndex{} - for _, mi := range ptr.Children { - if mi.Name != name { - continue - } - - itii.Name = name - itii.TagId = mi.TagId - itii.Index = index - - ptr = mi - - break - } - - if itii == empty { - log.Panicf("ifd child with name [%s] not registered: [%s]", name, pathPhrase) - } - - lineage[i] = itii - } - - return lineage, nil -} - -// FqPathPhraseFromLineage returns the fully-qualified IFD path from the slice. -func (im *IfdMapping) FqPathPhraseFromLineage(lineage []IfdTagIdAndIndex) (fqPathPhrase string) { - fqPathParts := make([]string, len(lineage)) - for i, itii := range lineage { - if itii.Index > 0 { - fqPathParts[i] = fmt.Sprintf("%s%d", itii.Name, itii.Index) - } else { - fqPathParts[i] = itii.Name - } - } - - return strings.Join(fqPathParts, "/") -} - -// PathPhraseFromLineage returns the non-fully-qualified IFD path from the -// slice. -func (im *IfdMapping) PathPhraseFromLineage(lineage []IfdTagIdAndIndex) (pathPhrase string) { - pathParts := make([]string, len(lineage)) - for i, itii := range lineage { - pathParts[i] = itii.Name - } - - return strings.Join(pathParts, "/") -} - -// StripPathPhraseIndices returns a non-fully-qualified path-phrase (no -// indices). -func (im *IfdMapping) StripPathPhraseIndices(pathPhrase string) (strippedPathPhrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - lineage, err := im.ResolvePath(pathPhrase) - log.PanicIf(err) - - strippedPathPhrase = im.PathPhraseFromLineage(lineage) - return strippedPathPhrase, nil -} - -// Add puts the given IFD at the given position of the tree. The position of the -// tree is referred to as the placement and is represented by a set of tag-IDs, -// where the leftmost is the root tag and the tags going to the right are -// progressive descendants. -func (im *IfdMapping) Add(parentPlacement []uint16, tagId uint16, name string) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! It would be nicer to provide a list of names in the placement rather than tag-IDs. - - ptr, err := im.Get(parentPlacement) - log.PanicIf(err) - - path := make([]string, len(parentPlacement)+1) - if len(parentPlacement) > 0 { - copy(path, ptr.Path) - } - - path[len(path)-1] = name - - placement := make([]uint16, len(parentPlacement)+1) - if len(placement) > 0 { - copy(placement, ptr.Placement) - } - - placement[len(placement)-1] = tagId - - childIfd := &MappedIfd{ - ParentTagId: ptr.TagId, - Path: path, - Placement: placement, - Name: name, - TagId: tagId, - Children: make(map[uint16]*MappedIfd), - } - - if _, found := ptr.Children[tagId]; found == true { - log.Panicf("child IFD with tag-ID (%04x) already registered under IFD [%s] with tag-ID (%04x)", tagId, ptr.Name, ptr.TagId) - } - - ptr.Children[tagId] = childIfd - - return nil -} - -func (im *IfdMapping) dumpLineages(stack []*MappedIfd, input []string) (output []string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - currentIfd := stack[len(stack)-1] - - output = input - for _, childIfd := range currentIfd.Children { - stackCopy := make([]*MappedIfd, len(stack)+1) - - copy(stackCopy, stack) - stackCopy[len(stack)] = childIfd - - // Add to output, but don't include the obligatory root node. - parts := make([]string, len(stackCopy)-1) - for i, mi := range stackCopy[1:] { - parts[i] = mi.Name - } - - output = append(output, strings.Join(parts, "/")) - - output, err = im.dumpLineages(stackCopy, output) - log.PanicIf(err) - } - - return output, nil -} - -// DumpLineages returns a slice of strings representing all mappings. -func (im *IfdMapping) DumpLineages() (output []string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - stack := []*MappedIfd{im.rootNode} - output = make([]string, 0) - - output, err = im.dumpLineages(stack, output) - log.PanicIf(err) - - return output, nil -} - -// LoadStandardIfds loads the standard IFDs into the mapping. -func LoadStandardIfds(im *IfdMapping) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = im.Add( - []uint16{}, - IfdStandardIfdIdentity.TagId(), IfdStandardIfdIdentity.Name()) - - log.PanicIf(err) - - err = im.Add( - []uint16{IfdStandardIfdIdentity.TagId()}, - IfdExifStandardIfdIdentity.TagId(), IfdExifStandardIfdIdentity.Name()) - - log.PanicIf(err) - - err = im.Add( - []uint16{IfdStandardIfdIdentity.TagId(), IfdExifStandardIfdIdentity.TagId()}, - IfdExifIopStandardIfdIdentity.TagId(), IfdExifIopStandardIfdIdentity.Name()) - - log.PanicIf(err) - - err = im.Add( - []uint16{IfdStandardIfdIdentity.TagId()}, - IfdGpsInfoStandardIfdIdentity.TagId(), IfdGpsInfoStandardIfdIdentity.Name()) - - log.PanicIf(err) - - return nil -} - -// IfdTag describes a single IFD tag and its parent (if any). -type IfdTag struct { - parentIfdTag *IfdTag - tagId uint16 - name string -} - -func NewIfdTag(parentIfdTag *IfdTag, tagId uint16, name string) IfdTag { - return IfdTag{ - parentIfdTag: parentIfdTag, - tagId: tagId, - name: name, - } -} - -// ParentIfd returns the IfdTag of this IFD's parent. -func (it IfdTag) ParentIfd() *IfdTag { - return it.parentIfdTag -} - -// TagId returns the tag-ID of this IFD. -func (it IfdTag) TagId() uint16 { - return it.tagId -} - -// Name returns the simple name of this IFD. -func (it IfdTag) Name() string { - return it.name -} - -// String returns a descriptive string. -func (it IfdTag) String() string { - parentIfdPhrase := "" - if it.parentIfdTag != nil { - parentIfdPhrase = fmt.Sprintf(" PARENT=(0x%04x)[%s]", it.parentIfdTag.tagId, it.parentIfdTag.name) - } - - return fmt.Sprintf("IfdTag", it.tagId, it.name, parentIfdPhrase) -} - -var ( - // rootStandardIfd is the standard root IFD. - rootStandardIfd = NewIfdTag(nil, 0x0000, "IFD") // IFD - - // exifStandardIfd is the standard "Exif" IFD. - exifStandardIfd = NewIfdTag(&rootStandardIfd, 0x8769, "Exif") // IFD/Exif - - // iopStandardIfd is the standard "Iop" IFD. - iopStandardIfd = NewIfdTag(&exifStandardIfd, 0xA005, "Iop") // IFD/Exif/Iop - - // gpsInfoStandardIfd is the standard "GPS" IFD. - gpsInfoStandardIfd = NewIfdTag(&rootStandardIfd, 0x8825, "GPSInfo") // IFD/GPSInfo -) - -// IfdIdentityPart represents one component in an IFD path. -type IfdIdentityPart struct { - Name string - Index int -} - -// String returns a fully-qualified IFD path. -func (iip IfdIdentityPart) String() string { - if iip.Index > 0 { - return fmt.Sprintf("%s%d", iip.Name, iip.Index) - } else { - return iip.Name - } -} - -// UnindexedString returned a non-fully-qualified IFD path. -func (iip IfdIdentityPart) UnindexedString() string { - return iip.Name -} - -// IfdIdentity represents a single IFD path and provides access to various -// information and representations. -// -// Only global instances can be used for equality checks. -type IfdIdentity struct { - ifdTag IfdTag - parts []IfdIdentityPart - ifdPath string - fqIfdPath string -} - -// NewIfdIdentity returns a new IfdIdentity struct. -func NewIfdIdentity(ifdTag IfdTag, parts ...IfdIdentityPart) (ii *IfdIdentity) { - ii = &IfdIdentity{ - ifdTag: ifdTag, - parts: parts, - } - - ii.ifdPath = ii.getIfdPath() - ii.fqIfdPath = ii.getFqIfdPath() - - return ii -} - -// NewIfdIdentityFromString parses a string like "IFD/Exif" or "IFD1" or -// something more exotic with custom IFDs ("SomeIFD4/SomeChildIFD6"). Note that -// this will valid the unindexed IFD structure (because the standard tags from -// the specification are unindexed), but not, obviously, any indices (e.g. -// the numbers in "IFD0", "IFD1", "SomeIFD4/SomeChildIFD6"). It is -// required for the caller to check whether these specific instances -// were actually parsed out of the stream. -func NewIfdIdentityFromString(im *IfdMapping, fqIfdPath string) (ii *IfdIdentity, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - lineage, err := im.ResolvePath(fqIfdPath) - log.PanicIf(err) - - var lastIt *IfdTag - identityParts := make([]IfdIdentityPart, len(lineage)) - for i, itii := range lineage { - // Build out the tag that will eventually point to the IFD represented - // by the right-most part in the IFD path. - - it := &IfdTag{ - parentIfdTag: lastIt, - tagId: itii.TagId, - name: itii.Name, - } - - lastIt = it - - // Create the next IfdIdentity part. - - iip := IfdIdentityPart{ - Name: itii.Name, - Index: itii.Index, - } - - identityParts[i] = iip - } - - ii = NewIfdIdentity(*lastIt, identityParts...) - return ii, nil -} - -func (ii *IfdIdentity) getFqIfdPath() string { - partPhrases := make([]string, len(ii.parts)) - for i, iip := range ii.parts { - partPhrases[i] = iip.String() - } - - return strings.Join(partPhrases, "/") -} - -func (ii *IfdIdentity) getIfdPath() string { - partPhrases := make([]string, len(ii.parts)) - for i, iip := range ii.parts { - partPhrases[i] = iip.UnindexedString() - } - - return strings.Join(partPhrases, "/") -} - -// String returns a fully-qualified IFD path. -func (ii *IfdIdentity) String() string { - return ii.fqIfdPath -} - -// UnindexedString returns a non-fully-qualified IFD path. -func (ii *IfdIdentity) UnindexedString() string { - return ii.ifdPath -} - -// IfdTag returns the tag struct behind this IFD. -func (ii *IfdIdentity) IfdTag() IfdTag { - return ii.ifdTag -} - -// TagId returns the tag-ID of the IFD. -func (ii *IfdIdentity) TagId() uint16 { - return ii.ifdTag.TagId() -} - -// LeafPathPart returns the last right-most path-part, which represents the -// current IFD. -func (ii *IfdIdentity) LeafPathPart() IfdIdentityPart { - return ii.parts[len(ii.parts)-1] -} - -// Name returns the simple name of this IFD. -func (ii *IfdIdentity) Name() string { - return ii.LeafPathPart().Name -} - -// Index returns the index of this IFD (more then one IFD under a parent IFD -// will be numbered [0..n]). -func (ii *IfdIdentity) Index() int { - return ii.LeafPathPart().Index -} - -// Equals returns true if the two IfdIdentity instances are effectively -// identical. -// -// Since there's no way to get a specific fully-qualified IFD path without a -// certain slice of parts and all other fields are also derived from this, -// checking that the fully-qualified IFD path is equals is sufficient. -func (ii *IfdIdentity) Equals(ii2 *IfdIdentity) bool { - return ii.String() == ii2.String() -} - -// NewChild creates an IfdIdentity for an IFD that is a child of the current -// IFD. -func (ii *IfdIdentity) NewChild(childIfdTag IfdTag, index int) (iiChild *IfdIdentity) { - if *childIfdTag.parentIfdTag != ii.ifdTag { - log.Panicf("can not add child; we are not the parent:\nUS=%v\nCHILD=%v", ii.ifdTag, childIfdTag) - } - - childPart := IfdIdentityPart{childIfdTag.name, index} - childParts := append(ii.parts, childPart) - - iiChild = NewIfdIdentity(childIfdTag, childParts...) - return iiChild -} - -// NewSibling creates an IfdIdentity for an IFD that is a sibling to the current -// one. -func (ii *IfdIdentity) NewSibling(index int) (iiSibling *IfdIdentity) { - parts := make([]IfdIdentityPart, len(ii.parts)) - - copy(parts, ii.parts) - parts[len(parts)-1].Index = index - - iiSibling = NewIfdIdentity(ii.ifdTag, parts...) - return iiSibling -} - -var ( - // IfdStandardIfdIdentity represents the IFD path for IFD0. - IfdStandardIfdIdentity = NewIfdIdentity(rootStandardIfd, IfdIdentityPart{"IFD", 0}) - - // IfdExifStandardIfdIdentity represents the IFD path for IFD0/Exif0. - IfdExifStandardIfdIdentity = IfdStandardIfdIdentity.NewChild(exifStandardIfd, 0) - - // IfdExifIopStandardIfdIdentity represents the IFD path for IFD0/Exif0/Iop0. - IfdExifIopStandardIfdIdentity = IfdExifStandardIfdIdentity.NewChild(iopStandardIfd, 0) - - // IfdGPSInfoStandardIfdIdentity represents the IFD path for IFD0/GPSInfo0. - IfdGpsInfoStandardIfdIdentity = IfdStandardIfdIdentity.NewChild(gpsInfoStandardIfd, 0) - - // Ifd1StandardIfdIdentity represents the IFD path for IFD1. - Ifd1StandardIfdIdentity = NewIfdIdentity(rootStandardIfd, IfdIdentityPart{"IFD", 1}) -) - -var ( - IfdPathStandard = IfdStandardIfdIdentity - IfdPathStandardExif = IfdExifStandardIfdIdentity - IfdPathStandardExifIop = IfdExifIopStandardIfdIdentity - IfdPathStandardGps = IfdGpsInfoStandardIfdIdentity -) diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/parser.go b/vendor/github.com/dsoprea/go-exif/v2/common/parser.go deleted file mode 100644 index bbdd8f53a..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/parser.go +++ /dev/null @@ -1,219 +0,0 @@ -package exifcommon - -import ( - "bytes" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - parserLogger = log.NewLogger("exifcommon.parser") -) - -// Parser knows how to parse all well-defined, encoded EXIF types. -type Parser struct { -} - -// ParseBytesknows how to parse a byte-type value. -func (p *Parser) ParseBytes(data []byte, unitCount uint32) (value []uint8, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeByte.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = []uint8(data[:count]) - - return value, nil -} - -// ParseAscii returns a string and auto-strips the trailing NUL character that -// should be at the end of the encoding. -func (p *Parser) ParseAscii(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeAscii.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - if len(data) == 0 || data[count-1] != 0 { - s := string(data[:count]) - parserLogger.Warningf(nil, "ascii not terminated with nul as expected: [%v]", s) - - return s, nil - } - - // Auto-strip the NUL from the end. It serves no purpose outside of - // encoding semantics. - - return string(data[:count-1]), nil -} - -// ParseAsciiNoNul returns a string without any consideration for a trailing NUL -// character. -func (p *Parser) ParseAsciiNoNul(data []byte, unitCount uint32) (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeAscii.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - return string(data[:count]), nil -} - -// ParseShorts knows how to parse an encoded list of shorts. -func (p *Parser) ParseShorts(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeShort.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]uint16, count) - for i := 0; i < count; i++ { - value[i] = byteOrder.Uint16(data[i*2:]) - } - - return value, nil -} - -// ParseLongs knows how to encode an encoded list of unsigned longs. -func (p *Parser) ParseLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeLong.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]uint32, count) - for i := 0; i < count; i++ { - value[i] = byteOrder.Uint32(data[i*4:]) - } - - return value, nil -} - -// ParseRationals knows how to parse an encoded list of unsigned rationals. -func (p *Parser) ParseRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeRational.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - value = make([]Rational, count) - for i := 0; i < count; i++ { - value[i].Numerator = byteOrder.Uint32(data[i*8:]) - value[i].Denominator = byteOrder.Uint32(data[i*8+4:]) - } - - return value, nil -} - -// ParseSignedLongs knows how to parse an encoded list of signed longs. -func (p *Parser) ParseSignedLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeSignedLong.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - b := bytes.NewBuffer(data) - - value = make([]int32, count) - for i := 0; i < count; i++ { - err := binary.Read(b, byteOrder, &value[i]) - log.PanicIf(err) - } - - return value, nil -} - -// ParseSignedRationals knows how to parse an encoded list of signed -// rationals. -func (p *Parser) ParseSignedRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - count := int(unitCount) - - if len(data) < (TypeSignedRational.Size() * count) { - log.Panic(ErrNotEnoughData) - } - - b := bytes.NewBuffer(data) - - value = make([]SignedRational, count) - for i := 0; i < count; i++ { - err = binary.Read(b, byteOrder, &value[i].Numerator) - log.PanicIf(err) - - err = binary.Read(b, byteOrder, &value[i].Denominator) - log.PanicIf(err) - } - - return value, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/testing_common.go b/vendor/github.com/dsoprea/go-exif/v2/common/testing_common.go deleted file mode 100644 index f04fa22b6..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/testing_common.go +++ /dev/null @@ -1,88 +0,0 @@ -package exifcommon - -import ( - "os" - "path" - - "encoding/binary" - "io/ioutil" - - "github.com/dsoprea/go-logging" -) - -var ( - moduleRootPath = "" - - testExifData []byte = nil - - // EncodeDefaultByteOrder is the default byte-order for encoding operations. - EncodeDefaultByteOrder = binary.BigEndian - - // Default byte order for tests. - TestDefaultByteOrder = binary.BigEndian -) - -func GetModuleRootPath() string { - if moduleRootPath == "" { - moduleRootPath = os.Getenv("EXIF_MODULE_ROOT_PATH") - if moduleRootPath != "" { - return moduleRootPath - } - - currentWd, err := os.Getwd() - log.PanicIf(err) - - currentPath := currentWd - - visited := make([]string, 0) - - for { - tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") - - _, err := os.Stat(tryStampFilepath) - if err != nil && os.IsNotExist(err) != true { - log.Panic(err) - } else if err == nil { - break - } - - visited = append(visited, tryStampFilepath) - - currentPath = path.Dir(currentPath) - if currentPath == "/" { - log.Panicf("could not find module-root: %v", visited) - } - } - - moduleRootPath = currentPath - } - - return moduleRootPath -} - -func GetTestAssetsPath() string { - moduleRootPath := GetModuleRootPath() - assetsPath := path.Join(moduleRootPath, "assets") - - return assetsPath -} - -func getTestImageFilepath() string { - assetsPath := GetTestAssetsPath() - testImageFilepath := path.Join(assetsPath, "NDM_8901.jpg") - return testImageFilepath -} - -func getTestExifData() []byte { - if testExifData == nil { - assetsPath := GetTestAssetsPath() - filepath := path.Join(assetsPath, "NDM_8901.jpg.exif") - - var err error - - testExifData, err = ioutil.ReadFile(filepath) - log.PanicIf(err) - } - - return testExifData -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/type.go b/vendor/github.com/dsoprea/go-exif/v2/common/type.go deleted file mode 100644 index 86b38d044..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/type.go +++ /dev/null @@ -1,452 +0,0 @@ -package exifcommon - -import ( - "errors" - "fmt" - "reflect" - "strconv" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - typeLogger = log.NewLogger("exif.type") -) - -var ( - // ErrNotEnoughData is used when there isn't enough data to accommodate what - // we're trying to parse (sizeof(type) * unit_count). - ErrNotEnoughData = errors.New("not enough data for type") - - // ErrWrongType is used when we try to parse anything other than the - // current type. - ErrWrongType = errors.New("wrong type, can not parse") - - // ErrUnhandledUndefinedTypedTag is used when we try to parse a tag that's - // recorded as an "unknown" type but not a documented tag (therefore - // leaving us not knowning how to read it). - ErrUnhandledUndefinedTypedTag = errors.New("not a standard unknown-typed tag") -) - -// TagTypePrimitive is a type-alias that let's us easily lookup type properties. -type TagTypePrimitive uint16 - -const ( - // TypeByte describes an encoded list of bytes. - TypeByte TagTypePrimitive = 1 - - // TypeAscii describes an encoded list of characters that is terminated - // with a NUL in its encoded form. - TypeAscii TagTypePrimitive = 2 - - // TypeShort describes an encoded list of shorts. - TypeShort TagTypePrimitive = 3 - - // TypeLong describes an encoded list of longs. - TypeLong TagTypePrimitive = 4 - - // TypeRational describes an encoded list of rationals. - TypeRational TagTypePrimitive = 5 - - // TypeUndefined describes an encoded value that has a complex/non-clearcut - // interpretation. - TypeUndefined TagTypePrimitive = 7 - - // We've seen type-8, but have no documentation on it. - - // TypeSignedLong describes an encoded list of signed longs. - TypeSignedLong TagTypePrimitive = 9 - - // TypeSignedRational describes an encoded list of signed rationals. - TypeSignedRational TagTypePrimitive = 10 - - // TypeAsciiNoNul is just a pseudo-type, for our own purposes. - TypeAsciiNoNul TagTypePrimitive = 0xf0 -) - -// String returns the name of the type -func (typeType TagTypePrimitive) String() string { - return TypeNames[typeType] -} - -// Size returns the size of one atomic unit of the type. -func (tagType TagTypePrimitive) Size() int { - if tagType == TypeByte { - return 1 - } else if tagType == TypeAscii || tagType == TypeAsciiNoNul { - return 1 - } else if tagType == TypeShort { - return 2 - } else if tagType == TypeLong { - return 4 - } else if tagType == TypeRational { - return 8 - } else if tagType == TypeSignedLong { - return 4 - } else if tagType == TypeSignedRational { - return 8 - } else { - log.Panicf("can not determine tag-value size for type (%d): [%s]", tagType, TypeNames[tagType]) - - // Never called. - return 0 - } -} - -// IsValid returns true if tagType is a valid type. -func (tagType TagTypePrimitive) IsValid() bool { - - // TODO(dustin): Add test - - return tagType == TypeByte || - tagType == TypeAscii || - tagType == TypeAsciiNoNul || - tagType == TypeShort || - tagType == TypeLong || - tagType == TypeRational || - tagType == TypeSignedLong || - tagType == TypeSignedRational || - tagType == TypeUndefined -} - -var ( - // TODO(dustin): Rename TypeNames() to typeNames() and add getter. - TypeNames = map[TagTypePrimitive]string{ - TypeByte: "BYTE", - TypeAscii: "ASCII", - TypeShort: "SHORT", - TypeLong: "LONG", - TypeRational: "RATIONAL", - TypeUndefined: "UNDEFINED", - TypeSignedLong: "SLONG", - TypeSignedRational: "SRATIONAL", - - TypeAsciiNoNul: "_ASCII_NO_NUL", - } - - typeNamesR = map[string]TagTypePrimitive{} -) - -// Rational describes an unsigned rational value. -type Rational struct { - // Numerator is the numerator of the rational value. - Numerator uint32 - - // Denominator is the numerator of the rational value. - Denominator uint32 -} - -// SignedRational describes a signed rational value. -type SignedRational struct { - // Numerator is the numerator of the rational value. - Numerator int32 - - // Denominator is the numerator of the rational value. - Denominator int32 -} - -// Format returns a stringified value for the given encoding. Automatically -// parses. Automatically calculates count based on type size. This function -// also supports undefined-type values (the ones that we support, anyway) by -// way of the String() method that they all require. We can't be more specific -// because we're a base package and we can't refer to it. -func FormatFromType(value interface{}, justFirst bool) (phrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test - - switch t := value.(type) { - case []byte: - return DumpBytesToString(t), nil - case string: - return t, nil - case []uint16: - if len(t) == 0 { - return "", nil - } - - if justFirst == true { - var valueSuffix string - if len(t) > 1 { - valueSuffix = "..." - } - - return fmt.Sprintf("%v%s", t[0], valueSuffix), nil - } - - return fmt.Sprintf("%v", t), nil - case []uint32: - if len(t) == 0 { - return "", nil - } - - if justFirst == true { - var valueSuffix string - if len(t) > 1 { - valueSuffix = "..." - } - - return fmt.Sprintf("%v%s", t[0], valueSuffix), nil - } - - return fmt.Sprintf("%v", t), nil - case []Rational: - if len(t) == 0 { - return "", nil - } - - parts := make([]string, len(t)) - for i, r := range t { - parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) - - if justFirst == true { - break - } - } - - if justFirst == true { - var valueSuffix string - if len(t) > 1 { - valueSuffix = "..." - } - - return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil - } - - return fmt.Sprintf("%v", parts), nil - case []int32: - if len(t) == 0 { - return "", nil - } - - if justFirst == true { - var valueSuffix string - if len(t) > 1 { - valueSuffix = "..." - } - - return fmt.Sprintf("%v%s", t[0], valueSuffix), nil - } - - return fmt.Sprintf("%v", t), nil - case []SignedRational: - if len(t) == 0 { - return "", nil - } - - parts := make([]string, len(t)) - for i, r := range t { - parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) - - if justFirst == true { - break - } - } - - if justFirst == true { - var valueSuffix string - if len(t) > 1 { - valueSuffix = "..." - } - - return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil - } - - return fmt.Sprintf("%v", parts), nil - case fmt.Stringer: - // An undefined value that is documented (or that we otherwise support). - return t.String(), nil - default: - // Affects only "unknown" values, in general. - log.Panicf("type can not be formatted into string: %v", reflect.TypeOf(value).Name()) - - // Never called. - return "", nil - } -} - -// Format returns a stringified value for the given encoding. Automatically -// parses. Automatically calculates count based on type size. -func FormatFromBytes(rawBytes []byte, tagType TagTypePrimitive, justFirst bool, byteOrder binary.ByteOrder) (phrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test - - typeSize := tagType.Size() - - if len(rawBytes)%typeSize != 0 { - log.Panicf("byte-count (%d) does not align for [%s] type with a size of (%d) bytes", len(rawBytes), TypeNames[tagType], typeSize) - } - - // unitCount is the calculated unit-count. This should equal the original - // value from the tag (pre-resolution). - unitCount := uint32(len(rawBytes) / typeSize) - - // Truncate the items if it's not bytes or a string and we just want the first. - - var value interface{} - - switch tagType { - case TypeByte: - var err error - - value, err = parser.ParseBytes(rawBytes, unitCount) - log.PanicIf(err) - case TypeAscii: - var err error - - value, err = parser.ParseAscii(rawBytes, unitCount) - log.PanicIf(err) - case TypeAsciiNoNul: - var err error - - value, err = parser.ParseAsciiNoNul(rawBytes, unitCount) - log.PanicIf(err) - case TypeShort: - var err error - - value, err = parser.ParseShorts(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - case TypeLong: - var err error - - value, err = parser.ParseLongs(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - case TypeRational: - var err error - - value, err = parser.ParseRationals(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - case TypeSignedLong: - var err error - - value, err = parser.ParseSignedLongs(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - case TypeSignedRational: - var err error - - value, err = parser.ParseSignedRationals(rawBytes, unitCount, byteOrder) - log.PanicIf(err) - default: - // Affects only "unknown" values, in general. - log.Panicf("value of type [%s] can not be formatted into string", tagType.String()) - - // Never called. - return "", nil - } - - phrase, err = FormatFromType(value, justFirst) - log.PanicIf(err) - - return phrase, nil -} - -// TranslateStringToType converts user-provided strings to properly-typed -// values. If a string, returns a string. Else, assumes that it's a single -// number. If a list needs to be processed, it is the caller's responsibility to -// split it (according to whichever convention has been established). -func TranslateStringToType(tagType TagTypePrimitive, valueString string) (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if tagType == TypeUndefined { - // The caller should just call String() on the decoded type. - log.Panicf("undefined-type values are not supported") - } - - if tagType == TypeByte { - wide, err := strconv.ParseInt(valueString, 16, 8) - log.PanicIf(err) - - return byte(wide), nil - } else if tagType == TypeAscii || tagType == TypeAsciiNoNul { - // Whether or not we're putting an NUL on the end is only relevant for - // byte-level encoding. This function really just supports a user - // interface. - - return valueString, nil - } else if tagType == TypeShort { - n, err := strconv.ParseUint(valueString, 10, 16) - log.PanicIf(err) - - return uint16(n), nil - } else if tagType == TypeLong { - n, err := strconv.ParseUint(valueString, 10, 32) - log.PanicIf(err) - - return uint32(n), nil - } else if tagType == TypeRational { - parts := strings.SplitN(valueString, "/", 2) - - numerator, err := strconv.ParseUint(parts[0], 10, 32) - log.PanicIf(err) - - denominator, err := strconv.ParseUint(parts[1], 10, 32) - log.PanicIf(err) - - return Rational{ - Numerator: uint32(numerator), - Denominator: uint32(denominator), - }, nil - } else if tagType == TypeSignedLong { - n, err := strconv.ParseInt(valueString, 10, 32) - log.PanicIf(err) - - return int32(n), nil - } else if tagType == TypeSignedRational { - parts := strings.SplitN(valueString, "/", 2) - - numerator, err := strconv.ParseInt(parts[0], 10, 32) - log.PanicIf(err) - - denominator, err := strconv.ParseInt(parts[1], 10, 32) - log.PanicIf(err) - - return SignedRational{ - Numerator: int32(numerator), - Denominator: int32(denominator), - }, nil - } - - log.Panicf("from-string encoding for type not supported; this shouldn't happen: [%s]", tagType.String()) - return nil, nil -} - -// GetTypeByName returns the `TagTypePrimitive` for the given type name. -// Returns (0) if not valid. -func GetTypeByName(typeName string) (tagType TagTypePrimitive, found bool) { - tagType, found = typeNamesR[typeName] - return tagType, found -} - -// BasicTag describes a single tag for any purpose. -type BasicTag struct { - // FqIfdPath is the fully-qualified IFD-path. - FqIfdPath string - - // IfdPath is the unindexed IFD-path. - IfdPath string - - // TagId is the tag-ID. - TagId uint16 -} - -func init() { - for typeId, typeName := range TypeNames { - typeNamesR[typeName] = typeId - } -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/utility.go b/vendor/github.com/dsoprea/go-exif/v2/common/utility.go deleted file mode 100644 index 65165bf02..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/utility.go +++ /dev/null @@ -1,79 +0,0 @@ -package exifcommon - -import ( - "bytes" - "fmt" - "time" - - "github.com/dsoprea/go-logging" -) - -// DumpBytes prints a list of hex-encoded bytes. -func DumpBytes(data []byte) { - fmt.Printf("DUMP: ") - for _, x := range data { - fmt.Printf("%02x ", x) - } - - fmt.Printf("\n") -} - -// DumpBytesClause prints a list like DumpBytes(), but encapsulated in -// "[]byte { ... }". -func DumpBytesClause(data []byte) { - fmt.Printf("DUMP: ") - - fmt.Printf("[]byte { ") - - for i, x := range data { - fmt.Printf("0x%02x", x) - - if i < len(data)-1 { - fmt.Printf(", ") - } - } - - fmt.Printf(" }\n") -} - -// DumpBytesToString returns a stringified list of hex-encoded bytes. -func DumpBytesToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteRune(' ') - log.PanicIf(err) - } - } - - return b.String() -} - -// DumpBytesClauseToString returns a comma-separated list of hex-encoded bytes. -func DumpBytesClauseToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteString(", ") - log.PanicIf(err) - } - } - - return b.String() -} - -// ExifFullTimestampString produces a string like "2018:11:30 13:01:49" from a -// `time.Time` struct. It will attempt to convert to UTC first. -func ExifFullTimestampString(t time.Time) (fullTimestampPhrase string) { - t = t.UTC() - - return fmt.Sprintf("%04d:%02d:%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/value_context.go b/vendor/github.com/dsoprea/go-exif/v2/common/value_context.go deleted file mode 100644 index feb078ccf..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/value_context.go +++ /dev/null @@ -1,412 +0,0 @@ -package exifcommon - -import ( - "errors" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - parser *Parser -) - -var ( - // ErrNotFarValue indicates that an offset-based lookup was attempted for a - // non-offset-based (embedded) value. - ErrNotFarValue = errors.New("not a far value") -) - -// ValueContext embeds all of the parameters required to find and extract the -// actual tag value. -type ValueContext struct { - unitCount uint32 - valueOffset uint32 - rawValueOffset []byte - addressableData []byte - - tagType TagTypePrimitive - byteOrder binary.ByteOrder - - // undefinedValueTagType is the effective type to use if this is an - // "undefined" value. - undefinedValueTagType TagTypePrimitive - - ifdPath string - tagId uint16 -} - -// TODO(dustin): We can update newValueContext() to derive `valueOffset` itself (from `rawValueOffset`). - -// NewValueContext returns a new ValueContext struct. -func NewValueContext(ifdPath string, tagId uint16, unitCount, valueOffset uint32, rawValueOffset, addressableData []byte, tagType TagTypePrimitive, byteOrder binary.ByteOrder) *ValueContext { - return &ValueContext{ - unitCount: unitCount, - valueOffset: valueOffset, - rawValueOffset: rawValueOffset, - addressableData: addressableData, - - tagType: tagType, - byteOrder: byteOrder, - - ifdPath: ifdPath, - tagId: tagId, - } -} - -// SetUndefinedValueType sets the effective type if this is an unknown-type tag. -func (vc *ValueContext) SetUndefinedValueType(tagType TagTypePrimitive) { - if vc.tagType != TypeUndefined { - log.Panicf("can not set effective type for unknown-type tag because this is *not* an unknown-type tag") - } - - vc.undefinedValueTagType = tagType -} - -// UnitCount returns the embedded unit-count. -func (vc *ValueContext) UnitCount() uint32 { - return vc.unitCount -} - -// ValueOffset returns the value-offset decoded as a `uint32`. -func (vc *ValueContext) ValueOffset() uint32 { - return vc.valueOffset -} - -// RawValueOffset returns the uninterpreted value-offset. This is used for -// embedded values (values small enough to fit within the offset bytes rather -// than needing to be stored elsewhere and referred to by an actual offset). -func (vc *ValueContext) RawValueOffset() []byte { - return vc.rawValueOffset -} - -// AddressableData returns the block of data that we can dereference into. -func (vc *ValueContext) AddressableData() []byte { - return vc.addressableData -} - -// ByteOrder returns the byte-order of numbers. -func (vc *ValueContext) ByteOrder() binary.ByteOrder { - return vc.byteOrder -} - -// IfdPath returns the path of the IFD containing this tag. -func (vc *ValueContext) IfdPath() string { - return vc.ifdPath -} - -// TagId returns the ID of the tag that we represent. -func (vc *ValueContext) TagId() uint16 { - return vc.tagId -} - -// isEmbedded returns whether the value is embedded or a reference. This can't -// be precalculated since the size is not defined for all types (namely the -// "undefined" types). -func (vc *ValueContext) isEmbedded() bool { - tagType := vc.effectiveValueType() - - return (tagType.Size() * int(vc.unitCount)) <= 4 -} - -// SizeInBytes returns the number of bytes that this value requires. The -// underlying call will panic if the type is UNDEFINED. It is the -// responsibility of the caller to preemptively check that. -func (vc *ValueContext) SizeInBytes() int { - tagType := vc.effectiveValueType() - - return tagType.Size() * int(vc.unitCount) -} - -// effectiveValueType returns the effective type of the unknown-type tag or, if -// not unknown, the actual type. -func (vc *ValueContext) effectiveValueType() (tagType TagTypePrimitive) { - if vc.tagType == TypeUndefined { - tagType = vc.undefinedValueTagType - - if tagType == 0 { - log.Panicf("undefined-value type not set") - } - } else { - tagType = vc.tagType - } - - return tagType -} - -// readRawEncoded returns the encoded bytes for the value that we represent. -func (vc *ValueContext) readRawEncoded() (rawBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tagType := vc.effectiveValueType() - - unitSizeRaw := uint32(tagType.Size()) - - if vc.isEmbedded() == true { - byteLength := unitSizeRaw * vc.unitCount - return vc.rawValueOffset[:byteLength], nil - } - - return vc.addressableData[vc.valueOffset : vc.valueOffset+vc.unitCount*unitSizeRaw], nil -} - -// GetFarOffset returns the offset if the value is not embedded [within the -// pointer itself] or an error if an embedded value. -func (vc *ValueContext) GetFarOffset() (offset uint32, err error) { - if vc.isEmbedded() == true { - return 0, ErrNotFarValue - } - - return vc.valueOffset, nil -} - -// ReadRawEncoded returns the encoded bytes for the value that we represent. -func (vc *ValueContext) ReadRawEncoded() (rawBytes []byte, err error) { - - // TODO(dustin): Remove this method and rename readRawEncoded in its place. - - return vc.readRawEncoded() -} - -// Format returns a string representation for the value. -// -// Where the type is not ASCII, `justFirst` indicates whether to just stringify -// the first item in the slice (or return an empty string if the slice is -// empty). -// -// Since this method lacks the information to process undefined-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (vc *ValueContext) Format() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawBytes, err := vc.readRawEncoded() - log.PanicIf(err) - - phrase, err := FormatFromBytes(rawBytes, vc.effectiveValueType(), false, vc.byteOrder) - log.PanicIf(err) - - return phrase, nil -} - -// FormatFirst is similar to `Format` but only gets and stringifies the first -// item. -func (vc *ValueContext) FormatFirst() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawBytes, err := vc.readRawEncoded() - log.PanicIf(err) - - phrase, err := FormatFromBytes(rawBytes, vc.tagType, true, vc.byteOrder) - log.PanicIf(err) - - return phrase, nil -} - -// ReadBytes parses the encoded byte-array from the value-context. -func (vc *ValueContext) ReadBytes() (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseBytes(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -// ReadAscii parses the encoded NUL-terminated ASCII string from the value- -// context. -func (vc *ValueContext) ReadAscii() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseAscii(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -// ReadAsciiNoNul parses the non-NUL-terminated encoded ASCII string from the -// value-context. -func (vc *ValueContext) ReadAsciiNoNul() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseAsciiNoNul(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -// ReadShorts parses the list of encoded shorts from the value-context. -func (vc *ValueContext) ReadShorts() (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseShorts(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// ReadLongs parses the list of encoded, unsigned longs from the value-context. -func (vc *ValueContext) ReadLongs() (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseLongs(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// ReadRationals parses the list of encoded, unsigned rationals from the value- -// context. -func (vc *ValueContext) ReadRationals() (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseRationals(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// ReadSignedLongs parses the list of encoded, signed longs from the value-context. -func (vc *ValueContext) ReadSignedLongs() (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseSignedLongs(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// ReadSignedRationals parses the list of encoded, signed rationals from the -// value-context. -func (vc *ValueContext) ReadSignedRationals() (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseSignedRationals(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// Values knows how to resolve the given value. This value is always a list -// (undefined-values aside), so we're named accordingly. -// -// Since this method lacks the information to process unknown-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (vc *ValueContext) Values() (values interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if vc.tagType == TypeByte { - values, err = vc.ReadBytes() - log.PanicIf(err) - } else if vc.tagType == TypeAscii { - values, err = vc.ReadAscii() - log.PanicIf(err) - } else if vc.tagType == TypeAsciiNoNul { - values, err = vc.ReadAsciiNoNul() - log.PanicIf(err) - } else if vc.tagType == TypeShort { - values, err = vc.ReadShorts() - log.PanicIf(err) - } else if vc.tagType == TypeLong { - values, err = vc.ReadLongs() - log.PanicIf(err) - } else if vc.tagType == TypeRational { - values, err = vc.ReadRationals() - log.PanicIf(err) - } else if vc.tagType == TypeSignedLong { - values, err = vc.ReadSignedLongs() - log.PanicIf(err) - } else if vc.tagType == TypeSignedRational { - values, err = vc.ReadSignedRationals() - log.PanicIf(err) - } else if vc.tagType == TypeUndefined { - log.Panicf("will not parse undefined-type value") - - // Never called. - return nil, nil - } else { - log.Panicf("value of type [%s] is unparseable", vc.tagType) - // Never called. - return nil, nil - } - - return values, nil -} - -func init() { - parser = new(Parser) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/common/value_encoder.go b/vendor/github.com/dsoprea/go-exif/v2/common/value_encoder.go deleted file mode 100644 index 52e0eacfd..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/common/value_encoder.go +++ /dev/null @@ -1,229 +0,0 @@ -package exifcommon - -import ( - "bytes" - "reflect" - "time" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - typeEncodeLogger = log.NewLogger("exif.type_encode") -) - -// EncodedData encapsulates the compound output of an encoding operation. -type EncodedData struct { - Type TagTypePrimitive - Encoded []byte - - // TODO(dustin): Is this really necessary? We might have this just to correlate to the incoming stream format (raw bytes and a unit-count both for incoming and outgoing). - UnitCount uint32 -} - -// ValueEncoder knows how to encode values of every type to bytes. -type ValueEncoder struct { - byteOrder binary.ByteOrder -} - -// NewValueEncoder returns a new ValueEncoder. -func NewValueEncoder(byteOrder binary.ByteOrder) *ValueEncoder { - return &ValueEncoder{ - byteOrder: byteOrder, - } -} - -func (ve *ValueEncoder) encodeBytes(value []uint8) (ed EncodedData, err error) { - ed.Type = TypeByte - ed.Encoded = []byte(value) - ed.UnitCount = uint32(len(value)) - - return ed, nil -} - -func (ve *ValueEncoder) encodeAscii(value string) (ed EncodedData, err error) { - ed.Type = TypeAscii - - ed.Encoded = []byte(value) - ed.Encoded = append(ed.Encoded, 0) - - ed.UnitCount = uint32(len(ed.Encoded)) - - return ed, nil -} - -// encodeAsciiNoNul returns a string encoded as a byte-string without a trailing -// NUL byte. -// -// Note that: -// -// 1. This type can not be automatically encoded using `Encode()`. The default -// mode is to encode *with* a trailing NUL byte using `encodeAscii`. Only -// certain undefined-type tags using an unterminated ASCII string and these -// are exceptional in nature. -// -// 2. The presence of this method allows us to completely test the complimentary -// no-nul parser. -// -func (ve *ValueEncoder) encodeAsciiNoNul(value string) (ed EncodedData, err error) { - ed.Type = TypeAsciiNoNul - ed.Encoded = []byte(value) - ed.UnitCount = uint32(len(ed.Encoded)) - - return ed, nil -} - -func (ve *ValueEncoder) encodeShorts(value []uint16) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*2) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint16(ed.Encoded[i*2:(i+1)*2], value[i]) - } - - ed.Type = TypeShort - - return ed, nil -} - -func (ve *ValueEncoder) encodeLongs(value []uint32) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*4) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint32(ed.Encoded[i*4:(i+1)*4], value[i]) - } - - ed.Type = TypeLong - - return ed, nil -} - -func (ve *ValueEncoder) encodeRationals(value []Rational) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - ed.Encoded = make([]byte, ed.UnitCount*8) - - for i := uint32(0); i < ed.UnitCount; i++ { - ve.byteOrder.PutUint32(ed.Encoded[i*8+0:i*8+4], value[i].Numerator) - ve.byteOrder.PutUint32(ed.Encoded[i*8+4:i*8+8], value[i].Denominator) - } - - ed.Type = TypeRational - - return ed, nil -} - -func (ve *ValueEncoder) encodeSignedLongs(value []int32) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - - b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) - - for i := uint32(0); i < ed.UnitCount; i++ { - err := binary.Write(b, ve.byteOrder, value[i]) - log.PanicIf(err) - } - - ed.Type = TypeSignedLong - ed.Encoded = b.Bytes() - - return ed, nil -} - -func (ve *ValueEncoder) encodeSignedRationals(value []SignedRational) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ed.UnitCount = uint32(len(value)) - - b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) - - for i := uint32(0); i < ed.UnitCount; i++ { - err := binary.Write(b, ve.byteOrder, value[i].Numerator) - log.PanicIf(err) - - err = binary.Write(b, ve.byteOrder, value[i].Denominator) - log.PanicIf(err) - } - - ed.Type = TypeSignedRational - ed.Encoded = b.Bytes() - - return ed, nil -} - -// Encode returns bytes for the given value, infering type from the actual -// value. This does not support `TypeAsciiNoNull` (all strings are encoded as -// `TypeAscii`). -func (ve *ValueEncoder) Encode(value interface{}) (ed EncodedData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - switch value.(type) { - case []byte: - ed, err = ve.encodeBytes(value.([]byte)) - log.PanicIf(err) - case string: - ed, err = ve.encodeAscii(value.(string)) - log.PanicIf(err) - case []uint16: - ed, err = ve.encodeShorts(value.([]uint16)) - log.PanicIf(err) - case []uint32: - ed, err = ve.encodeLongs(value.([]uint32)) - log.PanicIf(err) - case []Rational: - ed, err = ve.encodeRationals(value.([]Rational)) - log.PanicIf(err) - case []int32: - ed, err = ve.encodeSignedLongs(value.([]int32)) - log.PanicIf(err) - case []SignedRational: - ed, err = ve.encodeSignedRationals(value.([]SignedRational)) - log.PanicIf(err) - case time.Time: - // For convenience, if the user doesn't want to deal with translation - // semantics with timestamps. - - t := value.(time.Time) - s := ExifFullTimestampString(t) - - ed, err = ve.encodeAscii(s) - log.PanicIf(err) - default: - log.Panicf("value not encodable: [%s] [%v]", reflect.TypeOf(value), value) - } - - return ed, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/error.go b/vendor/github.com/dsoprea/go-exif/v2/error.go deleted file mode 100644 index 2f00b08a4..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/error.go +++ /dev/null @@ -1,14 +0,0 @@ -package exif - -import ( - "errors" -) - -var ( - // ErrTagNotFound indicates that the tag was not found. - ErrTagNotFound = errors.New("tag not found") - - // ErrTagNotKnown indicates that the tag is not registered with us as a - // known tag. - ErrTagNotKnown = errors.New("tag is not known") -) diff --git a/vendor/github.com/dsoprea/go-exif/v2/exif.go b/vendor/github.com/dsoprea/go-exif/v2/exif.go deleted file mode 100644 index 20b723769..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/exif.go +++ /dev/null @@ -1,258 +0,0 @@ -package exif - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "os" - - "encoding/binary" - "io/ioutil" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -const ( - // ExifAddressableAreaStart is the absolute offset in the file that all - // offsets are relative to. - ExifAddressableAreaStart = uint32(0x0) - - // ExifDefaultFirstIfdOffset is essentially the number of bytes in addition - // to `ExifAddressableAreaStart` that you have to move in order to escape - // the rest of the header and get to the earliest point where we can put - // stuff (which has to be the first IFD). This is the size of the header - // sequence containing the two-character byte-order, two-character fixed- - // bytes, and the four bytes describing the first-IFD offset. - ExifDefaultFirstIfdOffset = uint32(2 + 2 + 4) -) - -const ( - // ExifSignatureLength is the number of bytes in the EXIF signature (which - // customarily includes the first IFD offset). - ExifSignatureLength = 8 -) - -var ( - exifLogger = log.NewLogger("exif.exif") - - ExifBigEndianSignature = [4]byte{'M', 'M', 0x00, 0x2a} - ExifLittleEndianSignature = [4]byte{'I', 'I', 0x2a, 0x00} -) - -var ( - ErrNoExif = errors.New("no exif data") - ErrExifHeaderError = errors.New("exif header error") -) - -// SearchAndExtractExif searches for an EXIF blob in the byte-slice. -func SearchAndExtractExif(data []byte) (rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - b := bytes.NewBuffer(data) - - rawExif, err = SearchAndExtractExifWithReader(b) - if err != nil { - if err == ErrNoExif { - return nil, err - } - - log.Panic(err) - } - - return rawExif, nil -} - -// SearchAndExtractExifWithReader searches for an EXIF blob using an -// `io.Reader`. We can't know how much long the EXIF data is without parsing it, -// so this will likely grab up a lot of the image-data, too. -func SearchAndExtractExifWithReader(r io.Reader) (rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Search for the beginning of the EXIF information. The EXIF is near the - // beginning of most JPEGs, so this likely doesn't have a high cost (at - // least, again, with JPEGs). - - br := bufio.NewReader(r) - discarded := 0 - - for { - window, err := br.Peek(ExifSignatureLength) - if err != nil { - if err == io.EOF { - return nil, ErrNoExif - } - - log.Panic(err) - } - - _, err = ParseExifHeader(window) - if err != nil { - if log.Is(err, ErrNoExif) == true { - // No EXIF. Move forward by one byte. - - _, err := br.Discard(1) - log.PanicIf(err) - - discarded++ - - continue - } - - // Some other error. - log.Panic(err) - } - - break - } - - exifLogger.Debugf(nil, "Found EXIF blob (%d) bytes from initial position.", discarded) - - rawExif, err = ioutil.ReadAll(br) - log.PanicIf(err) - - return rawExif, nil -} - -// SearchFileAndExtractExif returns a slice from the beginning of the EXIF data -// to the end of the file (it's not practical to try and calculate where the -// data actually ends). -func SearchFileAndExtractExif(filepath string) (rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Open the file. - - f, err := os.Open(filepath) - log.PanicIf(err) - - defer f.Close() - - rawExif, err = SearchAndExtractExifWithReader(f) - log.PanicIf(err) - - return rawExif, nil -} - -type ExifHeader struct { - ByteOrder binary.ByteOrder - FirstIfdOffset uint32 -} - -func (eh ExifHeader) String() string { - return fmt.Sprintf("ExifHeader", eh.ByteOrder, eh.FirstIfdOffset) -} - -// ParseExifHeader parses the bytes at the very top of the header. -// -// This will panic with ErrNoExif on any data errors so that we can double as -// an EXIF-detection routine. -func ParseExifHeader(data []byte) (eh ExifHeader, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Good reference: - // - // CIPA DC-008-2016; JEITA CP-3451D - // -> http://www.cipa.jp/std/documents/e/DC-008-Translation-2016-E.pdf - - if len(data) < ExifSignatureLength { - exifLogger.Warningf(nil, "Not enough data for EXIF header: (%d)", len(data)) - return eh, ErrNoExif - } - - if bytes.Equal(data[:4], ExifBigEndianSignature[:]) == true { - eh.ByteOrder = binary.BigEndian - } else if bytes.Equal(data[:4], ExifLittleEndianSignature[:]) == true { - eh.ByteOrder = binary.LittleEndian - } else { - return eh, ErrNoExif - } - - eh.FirstIfdOffset = eh.ByteOrder.Uint32(data[4:8]) - - return eh, nil -} - -// Visit recursively invokes a callback for every tag. -func Visit(rootIfdIdentity *exifcommon.IfdIdentity, ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, exifData []byte, visitor TagVisitorFn) (eh ExifHeader, furthestOffset uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - eh, err = ParseExifHeader(exifData) - log.PanicIf(err) - - ie := NewIfdEnumerate(ifdMapping, tagIndex, exifData, eh.ByteOrder) - - _, err = ie.Scan(rootIfdIdentity, eh.FirstIfdOffset, visitor) - log.PanicIf(err) - - furthestOffset = ie.FurthestOffset() - - return eh, furthestOffset, nil -} - -// Collect recursively builds a static structure of all IFDs and tags. -func Collect(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, exifData []byte) (eh ExifHeader, index IfdIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - eh, err = ParseExifHeader(exifData) - log.PanicIf(err) - - ie := NewIfdEnumerate(ifdMapping, tagIndex, exifData, eh.ByteOrder) - - index, err = ie.Collect(eh.FirstIfdOffset) - log.PanicIf(err) - - return eh, index, nil -} - -// BuildExifHeader constructs the bytes that go at the front of the stream. -func BuildExifHeader(byteOrder binary.ByteOrder, firstIfdOffset uint32) (headerBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - b := new(bytes.Buffer) - - var signatureBytes []byte - if byteOrder == binary.BigEndian { - signatureBytes = ExifBigEndianSignature[:] - } else { - signatureBytes = ExifLittleEndianSignature[:] - } - - _, err = b.Write(signatureBytes) - log.PanicIf(err) - - err = binary.Write(b, byteOrder, firstIfdOffset) - log.PanicIf(err) - - return b.Bytes(), nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/gps.go b/vendor/github.com/dsoprea/go-exif/v2/gps.go deleted file mode 100644 index d44ede1ad..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/gps.go +++ /dev/null @@ -1,117 +0,0 @@ -package exif - -import ( - "errors" - "fmt" - "time" - - "github.com/dsoprea/go-logging" - "github.com/golang/geo/s2" - - "github.com/dsoprea/go-exif/v2/common" -) - -var ( - // ErrGpsCoordinatesNotValid means that some part of the geographic data was - // unparseable. - ErrGpsCoordinatesNotValid = errors.New("GPS coordinates not valid") -) - -// GpsDegrees is a high-level struct representing geographic data. -type GpsDegrees struct { - // Orientation describes the N/E/S/W direction that this position is - // relative to. - Orientation byte - - // Degrees is a simple float representing the underlying rational degrees - // amount. - Degrees float64 - - // Minutes is a simple float representing the underlying rational minutes - // amount. - Minutes float64 - - // Seconds is a simple float representing the underlying ration seconds - // amount. - Seconds float64 -} - -// NewGpsDegreesFromRationals returns a GpsDegrees struct given the EXIF-encoded -// information. The refValue is the N/E/S/W direction that this position is -// relative to. -func NewGpsDegreesFromRationals(refValue string, rawCoordinate []exifcommon.Rational) (gd GpsDegrees, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(rawCoordinate) != 3 { - log.Panicf("new GpsDegrees struct requires a raw-coordinate with exactly three rationals") - } - - gd = GpsDegrees{ - Orientation: refValue[0], - Degrees: float64(rawCoordinate[0].Numerator) / float64(rawCoordinate[0].Denominator), - Minutes: float64(rawCoordinate[1].Numerator) / float64(rawCoordinate[1].Denominator), - Seconds: float64(rawCoordinate[2].Numerator) / float64(rawCoordinate[2].Denominator), - } - - return gd, nil -} - -// String provides returns a descriptive string. -func (d GpsDegrees) String() string { - return fmt.Sprintf("Degrees", string([]byte{d.Orientation}), d.Degrees, d.Minutes, d.Seconds) -} - -// Decimal calculates and returns the simplified float representation of the -// component degrees. -func (d GpsDegrees) Decimal() float64 { - decimal := float64(d.Degrees) + float64(d.Minutes)/60.0 + float64(d.Seconds)/3600.0 - - if d.Orientation == 'S' || d.Orientation == 'W' { - return -decimal - } - - return decimal -} - -// Raw returns a Rational struct that can be used to *write* coordinates. In -// practice, the denominator are typically (1) in the original EXIF data, and, -// that being the case, this will best preserve precision. -func (d GpsDegrees) Raw() []exifcommon.Rational { - return []exifcommon.Rational{ - {Numerator: uint32(d.Degrees), Denominator: 1}, - {Numerator: uint32(d.Minutes), Denominator: 1}, - {Numerator: uint32(d.Seconds), Denominator: 1}, - } -} - -// GpsInfo encapsulates all of the geographic information in one place. -type GpsInfo struct { - Latitude, Longitude GpsDegrees - Altitude int - Timestamp time.Time -} - -// String returns a descriptive string. -func (gi *GpsInfo) String() string { - return fmt.Sprintf("GpsInfo", - gi.Latitude.Decimal(), gi.Longitude.Decimal(), gi.Altitude, gi.Timestamp) -} - -// S2CellId returns the cell-ID of the geographic location on the earth. -func (gi *GpsInfo) S2CellId() s2.CellID { - latitude := gi.Latitude.Decimal() - longitude := gi.Longitude.Decimal() - - ll := s2.LatLngFromDegrees(latitude, longitude) - cellId := s2.CellIDFromLatLng(ll) - - if cellId.IsValid() == false { - panic(ErrGpsCoordinatesNotValid) - } - - return cellId -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/ifd.go b/vendor/github.com/dsoprea/go-exif/v2/ifd.go deleted file mode 100644 index 80872e624..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/ifd.go +++ /dev/null @@ -1,34 +0,0 @@ -package exif - -import ( - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -// TODO(dustin): This file now exists for backwards-compatibility only. - -// NewIfdMapping returns a new IfdMapping struct. -func NewIfdMapping() (ifdMapping *exifcommon.IfdMapping) { - return exifcommon.NewIfdMapping() -} - -// NewIfdMappingWithStandard retruns a new IfdMapping struct preloaded with the -// standard IFDs. -func NewIfdMappingWithStandard() (ifdMapping *exifcommon.IfdMapping) { - return exifcommon.NewIfdMappingWithStandard() -} - -// LoadStandardIfds loads the standard IFDs into the mapping. -func LoadStandardIfds(im *exifcommon.IfdMapping) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = exifcommon.LoadStandardIfds(im) - log.PanicIf(err) - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/ifd_builder.go b/vendor/github.com/dsoprea/go-exif/v2/ifd_builder.go deleted file mode 100644 index 64a09299c..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/ifd_builder.go +++ /dev/null @@ -1,1199 +0,0 @@ -package exif - -// NOTES: -// -// The thumbnail offset and length tags shouldn't be set directly. Use the -// (*IfdBuilder).SetThumbnail() method instead. - -import ( - "errors" - "fmt" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" - "github.com/dsoprea/go-exif/v2/undefined" -) - -var ( - ifdBuilderLogger = log.NewLogger("exif.ifd_builder") -) - -var ( - ErrTagEntryNotFound = errors.New("tag entry not found") - ErrChildIbNotFound = errors.New("child IB not found") -) - -type IfdBuilderTagValue struct { - valueBytes []byte - ib *IfdBuilder -} - -func (ibtv IfdBuilderTagValue) String() string { - if ibtv.IsBytes() == true { - var valuePhrase string - if len(ibtv.valueBytes) <= 8 { - valuePhrase = fmt.Sprintf("%v", ibtv.valueBytes) - } else { - valuePhrase = fmt.Sprintf("%v...", ibtv.valueBytes[:8]) - } - - return fmt.Sprintf("IfdBuilderTagValue", valuePhrase, len(ibtv.valueBytes)) - } else if ibtv.IsIb() == true { - return fmt.Sprintf("IfdBuilderTagValue", ibtv.ib) - } else { - log.Panicf("IBTV state undefined") - return "" - } -} - -func NewIfdBuilderTagValueFromBytes(valueBytes []byte) *IfdBuilderTagValue { - return &IfdBuilderTagValue{ - valueBytes: valueBytes, - } -} - -func NewIfdBuilderTagValueFromIfdBuilder(ib *IfdBuilder) *IfdBuilderTagValue { - return &IfdBuilderTagValue{ - ib: ib, - } -} - -// IsBytes returns true if the bytes are populated. This is always the case -// when we're loaded from a tag in an existing IFD. -func (ibtv IfdBuilderTagValue) IsBytes() bool { - return ibtv.valueBytes != nil -} - -func (ibtv IfdBuilderTagValue) Bytes() []byte { - if ibtv.IsBytes() == false { - log.Panicf("this tag is not a byte-slice value") - } else if ibtv.IsIb() == true { - log.Panicf("this tag is an IFD-builder value not a byte-slice") - } - - return ibtv.valueBytes -} - -func (ibtv IfdBuilderTagValue) IsIb() bool { - return ibtv.ib != nil -} - -func (ibtv IfdBuilderTagValue) Ib() *IfdBuilder { - if ibtv.IsIb() == false { - log.Panicf("this tag is not an IFD-builder value") - } else if ibtv.IsBytes() == true { - log.Panicf("this tag is a byte-slice, not a IFD-builder") - } - - return ibtv.ib -} - -type BuilderTag struct { - // ifdPath is the path of the IFD that hosts this tag. - ifdPath string - - tagId uint16 - typeId exifcommon.TagTypePrimitive - - // value is either a value that can be encoded, an IfdBuilder instance (for - // child IFDs), or an IfdTagEntry instance representing an existing, - // previously-stored tag. - value *IfdBuilderTagValue - - // byteOrder is the byte order. It's chiefly/originally here to support - // printing the value. - byteOrder binary.ByteOrder -} - -func NewBuilderTag(ifdPath string, tagId uint16, typeId exifcommon.TagTypePrimitive, value *IfdBuilderTagValue, byteOrder binary.ByteOrder) *BuilderTag { - return &BuilderTag{ - ifdPath: ifdPath, - tagId: tagId, - typeId: typeId, - value: value, - byteOrder: byteOrder, - } -} - -func NewChildIfdBuilderTag(ifdPath string, tagId uint16, value *IfdBuilderTagValue) *BuilderTag { - return &BuilderTag{ - ifdPath: ifdPath, - tagId: tagId, - typeId: exifcommon.TypeLong, - value: value, - } -} - -func (bt *BuilderTag) Value() (value *IfdBuilderTagValue) { - return bt.value -} - -func (bt *BuilderTag) String() string { - var valueString string - - if bt.value.IsBytes() == true { - var err error - - valueString, err = exifcommon.FormatFromBytes(bt.value.Bytes(), bt.typeId, false, bt.byteOrder) - log.PanicIf(err) - } else { - valueString = fmt.Sprintf("%v", bt.value) - } - - return fmt.Sprintf("BuilderTag", bt.ifdPath, bt.tagId, bt.typeId.String(), valueString) -} - -func (bt *BuilderTag) SetValue(byteOrder binary.ByteOrder, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - var ed exifcommon.EncodedData - if bt.typeId == exifcommon.TypeUndefined { - encodeable := value.(exifundefined.EncodeableValue) - - encoded, unitCount, err := exifundefined.Encode(encodeable, byteOrder) - log.PanicIf(err) - - ed = exifcommon.EncodedData{ - Type: exifcommon.TypeUndefined, - Encoded: encoded, - UnitCount: unitCount, - } - } else { - ve := exifcommon.NewValueEncoder(byteOrder) - - var err error - - ed, err = ve.Encode(value) - log.PanicIf(err) - } - - bt.value = NewIfdBuilderTagValueFromBytes(ed.Encoded) - - return nil -} - -// NewStandardBuilderTag constructs a `BuilderTag` instance. The type is looked -// up. `ii` is the type of IFD that owns this tag. -func NewStandardBuilderTag(ifdPath string, it *IndexedTag, byteOrder binary.ByteOrder, value interface{}) *BuilderTag { - // If there is more than one supported type, we'll go with the larger to - // encode with. It'll use the same amount of fixed-space, and we'll - // eliminate unnecessary overflows/issues. - tagType := it.GetEncodingType(value) - - var rawBytes []byte - if it.DoesSupportType(exifcommon.TypeUndefined) == true { - encodeable := value.(exifundefined.EncodeableValue) - - var err error - - rawBytes, _, err = exifundefined.Encode(encodeable, byteOrder) - log.PanicIf(err) - } else { - ve := exifcommon.NewValueEncoder(byteOrder) - - ed, err := ve.Encode(value) - log.PanicIf(err) - - rawBytes = ed.Encoded - } - - tagValue := NewIfdBuilderTagValueFromBytes(rawBytes) - - return NewBuilderTag( - ifdPath, - it.Id, - tagType, - tagValue, - byteOrder) -} - -type IfdBuilder struct { - ifdIdentity *exifcommon.IfdIdentity - - byteOrder binary.ByteOrder - - // Includes both normal tags and IFD tags (which point to child IFDs). - // TODO(dustin): Keep a separate list of children like with `Ifd`. - // TODO(dustin): Either rename this or `Entries` in `Ifd` to be the same thing. - tags []*BuilderTag - - // existingOffset will be the offset that this IFD is currently found at if - // it represents an IFD that has previously been stored (or 0 if not). - existingOffset uint32 - - // nextIb represents the next link if we're chaining to another. - nextIb *IfdBuilder - - // thumbnailData is populated with thumbnail data if there was thumbnail - // data. Otherwise, it's nil. - thumbnailData []byte - - ifdMapping *exifcommon.IfdMapping - tagIndex *TagIndex -} - -func NewIfdBuilder(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder) (ib *IfdBuilder) { - ib = &IfdBuilder{ - ifdIdentity: ii, - - byteOrder: byteOrder, - tags: make([]*BuilderTag, 0), - - ifdMapping: ifdMapping, - tagIndex: tagIndex, - } - - return ib -} - -// NewIfdBuilderWithExistingIfd creates a new IB using the same header type -// information as the given IFD. -func NewIfdBuilderWithExistingIfd(ifd *Ifd) (ib *IfdBuilder) { - ib = &IfdBuilder{ - ifdIdentity: ifd.IfdIdentity(), - - byteOrder: ifd.ByteOrder, - existingOffset: ifd.Offset, - ifdMapping: ifd.ifdMapping, - tagIndex: ifd.tagIndex, - } - - return ib -} - -// NewIfdBuilderFromExistingChain creates a chain of IB instances from an -// IFD chain generated from real data. -func NewIfdBuilderFromExistingChain(rootIfd *Ifd) (firstIb *IfdBuilder) { - var lastIb *IfdBuilder - i := 0 - for thisExistingIfd := rootIfd; thisExistingIfd != nil; thisExistingIfd = thisExistingIfd.NextIfd { - newIb := NewIfdBuilder( - rootIfd.ifdMapping, - rootIfd.tagIndex, - rootIfd.ifdIdentity, - thisExistingIfd.ByteOrder) - - if firstIb == nil { - firstIb = newIb - } else { - lastIb.SetNextIb(newIb) - } - - err := newIb.AddTagsFromExisting(thisExistingIfd, nil, nil) - log.PanicIf(err) - - lastIb = newIb - i++ - } - - return firstIb -} - -func (ib *IfdBuilder) IfdIdentity() *exifcommon.IfdIdentity { - return ib.ifdIdentity -} - -func (ib *IfdBuilder) NextIb() (nextIb *IfdBuilder, err error) { - return ib.nextIb, nil -} - -func (ib *IfdBuilder) ChildWithTagId(childIfdTagId uint16) (childIb *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for _, bt := range ib.tags { - if bt.value.IsIb() == false { - continue - } - - childIbThis := bt.value.Ib() - - if childIbThis.IfdIdentity().TagId() == childIfdTagId { - return childIbThis, nil - } - } - - log.Panic(ErrChildIbNotFound) - - // Never reached. - return nil, nil -} - -func getOrCreateIbFromRootIbInner(rootIb *IfdBuilder, parentIb *IfdBuilder, currentLineage []exifcommon.IfdTagIdAndIndex) (ib *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - thisIb := rootIb - - // Since we're calling ourselves recursively with incrementally different - // paths, the FQ IFD-path of the parent that called us needs to be passed - // in, in order for us to know it. - var parentLineage []exifcommon.IfdTagIdAndIndex - if parentIb != nil { - var err error - - parentLineage, err = thisIb.ifdMapping.ResolvePath(parentIb.IfdIdentity().String()) - log.PanicIf(err) - } - - // Process the current path part. - currentItIi := currentLineage[0] - - // Make sure the leftmost part of the FQ IFD-path agrees with the IB we - // were given. - - expectedFqRootIfdPath := "" - if parentLineage != nil { - expectedLineage := append(parentLineage, currentItIi) - expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(expectedLineage) - } else { - expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(currentLineage[:1]) - } - - if expectedFqRootIfdPath != thisIb.IfdIdentity().String() { - log.Panicf("the FQ IFD-path [%s] we were given does not match the builder's FQ IFD-path [%s]", expectedFqRootIfdPath, thisIb.IfdIdentity().String()) - } - - // If we actually wanted a sibling (currentItIi.Index > 0) then seek to it, - // appending new siblings, as required, until we get there. - for i := 0; i < currentItIi.Index; i++ { - if thisIb.nextIb == nil { - // Generate an FQ IFD-path for the sibling. It'll use the same - // non-FQ IFD-path as the current IB. - - iiSibling := thisIb.IfdIdentity().NewSibling(i + 1) - thisIb.nextIb = NewIfdBuilder(thisIb.ifdMapping, thisIb.tagIndex, iiSibling, thisIb.byteOrder) - } - - thisIb = thisIb.nextIb - } - - // There is no child IFD to process. We're done. - if len(currentLineage) == 1 { - return thisIb, nil - } - - // Establish the next child to be processed. - - childItii := currentLineage[1] - - var foundChild *IfdBuilder - for _, bt := range thisIb.tags { - if bt.value.IsIb() == false { - continue - } - - childIb := bt.value.Ib() - - if childIb.IfdIdentity().TagId() == childItii.TagId { - foundChild = childIb - break - } - } - - // If we didn't find the child, add it. - - if foundChild == nil { - currentIfdTag := thisIb.IfdIdentity().IfdTag() - - childIfdTag := - exifcommon.NewIfdTag( - ¤tIfdTag, - childItii.TagId, - childItii.Name) - - iiChild := thisIb.IfdIdentity().NewChild(childIfdTag, 0) - - foundChild = - NewIfdBuilder( - thisIb.ifdMapping, - thisIb.tagIndex, - iiChild, - thisIb.byteOrder) - - err = thisIb.AddChildIb(foundChild) - log.PanicIf(err) - } - - finalIb, err := getOrCreateIbFromRootIbInner(foundChild, thisIb, currentLineage[1:]) - log.PanicIf(err) - - return finalIb, nil -} - -// GetOrCreateIbFromRootIb returns an IB representing the requested IFD, even if -// an IB doesn't already exist for it. This function may call itself -// recursively. -func GetOrCreateIbFromRootIb(rootIb *IfdBuilder, fqIfdPath string) (ib *IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // lineage is a necessity of our recursion process. It doesn't include any - // parent IFDs on its left-side; it starts with the current IB only. - lineage, err := rootIb.ifdMapping.ResolvePath(fqIfdPath) - log.PanicIf(err) - - ib, err = getOrCreateIbFromRootIbInner(rootIb, nil, lineage) - log.PanicIf(err) - - return ib, nil -} - -func (ib *IfdBuilder) String() string { - nextIfdPhrase := "" - if ib.nextIb != nil { - // TODO(dustin): We were setting this to ii.String(), but we were getting hex-data when printing this after building from an existing chain. - nextIfdPhrase = ib.nextIb.IfdIdentity().UnindexedString() - } - - return fmt.Sprintf("IfdBuilder", ib.IfdIdentity().UnindexedString(), ib.IfdIdentity().TagId(), len(ib.tags), ib.existingOffset, nextIfdPhrase) -} - -func (ib *IfdBuilder) Tags() (tags []*BuilderTag) { - return ib.tags -} - -// SetThumbnail sets thumbnail data. -// -// NOTES: -// -// - We don't manage any facet of the thumbnail data. This is the -// responsibility of the user/developer. -// - This method will fail unless the thumbnail is set on a the root IFD. -// However, in order to be valid, it must be set on the second one, linked to -// by the first, as per the EXIF/TIFF specification. -// - We set the offset to (0) now but will allocate the data and properly assign -// the offset when the IB is encoded (later). -func (ib *IfdBuilder) SetThumbnail(data []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if ib.IfdIdentity().UnindexedString() != exifcommon.IfdStandardIfdIdentity.UnindexedString() { - log.Panicf("thumbnails can only go into a root Ifd (and only the second one)") - } - - // TODO(dustin): !! Add a test for this function. - - if data == nil || len(data) == 0 { - log.Panic("thumbnail is empty") - } - - ib.thumbnailData = data - - ibtvfb := NewIfdBuilderTagValueFromBytes(ib.thumbnailData) - offsetBt := - NewBuilderTag( - ib.IfdIdentity().UnindexedString(), - ThumbnailOffsetTagId, - exifcommon.TypeLong, - ibtvfb, - ib.byteOrder) - - err = ib.Set(offsetBt) - log.PanicIf(err) - - thumbnailSizeIt, err := ib.tagIndex.Get(ib.IfdIdentity(), ThumbnailSizeTagId) - log.PanicIf(err) - - sizeBt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), thumbnailSizeIt, ib.byteOrder, []uint32{uint32(len(ib.thumbnailData))}) - - err = ib.Set(sizeBt) - log.PanicIf(err) - - return nil -} - -func (ib *IfdBuilder) Thumbnail() []byte { - return ib.thumbnailData -} - -func (ib *IfdBuilder) printTagTree(levels int) { - indent := strings.Repeat(" ", levels*2) - - i := 0 - for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { - prefix := " " - if i > 0 { - prefix = ">" - } - - if levels == 0 { - fmt.Printf("%s%sIFD: %s INDEX=(%d)\n", indent, prefix, currentIb, i) - } else { - fmt.Printf("%s%sChild IFD: %s\n", indent, prefix, currentIb) - } - - if len(currentIb.tags) > 0 { - fmt.Printf("\n") - - for i, tag := range currentIb.tags { - isChildIb := false - _, err := ib.ifdMapping.GetChild(currentIb.IfdIdentity().UnindexedString(), tag.tagId) - if err == nil { - isChildIb = true - } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - tagName := "" - - // If a normal tag (not a child IFD) get the name. - if isChildIb == true { - tagName = "" - } else { - it, err := ib.tagIndex.Get(ib.ifdIdentity, tag.tagId) - if log.Is(err, ErrTagNotFound) == true { - tagName = "" - } else if err != nil { - log.Panic(err) - } else { - tagName = it.Name - } - } - - value := tag.Value() - - if value.IsIb() == true { - fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, value.Ib()) - } else { - fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, tag) - } - - if isChildIb == true { - if tag.value.IsIb() == false { - log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) - } - - fmt.Printf("\n") - - childIb := tag.value.Ib() - childIb.printTagTree(levels + 1) - } - } - - fmt.Printf("\n") - } - - i++ - } -} - -func (ib *IfdBuilder) PrintTagTree() { - ib.printTagTree(0) -} - -func (ib *IfdBuilder) printIfdTree(levels int) { - indent := strings.Repeat(" ", levels*2) - - i := 0 - for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { - prefix := " " - if i > 0 { - prefix = ">" - } - - fmt.Printf("%s%s%s\n", indent, prefix, currentIb) - - if len(currentIb.tags) > 0 { - for _, tag := range currentIb.tags { - isChildIb := false - _, err := ib.ifdMapping.GetChild(currentIb.IfdIdentity().UnindexedString(), tag.tagId) - if err == nil { - isChildIb = true - } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - if isChildIb == true { - if tag.value.IsIb() == false { - log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) - } - - childIb := tag.value.Ib() - childIb.printIfdTree(levels + 1) - } - } - } - - i++ - } -} - -func (ib *IfdBuilder) PrintIfdTree() { - ib.printIfdTree(0) -} - -func (ib *IfdBuilder) dumpToStrings(thisIb *IfdBuilder, prefix string, tagId uint16, lines []string) (linesOutput []string) { - if lines == nil { - linesOutput = make([]string, 0) - } else { - linesOutput = lines - } - - siblingIfdIndex := 0 - for ; thisIb != nil; thisIb = thisIb.nextIb { - line := fmt.Sprintf("IFD", prefix, thisIb.IfdIdentity().String(), siblingIfdIndex, thisIb.IfdIdentity().TagId(), tagId) - linesOutput = append(linesOutput, line) - - for i, tag := range thisIb.tags { - var childIb *IfdBuilder - childIfdName := "" - if tag.value.IsIb() == true { - childIb = tag.value.Ib() - childIfdName = childIb.IfdIdentity().UnindexedString() - } - - line := fmt.Sprintf("TAG", prefix, thisIb.IfdIdentity().String(), thisIb.IfdIdentity().TagId(), childIfdName, i, tag.tagId) - linesOutput = append(linesOutput, line) - - if childIb == nil { - continue - } - - childPrefix := "" - if prefix == "" { - childPrefix = fmt.Sprintf("%s", thisIb.IfdIdentity().UnindexedString()) - } else { - childPrefix = fmt.Sprintf("%s->%s", prefix, thisIb.IfdIdentity().UnindexedString()) - } - - linesOutput = thisIb.dumpToStrings(childIb, childPrefix, tag.tagId, linesOutput) - } - - siblingIfdIndex++ - } - - return linesOutput -} - -func (ib *IfdBuilder) DumpToStrings() (lines []string) { - return ib.dumpToStrings(ib, "", 0, lines) -} - -func (ib *IfdBuilder) SetNextIb(nextIb *IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ib.nextIb = nextIb - - return nil -} - -func (ib *IfdBuilder) DeleteN(tagId uint16, n int) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if n < 1 { - log.Panicf("N must be at least 1: (%d)", n) - } - - for n > 0 { - j := -1 - for i, bt := range ib.tags { - if bt.tagId == tagId { - j = i - break - } - } - - if j == -1 { - log.Panic(ErrTagEntryNotFound) - } - - ib.tags = append(ib.tags[:j], ib.tags[j+1:]...) - n-- - } - - return nil -} - -func (ib *IfdBuilder) DeleteFirst(tagId uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = ib.DeleteN(tagId, 1) - log.PanicIf(err) - - return nil -} - -func (ib *IfdBuilder) DeleteAll(tagId uint16) (n int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for { - err = ib.DeleteN(tagId, 1) - if log.Is(err, ErrTagEntryNotFound) == true { - break - } else if err != nil { - log.Panic(err) - } - - n++ - } - - return n, nil -} - -func (ib *IfdBuilder) ReplaceAt(position int, bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if position < 0 { - log.Panicf("replacement position must be 0 or greater") - } else if position >= len(ib.tags) { - log.Panicf("replacement position does not exist") - } - - ib.tags[position] = bt - - return nil -} - -func (ib *IfdBuilder) Replace(tagId uint16, bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - position, err := ib.Find(tagId) - log.PanicIf(err) - - ib.tags[position] = bt - - return nil -} - -// Set will add a new entry or update an existing entry. -func (ib *IfdBuilder) Set(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - position, err := ib.Find(bt.tagId) - if err == nil { - ib.tags[position] = bt - } else if log.Is(err, ErrTagEntryNotFound) == true { - err = ib.add(bt) - log.PanicIf(err) - } else { - log.Panic(err) - } - - return nil -} - -func (ib *IfdBuilder) FindN(tagId uint16, maxFound int) (found []int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found = make([]int, 0) - - for i, bt := range ib.tags { - if bt.tagId == tagId { - found = append(found, i) - if maxFound == 0 || len(found) >= maxFound { - break - } - } - } - - return found, nil -} - -func (ib *IfdBuilder) Find(tagId uint16) (position int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found, err := ib.FindN(tagId, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - return found[0], nil -} - -func (ib *IfdBuilder) FindTag(tagId uint16) (bt *BuilderTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - found, err := ib.FindN(tagId, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - position := found[0] - - return ib.tags[position], nil -} - -func (ib *IfdBuilder) FindTagWithName(tagName string) (bt *BuilderTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) - log.PanicIf(err) - - found, err := ib.FindN(it.Id, 1) - log.PanicIf(err) - - if len(found) == 0 { - log.Panic(ErrTagEntryNotFound) - } - - position := found[0] - - return ib.tags[position], nil -} - -func (ib *IfdBuilder) add(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if bt.ifdPath == "" { - log.Panicf("BuilderTag ifdPath is not set: %s", bt) - } else if bt.typeId == 0x0 { - log.Panicf("BuilderTag type-ID is not set: %s", bt) - } else if bt.value == nil { - log.Panicf("BuilderTag value is not set: %s", bt) - } - - ib.tags = append(ib.tags, bt) - return nil -} - -func (ib *IfdBuilder) Add(bt *BuilderTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if bt.value.IsIb() == true { - log.Panicf("child IfdBuilders must be added via AddChildIb() or AddTagsFromExisting(), not Add()") - } - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// AddChildIb adds a tag that branches to a new IFD. -func (ib *IfdBuilder) AddChildIb(childIb *IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if childIb.IfdIdentity().TagId() == 0 { - log.Panicf("IFD can not be used as a child IFD (not associated with a tag-ID): %v", childIb) - } else if childIb.byteOrder != ib.byteOrder { - log.Panicf("Child IFD does not have the same byte-order: [%s] != [%s]", childIb.byteOrder, ib.byteOrder) - } - - // Since no standard IFDs supports occur`ring more than once, check that a - // tag of this type has not been previously added. Note that we just search - // the current IFD and *not every* IFD. - for _, bt := range childIb.tags { - if bt.tagId == childIb.IfdIdentity().TagId() { - log.Panicf("child-IFD already added: %v", childIb.IfdIdentity().UnindexedString()) - } - } - - bt := ib.NewBuilderTagFromBuilder(childIb) - ib.tags = append(ib.tags, bt) - - return nil -} - -func (ib *IfdBuilder) NewBuilderTagFromBuilder(childIb *IfdBuilder) (bt *BuilderTag) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - value := NewIfdBuilderTagValueFromIfdBuilder(childIb) - - bt = NewChildIfdBuilderTag( - ib.IfdIdentity().UnindexedString(), - childIb.IfdIdentity().TagId(), - value) - - return bt -} - -// AddTagsFromExisting does a verbatim copy of the entries in `ifd` to this -// builder. It excludes child IFDs. These must be added explicitly via -// `AddChildIb()`. -func (ib *IfdBuilder) AddTagsFromExisting(ifd *Ifd, includeTagIds []uint16, excludeTagIds []uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - thumbnailData, err := ifd.Thumbnail() - if err == nil { - err = ib.SetThumbnail(thumbnailData) - log.PanicIf(err) - } else if log.Is(err, ErrNoThumbnail) == false { - log.Panic(err) - } - - for i, ite := range ifd.Entries { - if ite.IsThumbnailOffset() == true || ite.IsThumbnailSize() { - // These will be added on-the-fly when we encode. - continue - } - - if excludeTagIds != nil && len(excludeTagIds) > 0 { - found := false - for _, excludedTagId := range excludeTagIds { - if excludedTagId == ite.TagId() { - found = true - } - } - - if found == true { - continue - } - } - - if includeTagIds != nil && len(includeTagIds) > 0 { - // Whether or not there was a list of excludes, if there is a list - // of includes than the current tag has to be in it. - - found := false - for _, includedTagId := range includeTagIds { - if includedTagId == ite.TagId() { - found = true - break - } - } - - if found == false { - continue - } - } - - var bt *BuilderTag - - if ite.ChildIfdPath() != "" { - // If we want to add an IFD tag, we'll have to build it first and - // *then* add it via a different method. - - // Figure out which of the child-IFDs that are associated with - // this IFD represents this specific child IFD. - - var childIfd *Ifd - for _, thisChildIfd := range ifd.Children { - if thisChildIfd.ParentTagIndex != i { - continue - } else if thisChildIfd.ifdIdentity.TagId() != 0xffff && thisChildIfd.ifdIdentity.TagId() != ite.TagId() { - log.Panicf("child-IFD tag is not correct: TAG-POSITION=(%d) ITE=%s CHILD-IFD=%s", thisChildIfd.ParentTagIndex, ite, thisChildIfd) - } - - childIfd = thisChildIfd - break - } - - if childIfd == nil { - childTagIds := make([]string, len(ifd.Children)) - for j, childIfd := range ifd.Children { - childTagIds[j] = fmt.Sprintf("0x%04x (parent tag-position %d)", childIfd.ifdIdentity.TagId(), childIfd.ParentTagIndex) - } - - log.Panicf("could not find child IFD for child ITE: IFD-PATH=[%s] TAG-ID=(0x%04x) CURRENT-TAG-POSITION=(%d) CHILDREN=%v", ite.IfdPath(), ite.TagId(), i, childTagIds) - } - - childIb := NewIfdBuilderFromExistingChain(childIfd) - bt = ib.NewBuilderTagFromBuilder(childIb) - } else { - // Non-IFD tag. - - rawBytes, err := ite.GetRawBytes() - log.PanicIf(err) - - value := NewIfdBuilderTagValueFromBytes(rawBytes) - - bt = NewBuilderTag( - ifd.ifdIdentity.UnindexedString(), - ite.TagId(), - ite.TagType(), - value, - ib.byteOrder) - } - - err := ib.add(bt) - log.PanicIf(err) - } - - return nil -} - -// AddStandard quickly and easily composes and adds the tag using the -// information already known about a tag. Only works with standard tags. -func (ib *IfdBuilder) AddStandard(tagId uint16, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.Get(ib.IfdIdentity(), tagId) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// AddStandardWithName quickly and easily composes and adds the tag using the -// information already known about a tag (using the name). Only works with -// standard tags. -func (ib *IfdBuilder) AddStandardWithName(tagName string, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) - - err = ib.add(bt) - log.PanicIf(err) - - return nil -} - -// SetStandard quickly and easily composes and adds or replaces the tag using -// the information already known about a tag. Only works with standard tags. -func (ib *IfdBuilder) SetStandard(tagId uint16, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test for this function. - - it, err := ib.tagIndex.Get(ib.IfdIdentity(), tagId) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) - - i, err := ib.Find(tagId) - if err != nil { - if log.Is(err, ErrTagEntryNotFound) == false { - log.Panic(err) - } - - ib.tags = append(ib.tags, bt) - } else { - ib.tags[i] = bt - } - - return nil -} - -// SetStandardWithName quickly and easily composes and adds or replaces the -// tag using the information already known about a tag (using the name). Only -// works with standard tags. -func (ib *IfdBuilder) SetStandardWithName(tagName string, value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test for this function. - - it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) - log.PanicIf(err) - - bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) - - i, err := ib.Find(bt.tagId) - if err != nil { - if log.Is(err, ErrTagEntryNotFound) == false { - log.Panic(err) - } - - ib.tags = append(ib.tags, bt) - } else { - ib.tags[i] = bt - } - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/ifd_builder_encode.go b/vendor/github.com/dsoprea/go-exif/v2/ifd_builder_encode.go deleted file mode 100644 index a0bac3e5b..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/ifd_builder_encode.go +++ /dev/null @@ -1,532 +0,0 @@ -package exif - -import ( - "bytes" - "fmt" - "strings" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -const ( - // Tag-ID + Tag-Type + Unit-Count + Value/Offset. - IfdTagEntrySize = uint32(2 + 2 + 4 + 4) -) - -type ByteWriter struct { - b *bytes.Buffer - byteOrder binary.ByteOrder -} - -func NewByteWriter(b *bytes.Buffer, byteOrder binary.ByteOrder) (bw *ByteWriter) { - return &ByteWriter{ - b: b, - byteOrder: byteOrder, - } -} - -func (bw ByteWriter) writeAsBytes(value interface{}) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = binary.Write(bw.b, bw.byteOrder, value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteUint32(value uint32) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = bw.writeAsBytes(value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteUint16(value uint16) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - err = bw.writeAsBytes(value) - log.PanicIf(err) - - return nil -} - -func (bw ByteWriter) WriteFourBytes(value []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - len_ := len(value) - if len_ != 4 { - log.Panicf("value is not four-bytes: (%d)", len_) - } - - _, err = bw.b.Write(value) - log.PanicIf(err) - - return nil -} - -// ifdOffsetIterator keeps track of where the next IFD should be written by -// keeping track of where the offsets start, the data that has been added, and -// bumping the offset *when* the data is added. -type ifdDataAllocator struct { - offset uint32 - b bytes.Buffer -} - -func newIfdDataAllocator(ifdDataAddressableOffset uint32) *ifdDataAllocator { - return &ifdDataAllocator{ - offset: ifdDataAddressableOffset, - } -} - -func (ida *ifdDataAllocator) Allocate(value []byte) (offset uint32, err error) { - _, err = ida.b.Write(value) - log.PanicIf(err) - - offset = ida.offset - ida.offset += uint32(len(value)) - - return offset, nil -} - -func (ida *ifdDataAllocator) NextOffset() uint32 { - return ida.offset -} - -func (ida *ifdDataAllocator) Bytes() []byte { - return ida.b.Bytes() -} - -// IfdByteEncoder converts an IB to raw bytes (for writing) while also figuring -// out all of the allocations and indirection that is required for extended -// data. -type IfdByteEncoder struct { - // journal holds a list of actions taken while encoding. - journal [][3]string -} - -func NewIfdByteEncoder() (ibe *IfdByteEncoder) { - return &IfdByteEncoder{ - journal: make([][3]string, 0), - } -} - -func (ibe *IfdByteEncoder) Journal() [][3]string { - return ibe.journal -} - -func (ibe *IfdByteEncoder) TableSize(entryCount int) uint32 { - // Tag-Count + (Entry-Size * Entry-Count) + Next-IFD-Offset. - return uint32(2) + (IfdTagEntrySize * uint32(entryCount)) + uint32(4) -} - -func (ibe *IfdByteEncoder) pushToJournal(where, direction, format string, args ...interface{}) { - event := [3]string{ - direction, - where, - fmt.Sprintf(format, args...), - } - - ibe.journal = append(ibe.journal, event) -} - -// PrintJournal prints a hierarchical representation of the steps taken during -// encoding. -func (ibe *IfdByteEncoder) PrintJournal() { - maxWhereLength := 0 - for _, event := range ibe.journal { - where := event[1] - - len_ := len(where) - if len_ > maxWhereLength { - maxWhereLength = len_ - } - } - - level := 0 - for i, event := range ibe.journal { - direction := event[0] - where := event[1] - message := event[2] - - if direction != ">" && direction != "<" && direction != "-" { - log.Panicf("journal operation not valid: [%s]", direction) - } - - if direction == "<" { - if level <= 0 { - log.Panicf("journal operations unbalanced (too many closes)") - } - - level-- - } - - indent := strings.Repeat(" ", level) - - fmt.Printf("%3d %s%s %s: %s\n", i, indent, direction, where, message) - - if direction == ">" { - level++ - } - } - - if level != 0 { - log.Panicf("journal operations unbalanced (too many opens)") - } -} - -// encodeTagToBytes encodes the given tag to a byte stream. If -// `nextIfdOffsetToWrite` is more than (0), recurse into child IFDs -// (`nextIfdOffsetToWrite` is required in order for them to know where the its -// IFD data will be written, in order for them to know the offset of where -// their allocated-data block will start, which follows right behind). -func (ibe *IfdByteEncoder) encodeTagToBytes(ib *IfdBuilder, bt *BuilderTag, bw *ByteWriter, ida *ifdDataAllocator, nextIfdOffsetToWrite uint32) (childIfdBlock []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Write tag-ID. - err = bw.WriteUint16(bt.tagId) - log.PanicIf(err) - - // Works for both values and child IFDs (which have an official size of - // LONG). - err = bw.WriteUint16(uint16(bt.typeId)) - log.PanicIf(err) - - // Write unit-count. - - if bt.value.IsBytes() == true { - effectiveType := bt.typeId - if bt.typeId == exifcommon.TypeUndefined { - effectiveType = exifcommon.TypeByte - } - - // It's a non-unknown value.Calculate the count of values of - // the type that we're writing and the raw bytes for the whole list. - - typeSize := uint32(effectiveType.Size()) - - valueBytes := bt.value.Bytes() - - len_ := len(valueBytes) - unitCount := uint32(len_) / typeSize - - if _, found := tagsWithoutAlignment[bt.tagId]; found == false { - remainder := uint32(len_) % typeSize - - if remainder > 0 { - log.Panicf("tag (0x%04x) value of (%d) bytes not evenly divisible by type-size (%d)", bt.tagId, len_, typeSize) - } - } - - err = bw.WriteUint32(unitCount) - log.PanicIf(err) - - // Write four-byte value/offset. - - if len_ > 4 { - offset, err := ida.Allocate(valueBytes) - log.PanicIf(err) - - err = bw.WriteUint32(offset) - log.PanicIf(err) - } else { - fourBytes := make([]byte, 4) - copy(fourBytes, valueBytes) - - err = bw.WriteFourBytes(fourBytes) - log.PanicIf(err) - } - } else { - if bt.value.IsIb() == false { - log.Panicf("tag value is not a byte-slice but also not a child IB: %v", bt) - } - - // Write unit-count (one LONG representing one offset). - err = bw.WriteUint32(1) - log.PanicIf(err) - - if nextIfdOffsetToWrite > 0 { - var err error - - ibe.pushToJournal("encodeTagToBytes", ">", "[%s]->[%s]", ib.IfdIdentity().UnindexedString(), bt.value.Ib().IfdIdentity().UnindexedString()) - - // Create the block of IFD data and everything it requires. - childIfdBlock, err = ibe.encodeAndAttachIfd(bt.value.Ib(), nextIfdOffsetToWrite) - log.PanicIf(err) - - ibe.pushToJournal("encodeTagToBytes", "<", "[%s]->[%s]", bt.value.Ib().IfdIdentity().UnindexedString(), ib.IfdIdentity().UnindexedString()) - - // Use the next-IFD offset for it. The IFD will actually get - // attached after we return. - err = bw.WriteUint32(nextIfdOffsetToWrite) - log.PanicIf(err) - - } else { - // No child-IFDs are to be allocated. Finish the entry with a NULL - // pointer. - - ibe.pushToJournal("encodeTagToBytes", "-", "*Not* descending to child: [%s]", bt.value.Ib().IfdIdentity().UnindexedString()) - - err = bw.WriteUint32(0) - log.PanicIf(err) - } - } - - return childIfdBlock, nil -} - -// encodeIfdToBytes encodes the given IB to a byte-slice. We are given the -// offset at which this IFD will be written. This method is used called both to -// pre-determine how big the table is going to be (so that we can calculate the -// address to allocate data at) as well as to write the final table. -// -// It is necessary to fully realize the table in order to predetermine its size -// because it is not enough to know the size of the table: If there are child -// IFDs, we will not be able to allocate them without first knowing how much -// data we need to allocate for the current IFD. -func (ibe *IfdByteEncoder) encodeIfdToBytes(ib *IfdBuilder, ifdAddressableOffset uint32, nextIfdOffsetToWrite uint32, setNextIb bool) (data []byte, tableSize uint32, dataSize uint32, childIfdSizes []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ibe.pushToJournal("encodeIfdToBytes", ">", "%s", ib) - - tableSize = ibe.TableSize(len(ib.tags)) - - b := new(bytes.Buffer) - bw := NewByteWriter(b, ib.byteOrder) - - // Write tag count. - err = bw.WriteUint16(uint16(len(ib.tags))) - log.PanicIf(err) - - ida := newIfdDataAllocator(ifdAddressableOffset) - - childIfdBlocks := make([][]byte, 0) - - // Write raw bytes for each tag entry. Allocate larger data to be referred - // to in the follow-up data-block as required. Any "unknown"-byte tags that - // we can't parse will not be present here (using AddTagsFromExisting(), at - // least). - for _, bt := range ib.tags { - childIfdBlock, err := ibe.encodeTagToBytes(ib, bt, bw, ida, nextIfdOffsetToWrite) - log.PanicIf(err) - - if childIfdBlock != nil { - // We aren't allowed to have non-nil child IFDs if we're just - // sizing things up. - if nextIfdOffsetToWrite == 0 { - log.Panicf("no IFD offset provided for child-IFDs; no new child-IFDs permitted") - } - - nextIfdOffsetToWrite += uint32(len(childIfdBlock)) - childIfdBlocks = append(childIfdBlocks, childIfdBlock) - } - } - - dataBytes := ida.Bytes() - dataSize = uint32(len(dataBytes)) - - childIfdSizes = make([]uint32, len(childIfdBlocks)) - childIfdsTotalSize := uint32(0) - for i, childIfdBlock := range childIfdBlocks { - len_ := uint32(len(childIfdBlock)) - childIfdSizes[i] = len_ - childIfdsTotalSize += len_ - } - - // N the link from this IFD to the next IFD that will be written in the - // next cycle. - if setNextIb == true { - // Write address of next IFD in chain. This will be the original - // allocation offset plus the size of everything we have allocated for - // this IFD and its child-IFDs. - // - // It is critical that this number is stepped properly. We experienced - // an issue whereby it first looked like we were duplicating the IFD and - // then that we were duplicating the tags in the wrong IFD, and then - // finally we determined that the next-IFD offset for the first IFD was - // accidentally pointing back to the EXIF IFD, so we were visiting it - // twice when visiting through the tags after decoding. It was an - // expensive bug to find. - - ibe.pushToJournal("encodeIfdToBytes", "-", "Setting 'next' IFD to (0x%08x).", nextIfdOffsetToWrite) - - err := bw.WriteUint32(nextIfdOffsetToWrite) - log.PanicIf(err) - } else { - err := bw.WriteUint32(0) - log.PanicIf(err) - } - - _, err = b.Write(dataBytes) - log.PanicIf(err) - - // Append any child IFD blocks after our table and data blocks. These IFDs - // were equipped with the appropriate offset information so it's expected - // that all offsets referred to by these will be correct. - // - // Note that child-IFDs are append after the current IFD and before the - // next IFD, as opposed to the root IFDs, which are chained together but - // will be interrupted by these child-IFDs (which is expected, per the - // standard). - - for _, childIfdBlock := range childIfdBlocks { - _, err = b.Write(childIfdBlock) - log.PanicIf(err) - } - - ibe.pushToJournal("encodeIfdToBytes", "<", "%s", ib) - - return b.Bytes(), tableSize, dataSize, childIfdSizes, nil -} - -// encodeAndAttachIfd is a reentrant function that processes the IFD chain. -func (ibe *IfdByteEncoder) encodeAndAttachIfd(ib *IfdBuilder, ifdAddressableOffset uint32) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ibe.pushToJournal("encodeAndAttachIfd", ">", "%s", ib) - - b := new(bytes.Buffer) - - i := 0 - - for thisIb := ib; thisIb != nil; thisIb = thisIb.nextIb { - - // Do a dry-run in order to pre-determine its size requirement. - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Beginning encoding process: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Calculating size: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) - - _, tableSize, allocatedDataSize, _, err := ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, 0, false) - log.PanicIf(err) - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Finished calculating size: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) - - ifdAddressableOffset += tableSize - nextIfdOffsetToWrite := ifdAddressableOffset + allocatedDataSize - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Next IFD will be written at offset (0x%08x)", nextIfdOffsetToWrite) - - // Write our IFD as well as any child-IFDs (now that we know the offset - // where new IFDs and their data will be allocated). - - setNextIb := thisIb.nextIb != nil - - ibe.pushToJournal("encodeAndAttachIfd", ">", "Encoding starting: (%d) [%s] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, thisIb.IfdIdentity().UnindexedString(), nextIfdOffsetToWrite) - - tableAndAllocated, effectiveTableSize, effectiveAllocatedDataSize, childIfdSizes, err := - ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, nextIfdOffsetToWrite, setNextIb) - - log.PanicIf(err) - - if effectiveTableSize != tableSize { - log.Panicf("written table size does not match the pre-calculated table size: (%d) != (%d) %s", effectiveTableSize, tableSize, ib) - } else if effectiveAllocatedDataSize != allocatedDataSize { - log.Panicf("written allocated-data size does not match the pre-calculated allocated-data size: (%d) != (%d) %s", effectiveAllocatedDataSize, allocatedDataSize, ib) - } - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Encoding done: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) - - totalChildIfdSize := uint32(0) - for _, childIfdSize := range childIfdSizes { - totalChildIfdSize += childIfdSize - } - - if len(tableAndAllocated) != int(tableSize+allocatedDataSize+totalChildIfdSize) { - log.Panicf("IFD table and data is not a consistent size: (%d) != (%d)", len(tableAndAllocated), tableSize+allocatedDataSize+totalChildIfdSize) - } - - // TODO(dustin): We might want to verify the original tableAndAllocated length, too. - - _, err = b.Write(tableAndAllocated) - log.PanicIf(err) - - // Advance past what we've allocated, thus far. - - ifdAddressableOffset += allocatedDataSize + totalChildIfdSize - - ibe.pushToJournal("encodeAndAttachIfd", "<", "Finishing encoding process: (%d) [%s] [FINAL:] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, ib.IfdIdentity().UnindexedString(), nextIfdOffsetToWrite) - - i++ - } - - ibe.pushToJournal("encodeAndAttachIfd", "<", "%s", ib) - - return b.Bytes(), nil -} - -// EncodeToExifPayload is the base encoding step that transcribes the entire IB -// structure to its on-disk layout. -func (ibe *IfdByteEncoder) EncodeToExifPayload(ib *IfdBuilder) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - data, err = ibe.encodeAndAttachIfd(ib, ExifDefaultFirstIfdOffset) - log.PanicIf(err) - - return data, nil -} - -// EncodeToExif calls EncodeToExifPayload and then packages the result into a -// complete EXIF block. -func (ibe *IfdByteEncoder) EncodeToExif(ib *IfdBuilder) (data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - encodedIfds, err := ibe.EncodeToExifPayload(ib) - log.PanicIf(err) - - // Wrap the IFD in a formal EXIF block. - - b := new(bytes.Buffer) - - headerBytes, err := BuildExifHeader(ib.byteOrder, ExifDefaultFirstIfdOffset) - log.PanicIf(err) - - _, err = b.Write(headerBytes) - log.PanicIf(err) - - _, err = b.Write(encodedIfds) - log.PanicIf(err) - - return b.Bytes(), nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/ifd_enumerate.go b/vendor/github.com/dsoprea/go-exif/v2/ifd_enumerate.go deleted file mode 100644 index 33a5f84b3..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/ifd_enumerate.go +++ /dev/null @@ -1,1521 +0,0 @@ -package exif - -import ( - "bytes" - "errors" - "fmt" - "io" - "strconv" - "strings" - "time" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" - "github.com/dsoprea/go-exif/v2/undefined" -) - -var ( - ifdEnumerateLogger = log.NewLogger("exif.ifd_enumerate") -) - -var ( - // ErrNoThumbnail means that no thumbnail was found. - ErrNoThumbnail = errors.New("no thumbnail") - - // ErrNoGpsTags means that no GPS info was found. - ErrNoGpsTags = errors.New("no gps tags") - - // ErrTagTypeNotValid means that the tag-type is not valid. - ErrTagTypeNotValid = errors.New("tag type invalid") - - // ErrOffsetInvalid means that the file offset is not valid. - ErrOffsetInvalid = errors.New("file offset invalid") -) - -var ( - // ValidGpsVersions is the list of recognized EXIF GPS versions/signatures. - ValidGpsVersions = [][4]byte{ - // 2.0.0.0 appears to have a very similar format to 2.2.0.0, so enabling - // it under that assumption. - // - // IFD-PATH=[IFD] ID=(0x8825) NAME=[GPSTag] COUNT=(1) TYPE=[LONG] VALUE=[114] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0000) NAME=[GPSVersionID] COUNT=(4) TYPE=[BYTE] VALUE=[02 00 00 00] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0001) NAME=[GPSLatitudeRef] COUNT=(2) TYPE=[ASCII] VALUE=[S] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0002) NAME=[GPSLatitude] COUNT=(3) TYPE=[RATIONAL] VALUE=[38/1...] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0003) NAME=[GPSLongitudeRef] COUNT=(2) TYPE=[ASCII] VALUE=[E] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0004) NAME=[GPSLongitude] COUNT=(3) TYPE=[RATIONAL] VALUE=[144/1...] - // IFD-PATH=[IFD/GPSInfo] ID=(0x0012) NAME=[GPSMapDatum] COUNT=(7) TYPE=[ASCII] VALUE=[WGS-84] - // - {2, 0, 0, 0}, - - {2, 2, 0, 0}, - - // Suddenly appeared at the default in 2.31: https://home.jeita.or.jp/tsc/std-pdf/CP-3451D.pdf - // - // Note that the presence of 2.3.0.0 doesn't seem to guarantee - // coordinates. In some cases, we seen just the following: - // - // GPS Tag Version |2.3.0.0 - // GPS Receiver Status |V - // Geodetic Survey Data|WGS-84 - // GPS Differential Cor|0 - // - {2, 3, 0, 0}, - } -) - -// byteParser knows how to decode an IFD and all of the tags it -// describes. -// -// The IFDs and the actual values can float throughout the EXIF block, but the -// IFD itself is just a minor header followed by a set of repeating, -// statically-sized records. So, the tags (though notnecessarily their values) -// are fairly simple to enumerate. -type byteParser struct { - byteOrder binary.ByteOrder - addressableData []byte - ifdOffset uint32 - currentOffset uint32 -} - -func newByteParser(addressableData []byte, byteOrder binary.ByteOrder, ifdOffset uint32) (bp *byteParser, err error) { - if ifdOffset >= uint32(len(addressableData)) { - return nil, ErrOffsetInvalid - } - - // TODO(dustin): Add test - - bp = &byteParser{ - addressableData: addressableData, - byteOrder: byteOrder, - currentOffset: ifdOffset, - } - - return bp, nil -} - -// getUint16 reads a uint16 and advances both our current and our current -// accumulator (which allows us to know how far to seek to the beginning of the -// next IFD when it's time to jump). -func (bp *byteParser) getUint16() (value uint16, raw []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - needBytes := uint32(2) - - if bp.currentOffset+needBytes > uint32(len(bp.addressableData)) { - return 0, nil, io.EOF - } - - raw = bp.addressableData[bp.currentOffset : bp.currentOffset+needBytes] - value = bp.byteOrder.Uint16(raw) - - bp.currentOffset += uint32(needBytes) - - return value, raw, nil -} - -// getUint32 reads a uint32 and advances both our current and our current -// accumulator (which allows us to know how far to seek to the beginning of the -// next IFD when it's time to jump). -func (bp *byteParser) getUint32() (value uint32, raw []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - needBytes := uint32(4) - - if bp.currentOffset+needBytes > uint32(len(bp.addressableData)) { - return 0, nil, io.EOF - } - - raw = bp.addressableData[bp.currentOffset : bp.currentOffset+needBytes] - value = bp.byteOrder.Uint32(raw) - - bp.currentOffset += uint32(needBytes) - - return value, raw, nil -} - -// CurrentOffset returns the starting offset but the number of bytes that we -// have parsed. This is arithmetic-based tracking, not a seek(0) operation. -func (bp *byteParser) CurrentOffset() uint32 { - return bp.currentOffset -} - -// IfdEnumerate is the main enumeration type. It knows how to parse the IFD -// containers in the EXIF blob. -type IfdEnumerate struct { - exifData []byte - byteOrder binary.ByteOrder - tagIndex *TagIndex - ifdMapping *exifcommon.IfdMapping - furthestOffset uint32 -} - -// NewIfdEnumerate returns a new instance of IfdEnumerate. -func NewIfdEnumerate(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, exifData []byte, byteOrder binary.ByteOrder) *IfdEnumerate { - return &IfdEnumerate{ - exifData: exifData, - byteOrder: byteOrder, - ifdMapping: ifdMapping, - tagIndex: tagIndex, - } -} - -func (ie *IfdEnumerate) getByteParser(ifdOffset uint32) (bp *byteParser, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - bp, err = - newByteParser( - ie.exifData[ExifAddressableAreaStart:], - ie.byteOrder, - ifdOffset) - - if err != nil { - if err == ErrOffsetInvalid { - return nil, err - } - - log.Panic(err) - } - - return bp, nil -} - -func (ie *IfdEnumerate) parseTag(ii *exifcommon.IfdIdentity, tagPosition int, bp *byteParser) (ite *IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tagId, _, err := bp.getUint16() - log.PanicIf(err) - - tagTypeRaw, _, err := bp.getUint16() - log.PanicIf(err) - - tagType := exifcommon.TagTypePrimitive(tagTypeRaw) - - unitCount, _, err := bp.getUint32() - log.PanicIf(err) - - valueOffset, rawValueOffset, err := bp.getUint32() - log.PanicIf(err) - - if tagType.IsValid() == false { - ite = &IfdTagEntry{ - tagId: tagId, - tagType: tagType, - } - - log.Panic(ErrTagTypeNotValid) - } - - ite = newIfdTagEntry( - ii, - tagId, - tagPosition, - tagType, - unitCount, - valueOffset, - rawValueOffset, - ie.exifData[ExifAddressableAreaStart:], - ie.byteOrder) - - ifdPath := ii.UnindexedString() - - // If it's an IFD but not a standard one, it'll just be seen as a LONG - // (the standard IFD tag type), later, unless we skip it because it's - // [likely] not even in the standard list of known tags. - mi, err := ie.ifdMapping.GetChild(ifdPath, tagId) - if err == nil { - currentIfdTag := ii.IfdTag() - - childIt := exifcommon.NewIfdTag(¤tIfdTag, tagId, mi.Name) - iiChild := ii.NewChild(childIt, 0) - ite.SetChildIfd(iiChild) - - // We also need to set `tag.ChildFqIfdPath` but can't do it here - // because we don't have the IFD index. - } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { - log.Panic(err) - } - - return ite, nil -} - -// TagVisitorFn is called for each tag when enumerating through the EXIF. -type TagVisitorFn func(fqIfdPath string, ifdIndex int, ite *IfdTagEntry) (err error) - -// postparseTag do some tag-level processing here following the parse of each. -func (ie *IfdEnumerate) postparseTag(ite *IfdTagEntry, med *MiscellaneousExifData) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - ii := ite.IfdIdentity() - - tagId := ite.TagId() - tagType := ite.TagType() - - it, err := ie.tagIndex.Get(ii, tagId) - if err == nil { - ite.setTagName(it.Name) - } else { - if err != ErrTagNotFound { - log.Panic(err) - } - - // This is an unknown tag. - - originalBt := exifcommon.BasicTag{ - FqIfdPath: ii.String(), - IfdPath: ii.UnindexedString(), - TagId: tagId, - } - - if med != nil { - med.unknownTags[originalBt] = exifcommon.BasicTag{} - } - - utilityLogger.Debugf(nil, - "Tag (0x%04x) is not valid for IFD [%s]. Attempting secondary "+ - "lookup.", tagId, ii.String()) - - // This will overwrite the existing `it` and `err`. Since `FindFirst()` - // might generate different Errors than `Get()`, the log message above - // is import to try and mitigate confusion in that case. - it, err = ie.tagIndex.FindFirst(tagId, tagType, nil) - if err != nil { - if err != ErrTagNotFound { - log.Panic(err) - } - - // This is supposed to be a convenience function and if we were - // to keep the name empty or set it to some placeholder, it - // might be mismanaged by the package that is calling us. If - // they want to specifically manage these types of tags, they - // can use more advanced functionality to specifically -handle - // unknown tags. - utilityLogger.Warningf(nil, - "Tag with ID (0x%04x) in IFD [%s] is not recognized and "+ - "will be ignored.", tagId, ii.String()) - - return ErrTagNotFound - } - - ite.setTagName(it.Name) - - utilityLogger.Warningf(nil, - "Tag with ID (0x%04x) is not valid for IFD [%s], but it *is* "+ - "valid as tag [%s] under IFD [%s] and has the same type "+ - "[%s], so we will use that. This EXIF blob was probably "+ - "written by a buggy implementation.", - tagId, ii.UnindexedString(), it.Name, it.IfdPath, - tagType) - - if med != nil { - med.unknownTags[originalBt] = exifcommon.BasicTag{ - IfdPath: it.IfdPath, - TagId: tagId, - } - } - } - - // This is a known tag (from the standard, unless the user did - // something different). - - // Skip any tags that have a type that doesn't match the type in the - // index (which is loaded with the standard and accept tag - // information unless configured otherwise). - // - // We've run into multiple instances of the same tag, where a) no - // tag should ever be repeated, and b) all but one had an incorrect - // type and caused parsing/conversion woes. So, this is a quick fix - // for those scenarios. - if it.DoesSupportType(tagType) == false { - ifdEnumerateLogger.Warningf(nil, - "Skipping tag [%s] (0x%04x) [%s] with an unexpected type: %v ∉ %v", - ii.UnindexedString(), tagId, it.Name, - tagType, it.SupportedTypes) - - return ErrTagNotFound - } - - return nil -} - -// parseIfd decodes the IFD block that we're currently sitting on the first -// byte of. -func (ie *IfdEnumerate) parseIfd(ii *exifcommon.IfdIdentity, bp *byteParser, visitor TagVisitorFn, doDescend bool, med *MiscellaneousExifData) (nextIfdOffset uint32, entries []*IfdTagEntry, thumbnailData []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tagCount, _, err := bp.getUint16() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "IFD [%s] tag-count: (%d)", ii.String(), tagCount) - - entries = make([]*IfdTagEntry, 0) - - var enumeratorThumbnailOffset *IfdTagEntry - var enumeratorThumbnailSize *IfdTagEntry - - for i := 0; i < int(tagCount); i++ { - ite, err := ie.parseTag(ii, i, bp) - if err != nil { - if log.Is(err, ErrTagTypeNotValid) == true { - // Technically, we have the type on-file in the tags-index, but - // if the type stored alongside the data disagrees with it, - // which it apparently does, all bets are off. - ifdEnumerateLogger.Warningf(nil, "Tag (0x%04x) in IFD [%s] at position (%d) has invalid type (%d) and will be skipped.", ite.tagId, ii, i, ite.tagType) - continue - } - - log.Panic(err) - } - - err = ie.postparseTag(ite, med) - if err == nil { - if err == ErrTagNotFound { - continue - } - - log.PanicIf(err) - } - - tagId := ite.TagId() - - if visitor != nil { - err := visitor(ii.String(), ii.Index(), ite) - log.PanicIf(err) - } - - if ite.IsThumbnailOffset() == true { - ifdEnumerateLogger.Debugf(nil, "Skipping the thumbnail offset tag (0x%04x). Use accessors to get it or set it.", tagId) - - enumeratorThumbnailOffset = ite - entries = append(entries, ite) - - continue - } else if ite.IsThumbnailSize() == true { - ifdEnumerateLogger.Debugf(nil, "Skipping the thumbnail size tag (0x%04x). Use accessors to get it or set it.", tagId) - - enumeratorThumbnailSize = ite - entries = append(entries, ite) - - continue - } - - if ite.TagType() != exifcommon.TypeUndefined { - // If this tag's value is an offset, bump our max-offset value to - // what that offset is plus however large that value is. - - vc := ite.getValueContext() - - farOffset, err := vc.GetFarOffset() - if err == nil { - candidateOffset := farOffset + uint32(vc.SizeInBytes()) - if candidateOffset > ie.furthestOffset { - ie.furthestOffset = candidateOffset - } - } else if err != exifcommon.ErrNotFarValue { - log.PanicIf(err) - } - } - - // If it's an IFD but not a standard one, it'll just be seen as a LONG - // (the standard IFD tag type), later, unless we skip it because it's - // [likely] not even in the standard list of known tags. - if ite.ChildIfdPath() != "" { - if doDescend == true { - ifdEnumerateLogger.Debugf(nil, "Descending from IFD [%s] to IFD [%s].", ii, ite.ChildIfdPath()) - - currentIfdTag := ii.IfdTag() - - childIfdTag := - exifcommon.NewIfdTag( - ¤tIfdTag, - ite.TagId(), - ite.ChildIfdName()) - - iiChild := ii.NewChild(childIfdTag, 0) - - err := ie.scan(iiChild, ite.getValueOffset(), visitor, med) - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Ascending from IFD [%s] to IFD [%s].", ite.ChildIfdPath(), ii) - } - } - - entries = append(entries, ite) - } - - if enumeratorThumbnailOffset != nil && enumeratorThumbnailSize != nil { - thumbnailData, err = ie.parseThumbnail(enumeratorThumbnailOffset, enumeratorThumbnailSize) - log.PanicIf(err) - - // In this case, the value is always an offset. - offset := enumeratorThumbnailOffset.getValueOffset() - - // This this case, the value is always a length. - length := enumeratorThumbnailSize.getValueOffset() - - ifdEnumerateLogger.Debugf(nil, "Found thumbnail in IFD [%s]. Its offset is (%d) and is (%d) bytes.", ii, offset, length) - - furthestOffset := offset + length - - if furthestOffset > ie.furthestOffset { - ie.furthestOffset = furthestOffset - } - } - - nextIfdOffset, _, err = bp.getUint32() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Next IFD at offset: (%08x)", nextIfdOffset) - - return nextIfdOffset, entries, thumbnailData, nil -} - -func (ie *IfdEnumerate) parseThumbnail(offsetIte, lengthIte *IfdTagEntry) (thumbnailData []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - vRaw, err := lengthIte.Value() - log.PanicIf(err) - - vList := vRaw.([]uint32) - if len(vList) != 1 { - log.Panicf("not exactly one long: (%d)", len(vList)) - } - - length := vList[0] - - // The tag is official a LONG type, but it's actually an offset to a blob of bytes. - offsetIte.updateTagType(exifcommon.TypeByte) - offsetIte.updateUnitCount(length) - - thumbnailData, err = offsetIte.GetRawBytes() - log.PanicIf(err) - - return thumbnailData, nil -} - -// scan parses and enumerates the different IFD blocks and invokes a visitor -// callback for each tag. No information is kept or returned. -func (ie *IfdEnumerate) scan(iiGeneral *exifcommon.IfdIdentity, ifdOffset uint32, visitor TagVisitorFn, med *MiscellaneousExifData) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - for ifdIndex := 0; ; ifdIndex++ { - iiSibling := iiGeneral.NewSibling(ifdIndex) - - ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] at offset (0x%04x) (scan).", iiSibling.String(), ifdOffset) - - bp, err := ie.getByteParser(ifdOffset) - if err != nil { - if err == ErrOffsetInvalid { - ifdEnumerateLogger.Errorf(nil, nil, "IFD [%s] at offset (0x%04x) is unreachable. Terminating scan.", iiSibling.String(), ifdOffset) - break - } - - log.Panic(err) - } - - nextIfdOffset, _, _, err := ie.parseIfd(iiSibling, bp, visitor, true, med) - log.PanicIf(err) - - currentOffset := bp.CurrentOffset() - if currentOffset > ie.furthestOffset { - ie.furthestOffset = currentOffset - } - - if nextIfdOffset == 0 { - break - } - - ifdOffset = nextIfdOffset - } - - return nil -} - -// MiscellaneousExifData is reports additional data collected during the parse. -type MiscellaneousExifData struct { - // UnknownTags contains all tags that were invalid for their containing - // IFDs. The values represent alternative IFDs that were correctly matched - // to those tags and used instead. - unknownTags map[exifcommon.BasicTag]exifcommon.BasicTag -} - -// UnknownTags returns the unknown tags encountered during the scan. -func (med *MiscellaneousExifData) UnknownTags() map[exifcommon.BasicTag]exifcommon.BasicTag { - return med.unknownTags -} - -// Scan enumerates the different EXIF blocks (called IFDs). `rootIfdName` will -// be "IFD" in the TIFF standard. -func (ie *IfdEnumerate) Scan(iiRoot *exifcommon.IfdIdentity, ifdOffset uint32, visitor TagVisitorFn) (med *MiscellaneousExifData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - med = &MiscellaneousExifData{ - unknownTags: make(map[exifcommon.BasicTag]exifcommon.BasicTag), - } - - err = ie.scan(iiRoot, ifdOffset, visitor, med) - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Scan: It looks like the furthest offset that contained EXIF data in the EXIF blob was (%d) (Scan).", ie.FurthestOffset()) - - return med, nil -} - -// Ifd represents a single, parsed IFD. -type Ifd struct { - - // TODO(dustin): Add NextIfd(). - - ifdIdentity *exifcommon.IfdIdentity - - ByteOrder binary.ByteOrder - - Id int - - ParentIfd *Ifd - - // ParentTagIndex is our tag position in the parent IFD, if we had a parent - // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling - // instead of as a child). - ParentTagIndex int - - Offset uint32 - - Entries []*IfdTagEntry - EntriesByTagId map[uint16][]*IfdTagEntry - - Children []*Ifd - - ChildIfdIndex map[string]*Ifd - - NextIfdOffset uint32 - NextIfd *Ifd - - thumbnailData []byte - - ifdMapping *exifcommon.IfdMapping - tagIndex *TagIndex -} - -// IfdIdentity returns IFD identity that this struct represents. -func (ifd *Ifd) IfdIdentity() *exifcommon.IfdIdentity { - return ifd.ifdIdentity -} - -// ChildWithIfdPath returns an `Ifd` struct for the given child of the current -// IFD. -func (ifd *Ifd) ChildWithIfdPath(iiChild *exifcommon.IfdIdentity) (childIfd *Ifd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): This is a bridge while we're introducing the IFD type-system. We should be able to use the (IfdIdentity).Equals() method for this. - ifdPath := iiChild.UnindexedString() - - for _, childIfd := range ifd.Children { - if childIfd.ifdIdentity.UnindexedString() == ifdPath { - return childIfd, nil - } - } - - log.Panic(ErrTagNotFound) - return nil, nil -} - -// FindTagWithId returns a list of tags (usually just zero or one) that match -// the given tag ID. This is efficient. -func (ifd *Ifd) FindTagWithId(tagId uint16) (results []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - results, found := ifd.EntriesByTagId[tagId] - if found != true { - log.Panic(ErrTagNotFound) - } - - return results, nil -} - -// FindTagWithName returns a list of tags (usually just zero or one) that match -// the given tag name. This is not efficient (though the labor is trivial). -func (ifd *Ifd) FindTagWithName(tagName string) (results []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - it, err := ifd.tagIndex.GetWithName(ifd.ifdIdentity, tagName) - if log.Is(err, ErrTagNotFound) == true { - log.Panic(ErrTagNotKnown) - } else if err != nil { - log.Panic(err) - } - - results = make([]*IfdTagEntry, 0) - for _, ite := range ifd.Entries { - if ite.TagId() == it.Id { - results = append(results, ite) - } - } - - if len(results) == 0 { - log.Panic(ErrTagNotFound) - } - - return results, nil -} - -// String returns a description string. -func (ifd *Ifd) String() string { - parentOffset := uint32(0) - if ifd.ParentIfd != nil { - parentOffset = ifd.ParentIfd.Offset - } - - return fmt.Sprintf("Ifd", ifd.Id, ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index(), len(ifd.Entries), ifd.Offset, len(ifd.Children), parentOffset, ifd.NextIfdOffset) -} - -// Thumbnail returns the raw thumbnail bytes. This is typically directly -// readable by any standard image viewer. -func (ifd *Ifd) Thumbnail() (data []byte, err error) { - - if ifd.thumbnailData == nil { - return nil, ErrNoThumbnail - } - - return ifd.thumbnailData, nil -} - -// dumpTags recursively builds a list of tags from an IFD. -func (ifd *Ifd) dumpTags(tags []*IfdTagEntry) []*IfdTagEntry { - if tags == nil { - tags = make([]*IfdTagEntry, 0) - } - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, ite := range ifd.Entries { - tags = append(tags, ite) - - childIfdPath := ite.ChildIfdPath() - if childIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[childIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) - } - - tags = childIfd.dumpTags(tags) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - tags = ifd.NextIfd.dumpTags(tags) - } - - return tags -} - -// DumpTags prints the IFD hierarchy. -func (ifd *Ifd) DumpTags() []*IfdTagEntry { - return ifd.dumpTags(nil) -} - -func (ifd *Ifd) printTagTree(populateValues bool, index, level int, nextLink bool) { - indent := strings.Repeat(" ", level*2) - - prefix := " " - if nextLink { - prefix = ">" - } - - fmt.Printf("%s%sIFD: %s\n", indent, prefix, ifd) - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, ite := range ifd.Entries { - if ite.ChildIfdPath() != "" { - fmt.Printf("%s - TAG: %s\n", indent, ite) - } else { - // This will just add noise to the output (byte-tags are fully - // dumped). - if ite.IsThumbnailOffset() == true || ite.IsThumbnailSize() == true { - continue - } - - it, err := ifd.tagIndex.Get(ifd.ifdIdentity, ite.TagId()) - - tagName := "" - if err == nil { - tagName = it.Name - } - - var valuePhrase string - if populateValues == true { - var err error - - valuePhrase, err = ite.Format() - if err != nil { - if log.Is(err, exifcommon.ErrUnhandledUndefinedTypedTag) == true { - ifdEnumerateLogger.Warningf(nil, "Skipping non-standard undefined tag: [%s] (%04x)", ifd.ifdIdentity.UnindexedString(), ite.TagId()) - continue - } else if err == exifundefined.ErrUnparseableValue { - ifdEnumerateLogger.Warningf(nil, "Skipping unparseable undefined tag: [%s] (%04x) [%s]", ifd.ifdIdentity.UnindexedString(), ite.TagId(), it.Name) - continue - } - - log.Panic(err) - } - } else { - valuePhrase = "!UNRESOLVED" - } - - fmt.Printf("%s - TAG: %s NAME=[%s] VALUE=[%v]\n", indent, ite, tagName, valuePhrase) - } - - childIfdPath := ite.ChildIfdPath() - if childIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[childIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) - } - - childIfd.printTagTree(populateValues, 0, level+1, false) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - ifd.NextIfd.printTagTree(populateValues, index+1, level, true) - } -} - -// PrintTagTree prints the IFD hierarchy. -func (ifd *Ifd) PrintTagTree(populateValues bool) { - ifd.printTagTree(populateValues, 0, 0, false) -} - -func (ifd *Ifd) printIfdTree(level int, nextLink bool) { - indent := strings.Repeat(" ", level*2) - - prefix := " " - if nextLink { - prefix = ">" - } - - fmt.Printf("%s%s%s\n", indent, prefix, ifd) - - // Now, print the tags while also descending to child-IFDS as we encounter them. - - ifdsFoundCount := 0 - - for _, ite := range ifd.Entries { - childIfdPath := ite.ChildIfdPath() - if childIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[childIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) - } - - childIfd.printIfdTree(level+1, false) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - if ifd.NextIfd != nil { - ifd.NextIfd.printIfdTree(level, true) - } -} - -// PrintIfdTree prints the IFD hierarchy. -func (ifd *Ifd) PrintIfdTree() { - ifd.printIfdTree(0, false) -} - -func (ifd *Ifd) dumpTree(tagsDump []string, level int) []string { - if tagsDump == nil { - tagsDump = make([]string, 0) - } - - indent := strings.Repeat(" ", level*2) - - var ifdPhrase string - if ifd.ParentIfd != nil { - ifdPhrase = fmt.Sprintf("[%s]->[%s]:(%d)", ifd.ParentIfd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index()) - } else { - ifdPhrase = fmt.Sprintf("[ROOT]->[%s]:(%d)", ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index()) - } - - startBlurb := fmt.Sprintf("%s> IFD %s TOP", indent, ifdPhrase) - tagsDump = append(tagsDump, startBlurb) - - ifdsFoundCount := 0 - for _, ite := range ifd.Entries { - tagsDump = append(tagsDump, fmt.Sprintf("%s - (0x%04x)", indent, ite.TagId())) - - childIfdPath := ite.ChildIfdPath() - if childIfdPath != "" { - ifdsFoundCount++ - - childIfd, found := ifd.ChildIfdIndex[childIfdPath] - if found != true { - log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) - } - - tagsDump = childIfd.dumpTree(tagsDump, level+1) - } - } - - if len(ifd.Children) != ifdsFoundCount { - log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.Children), ifdsFoundCount) - } - - finishBlurb := fmt.Sprintf("%s< IFD %s BOTTOM", indent, ifdPhrase) - tagsDump = append(tagsDump, finishBlurb) - - if ifd.NextIfd != nil { - siblingBlurb := fmt.Sprintf("%s* LINKING TO SIBLING IFD [%s]:(%d)", indent, ifd.NextIfd.ifdIdentity.UnindexedString(), ifd.NextIfd.ifdIdentity.Index()) - tagsDump = append(tagsDump, siblingBlurb) - - tagsDump = ifd.NextIfd.dumpTree(tagsDump, level) - } - - return tagsDump -} - -// DumpTree returns a list of strings describing the IFD hierarchy. -func (ifd *Ifd) DumpTree() []string { - return ifd.dumpTree(nil, 0) -} - -// GpsInfo parses and consolidates the GPS info. This can only be called on the -// GPS IFD. -func (ifd *Ifd) GpsInfo() (gi *GpsInfo, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - gi = new(GpsInfo) - - if ifd.ifdIdentity.UnindexedString() != exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString() { - log.Panicf("GPS can only be read on GPS IFD: [%s] != [%s]", ifd.ifdIdentity.UnindexedString(), exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString()) - } - - if tags, found := ifd.EntriesByTagId[TagGpsVersionId]; found == false { - // We've seen this. We'll just have to default to assuming we're in a - // 2.2.0.0 format. - ifdEnumerateLogger.Warningf(nil, "No GPS version tag (0x%04x) found.", TagGpsVersionId) - } else { - versionBytes, err := tags[0].GetRawBytes() - log.PanicIf(err) - - hit := false - for _, acceptedGpsVersion := range ValidGpsVersions { - if bytes.Compare(versionBytes, acceptedGpsVersion[:]) == 0 { - hit = true - break - } - } - - if hit != true { - ifdEnumerateLogger.Warningf(nil, "GPS version not supported: %v", versionBytes) - log.Panic(ErrNoGpsTags) - } - } - - tags, found := ifd.EntriesByTagId[TagLatitudeId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "latitude not found") - log.Panic(ErrNoGpsTags) - } - - latitudeValue, err := tags[0].Value() - log.PanicIf(err) - - // Look for whether North or South. - tags, found = ifd.EntriesByTagId[TagLatitudeRefId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "latitude-ref not found") - log.Panic(ErrNoGpsTags) - } - - latitudeRefValue, err := tags[0].Value() - log.PanicIf(err) - - tags, found = ifd.EntriesByTagId[TagLongitudeId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "longitude not found") - log.Panic(ErrNoGpsTags) - } - - longitudeValue, err := tags[0].Value() - log.PanicIf(err) - - // Look for whether West or East. - tags, found = ifd.EntriesByTagId[TagLongitudeRefId] - if found == false { - ifdEnumerateLogger.Warningf(nil, "longitude-ref not found") - log.Panic(ErrNoGpsTags) - } - - longitudeRefValue, err := tags[0].Value() - log.PanicIf(err) - - // Parse location. - - latitudeRaw := latitudeValue.([]exifcommon.Rational) - - gi.Latitude, err = NewGpsDegreesFromRationals(latitudeRefValue.(string), latitudeRaw) - log.PanicIf(err) - - longitudeRaw := longitudeValue.([]exifcommon.Rational) - - gi.Longitude, err = NewGpsDegreesFromRationals(longitudeRefValue.(string), longitudeRaw) - log.PanicIf(err) - - // Parse altitude. - - altitudeTags, foundAltitude := ifd.EntriesByTagId[TagAltitudeId] - altitudeRefTags, foundAltitudeRef := ifd.EntriesByTagId[TagAltitudeRefId] - - if foundAltitude == true && foundAltitudeRef == true { - altitudePhrase, err := altitudeTags[0].Format() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Altitude is [%s].", altitudePhrase) - - altitudeValue, err := altitudeTags[0].Value() - log.PanicIf(err) - - altitudeRefPhrase, err := altitudeRefTags[0].Format() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Altitude-reference is [%s].", altitudeRefPhrase) - - altitudeRefValue, err := altitudeRefTags[0].Value() - log.PanicIf(err) - - altitudeRaw := altitudeValue.([]exifcommon.Rational) - if altitudeRaw[0].Denominator > 0 { - altitude := int(altitudeRaw[0].Numerator / altitudeRaw[0].Denominator) - - if altitudeRefValue.([]byte)[0] == 1 { - altitude *= -1 - } - - gi.Altitude = altitude - } - } - - // Parse timestamp from separate date and time tags. - - timestampTags, foundTimestamp := ifd.EntriesByTagId[TagTimestampId] - datestampTags, foundDatestamp := ifd.EntriesByTagId[TagDatestampId] - - if foundTimestamp == true && foundDatestamp == true { - datestampValue, err := datestampTags[0].Value() - log.PanicIf(err) - - datePhrase := datestampValue.(string) - ifdEnumerateLogger.Debugf(nil, "Date tag value is [%s].", datePhrase) - - // Normalize the separators. - datePhrase = strings.ReplaceAll(datePhrase, "-", ":") - - dateParts := strings.Split(datePhrase, ":") - - year, err1 := strconv.ParseUint(dateParts[0], 10, 16) - month, err2 := strconv.ParseUint(dateParts[1], 10, 8) - day, err3 := strconv.ParseUint(dateParts[2], 10, 8) - - if err1 == nil && err2 == nil && err3 == nil { - timestampValue, err := timestampTags[0].Value() - log.PanicIf(err) - - timePhrase, err := timestampTags[0].Format() - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Time tag value is [%s].", timePhrase) - - timestampRaw := timestampValue.([]exifcommon.Rational) - - hour := int(timestampRaw[0].Numerator / timestampRaw[0].Denominator) - minute := int(timestampRaw[1].Numerator / timestampRaw[1].Denominator) - second := int(timestampRaw[2].Numerator / timestampRaw[2].Denominator) - - gi.Timestamp = time.Date(int(year), time.Month(month), int(day), hour, minute, second, 0, time.UTC) - } - } - - return gi, nil -} - -// ParsedTagVisitor is a callback used if wanting to visit through all tags and -// child IFDs from the current IFD and going down. -type ParsedTagVisitor func(*Ifd, *IfdTagEntry) error - -// EnumerateTagsRecursively calls the given visitor function for every tag and -// IFD in the current IFD, recursively. -func (ifd *Ifd) EnumerateTagsRecursively(visitor ParsedTagVisitor) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for ptr := ifd; ptr != nil; ptr = ptr.NextIfd { - for _, ite := range ifd.Entries { - childIfdPath := ite.ChildIfdPath() - if childIfdPath != "" { - childIfd := ifd.ChildIfdIndex[childIfdPath] - - err := childIfd.EnumerateTagsRecursively(visitor) - log.PanicIf(err) - } else { - err := visitor(ifd, ite) - log.PanicIf(err) - } - } - } - - return nil -} - -// QueuedIfd is one IFD that has been identified but yet to be processed. -type QueuedIfd struct { - IfdIdentity *exifcommon.IfdIdentity - - Offset uint32 - Parent *Ifd - - // ParentTagIndex is our tag position in the parent IFD, if we had a parent - // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling - // instead of as a child). - ParentTagIndex int -} - -// IfdIndex collects a bunch of IFD and tag information stored in several -// different ways in order to provide convenient lookups. -type IfdIndex struct { - RootIfd *Ifd - Ifds []*Ifd - Tree map[int]*Ifd - Lookup map[string]*Ifd -} - -// Collect enumerates the different EXIF blocks (called IFDs) and builds out an -// index struct for referencing all of the parsed data. -func (ie *IfdEnumerate) Collect(rootIfdOffset uint32) (index IfdIndex, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add MiscellaneousExifData to IfdIndex - - tree := make(map[int]*Ifd) - ifds := make([]*Ifd, 0) - lookup := make(map[string]*Ifd) - - queue := []QueuedIfd{ - { - IfdIdentity: exifcommon.IfdStandardIfdIdentity, - Offset: rootIfdOffset, - }, - } - - edges := make(map[uint32]*Ifd) - - for { - if len(queue) == 0 { - break - } - - qi := queue[0] - ii := qi.IfdIdentity - - offset := qi.Offset - parentIfd := qi.Parent - - queue = queue[1:] - - ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] (%d) at offset (0x%04x) (Collect).", ii.String(), ii.Index(), offset) - - bp, err := ie.getByteParser(offset) - if err != nil { - if err == ErrOffsetInvalid { - return index, err - } - - log.Panic(err) - } - - // TODO(dustin): We don't need to pass the index in as a separate argument. Get from the II. - - nextIfdOffset, entries, thumbnailData, err := ie.parseIfd(ii, bp, nil, false, nil) - log.PanicIf(err) - - currentOffset := bp.CurrentOffset() - if currentOffset > ie.furthestOffset { - ie.furthestOffset = currentOffset - } - - id := len(ifds) - - entriesByTagId := make(map[uint16][]*IfdTagEntry) - for _, ite := range entries { - tagId := ite.TagId() - - tags, found := entriesByTagId[tagId] - if found == false { - tags = make([]*IfdTagEntry, 0) - } - - entriesByTagId[tagId] = append(tags, ite) - } - - ifd := &Ifd{ - ifdIdentity: ii, - - ByteOrder: ie.byteOrder, - - Id: id, - - ParentIfd: parentIfd, - ParentTagIndex: qi.ParentTagIndex, - - Offset: offset, - Entries: entries, - EntriesByTagId: entriesByTagId, - - // This is populated as each child is processed. - Children: make([]*Ifd, 0), - - NextIfdOffset: nextIfdOffset, - thumbnailData: thumbnailData, - - ifdMapping: ie.ifdMapping, - tagIndex: ie.tagIndex, - } - - // Add ourselves to a big list of IFDs. - ifds = append(ifds, ifd) - - // Install ourselves into a by-id lookup table (keys are unique). - tree[id] = ifd - - // Install into by-name buckets. - lookup[ii.String()] = ifd - - // Add a link from the previous IFD in the chain to us. - if previousIfd, found := edges[offset]; found == true { - previousIfd.NextIfd = ifd - } - - // Attach as a child to our parent (where we appeared as a tag in - // that IFD). - if parentIfd != nil { - parentIfd.Children = append(parentIfd.Children, ifd) - } - - // Determine if any of our entries is a child IFD and queue it. - for i, ite := range entries { - if ite.ChildIfdPath() == "" { - continue - } - - tagId := ite.TagId() - childIfdName := ite.ChildIfdName() - - currentIfdTag := ii.IfdTag() - - childIfdTag := - exifcommon.NewIfdTag( - ¤tIfdTag, - tagId, - childIfdName) - - iiChild := ii.NewChild(childIfdTag, 0) - - qi := QueuedIfd{ - IfdIdentity: iiChild, - - Offset: ite.getValueOffset(), - Parent: ifd, - ParentTagIndex: i, - } - - queue = append(queue, qi) - } - - // If there's another IFD in the chain. - if nextIfdOffset != 0 { - iiSibling := ii.NewSibling(ii.Index() + 1) - - // Allow the next link to know what the previous link was. - edges[nextIfdOffset] = ifd - - qi := QueuedIfd{ - IfdIdentity: iiSibling, - Offset: nextIfdOffset, - } - - queue = append(queue, qi) - } - } - - index.RootIfd = tree[0] - index.Ifds = ifds - index.Tree = tree - index.Lookup = lookup - - err = ie.setChildrenIndex(index.RootIfd) - log.PanicIf(err) - - ifdEnumerateLogger.Debugf(nil, "Collect: It looks like the furthest offset that contained EXIF data in the EXIF blob was (%d).", ie.FurthestOffset()) - - return index, nil -} - -func (ie *IfdEnumerate) setChildrenIndex(ifd *Ifd) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - childIfdIndex := make(map[string]*Ifd) - for _, childIfd := range ifd.Children { - childIfdIndex[childIfd.ifdIdentity.UnindexedString()] = childIfd - } - - ifd.ChildIfdIndex = childIfdIndex - - for _, childIfd := range ifd.Children { - err := ie.setChildrenIndex(childIfd) - log.PanicIf(err) - } - - return nil -} - -// FurthestOffset returns the furthest offset visited in the EXIF blob. This -// *does not* account for the locations of any undefined tags since we always -// evaluate the furthest offset, whether or not the user wants to know it. -// -// We are not willing to incur the cost of actually parsing those tags just to -// know their length when there are still undefined tags that are out there -// that we still won't have any idea how to parse, thus making this an -// approximation regardless of how clever we get. -func (ie *IfdEnumerate) FurthestOffset() uint32 { - - // TODO(dustin): Add test - - return ie.furthestOffset -} - -// ParseOneIfd is a hack to use an IE to parse a raw IFD block. Can be used for -// testing. The fqIfdPath ("fully-qualified IFD path") will be less qualified -// in that the numeric index will always be zero (the zeroth child) rather than -// the proper number (if its actually a sibling to the first child, for -// instance). -func ParseOneIfd(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder, ifdBlock []byte, visitor TagVisitorFn) (nextIfdOffset uint32, entries []*IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ie := NewIfdEnumerate(ifdMapping, tagIndex, make([]byte, 0), byteOrder) - - bp, err := newByteParser(ifdBlock, byteOrder, 0) - if err != nil { - if err == ErrOffsetInvalid { - return 0, nil, err - } - - log.Panic(err) - } - - nextIfdOffset, entries, _, err = ie.parseIfd(ii, bp, visitor, true, nil) - log.PanicIf(err) - - return nextIfdOffset, entries, nil -} - -// ParseOneTag is a hack to use an IE to parse a raw tag block. -func ParseOneTag(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder, tagBlock []byte) (ite *IfdTagEntry, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ie := NewIfdEnumerate(ifdMapping, tagIndex, make([]byte, 0), byteOrder) - - bp, err := newByteParser(tagBlock, byteOrder, 0) - if err != nil { - if err == ErrOffsetInvalid { - return nil, err - } - - log.Panic(err) - } - - ite, err = ie.parseTag(ii, 0, bp) - log.PanicIf(err) - - err = ie.postparseTag(ite, nil) - if err != nil { - if err == ErrTagNotFound { - return nil, err - } - - log.Panic(err) - } - - return ite, nil -} - -// FindIfdFromRootIfd returns the given `Ifd` given the root-IFD and path of the -// desired IFD. -func FindIfdFromRootIfd(rootIfd *Ifd, ifdPath string) (ifd *Ifd, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): !! Add test. - - lineage, err := rootIfd.ifdMapping.ResolvePath(ifdPath) - log.PanicIf(err) - - // Confirm the first IFD is our root IFD type, and then prune it because - // from then on we'll be searching down through our children. - - if len(lineage) == 0 { - log.Panicf("IFD path must be non-empty.") - } else if lineage[0].Name != exifcommon.IfdStandardIfdIdentity.Name() { - log.Panicf("First IFD path item must be [%s].", exifcommon.IfdStandardIfdIdentity.Name()) - } - - desiredRootIndex := lineage[0].Index - lineage = lineage[1:] - - // TODO(dustin): !! This is a poorly conceived fix that just doubles the work we already have to do below, which then interacts badly with the indices not being properly represented in the IFD-phrase. - // TODO(dustin): !! <-- However, we're not sure whether we shouldn't store a secondary IFD-path with the indices. Some IFDs may not necessarily restrict which IFD indices they can be a child of (only the IFD itself matters). Validation should be delegated to the caller. - thisIfd := rootIfd - for currentRootIndex := 0; currentRootIndex < desiredRootIndex; currentRootIndex++ { - if thisIfd.NextIfd == nil { - log.Panicf("Root-IFD index (%d) does not exist in the data.", currentRootIndex) - } - - thisIfd = thisIfd.NextIfd - } - - for _, itii := range lineage { - var hit *Ifd - for _, childIfd := range thisIfd.Children { - if childIfd.ifdIdentity.TagId() == itii.TagId { - hit = childIfd - break - } - } - - // If we didn't find the child, add it. - if hit == nil { - log.Panicf("IFD [%s] in [%s] not found: %s", itii.Name, ifdPath, thisIfd.Children) - } - - thisIfd = hit - - // If we didn't find the sibling, add it. - for i := 0; i < itii.Index; i++ { - if thisIfd.NextIfd == nil { - log.Panicf("IFD [%s] does not have (%d) occurrences/siblings", thisIfd.ifdIdentity.UnindexedString(), itii.Index) - } - - thisIfd = thisIfd.NextIfd - } - } - - return thisIfd, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/ifd_tag_entry.go b/vendor/github.com/dsoprea/go-exif/v2/ifd_tag_entry.go deleted file mode 100644 index 789a9981c..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/ifd_tag_entry.go +++ /dev/null @@ -1,297 +0,0 @@ -package exif - -import ( - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" - "github.com/dsoprea/go-exif/v2/undefined" -) - -var ( - iteLogger = log.NewLogger("exif.ifd_tag_entry") -) - -// IfdTagEntry refers to a tag in the loaded EXIF block. -type IfdTagEntry struct { - tagId uint16 - tagIndex int - tagType exifcommon.TagTypePrimitive - unitCount uint32 - valueOffset uint32 - rawValueOffset []byte - - // childIfdName is the right most atom in the IFD-path. We need this to - // construct the fully-qualified IFD-path. - childIfdName string - - // childIfdPath is the IFD-path of the child if this tag represents a child - // IFD. - childIfdPath string - - // childFqIfdPath is the IFD-path of the child if this tag represents a - // child IFD. Includes indices. - childFqIfdPath string - - // TODO(dustin): !! IB's host the child-IBs directly in the tag, but that's not the case here. Refactor to accommodate it for a consistent experience. - - ifdIdentity *exifcommon.IfdIdentity - - isUnhandledUnknown bool - - addressableData []byte - byteOrder binary.ByteOrder - - tagName string -} - -func newIfdTagEntry(ii *exifcommon.IfdIdentity, tagId uint16, tagIndex int, tagType exifcommon.TagTypePrimitive, unitCount uint32, valueOffset uint32, rawValueOffset []byte, addressableData []byte, byteOrder binary.ByteOrder) *IfdTagEntry { - return &IfdTagEntry{ - ifdIdentity: ii, - tagId: tagId, - tagIndex: tagIndex, - tagType: tagType, - unitCount: unitCount, - valueOffset: valueOffset, - rawValueOffset: rawValueOffset, - addressableData: addressableData, - byteOrder: byteOrder, - } -} - -// String returns a stringified representation of the struct. -func (ite *IfdTagEntry) String() string { - return fmt.Sprintf("IfdTagEntry", ite.ifdIdentity.String(), ite.tagId, ite.tagType.String(), ite.unitCount) -} - -// TagName returns the name of the tag. This is determined else and set after -// the parse (since it's not actually stored in the stream). If it's empty, it -// is because it is an unknown tag (nonstandard or otherwise unavailable in the -// tag-index). -func (ite *IfdTagEntry) TagName() string { - return ite.tagName -} - -// setTagName sets the tag-name. This provides the name for convenience and -// efficiency by determining it when most efficient while we're parsing rather -// than delegating it to the caller (or, worse, the user). -func (ite *IfdTagEntry) setTagName(tagName string) { - ite.tagName = tagName -} - -// IfdPath returns the fully-qualified path of the IFD that owns this tag. -func (ite *IfdTagEntry) IfdPath() string { - return ite.ifdIdentity.String() -} - -// TagId returns the ID of the tag that we represent. The combination of -// (IfdPath(), TagId()) is unique. -func (ite *IfdTagEntry) TagId() uint16 { - return ite.tagId -} - -// IsThumbnailOffset returns true if the tag has the IFD and tag-ID of a -// thumbnail offset. -func (ite *IfdTagEntry) IsThumbnailOffset() bool { - return ite.tagId == ThumbnailOffsetTagId && ite.ifdIdentity.String() == ThumbnailFqIfdPath -} - -// IsThumbnailSize returns true if the tag has the IFD and tag-ID of a thumbnail -// size. -func (ite *IfdTagEntry) IsThumbnailSize() bool { - return ite.tagId == ThumbnailSizeTagId && ite.ifdIdentity.String() == ThumbnailFqIfdPath -} - -// TagType is the type of value for this tag. -func (ite *IfdTagEntry) TagType() exifcommon.TagTypePrimitive { - return ite.tagType -} - -// updateTagType sets an alternatively interpreted tag-type. -func (ite *IfdTagEntry) updateTagType(tagType exifcommon.TagTypePrimitive) { - ite.tagType = tagType -} - -// UnitCount returns the unit-count of the tag's value. -func (ite *IfdTagEntry) UnitCount() uint32 { - return ite.unitCount -} - -// updateUnitCount sets an alternatively interpreted unit-count. -func (ite *IfdTagEntry) updateUnitCount(unitCount uint32) { - ite.unitCount = unitCount -} - -// getValueOffset is the four-byte offset converted to an integer to point to -// the location of its value in the EXIF block. The "get" parameter is obviously -// used in order to differentiate the naming of the method from the field. -func (ite *IfdTagEntry) getValueOffset() uint32 { - return ite.valueOffset -} - -// GetRawBytes renders a specific list of bytes from the value in this tag. -func (ite *IfdTagEntry) GetRawBytes() (rawBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext := ite.getValueContext() - - if ite.tagType == exifcommon.TypeUndefined { - value, err := exifundefined.Decode(valueContext) - if err != nil { - if err == exifcommon.ErrUnhandledUndefinedTypedTag { - ite.setIsUnhandledUnknown(true) - return nil, exifundefined.ErrUnparseableValue - } else if err == exifundefined.ErrUnparseableValue { - return nil, err - } else { - log.Panic(err) - } - } - - // Encode it back, in order to get the raw bytes. This is the best, - // general way to do it with an undefined tag. - - rawBytes, _, err := exifundefined.Encode(value, ite.byteOrder) - log.PanicIf(err) - - return rawBytes, nil - } - - rawBytes, err = valueContext.ReadRawEncoded() - log.PanicIf(err) - - return rawBytes, nil -} - -// Value returns the specific, parsed, typed value from the tag. -func (ite *IfdTagEntry) Value() (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext := ite.getValueContext() - - if ite.tagType == exifcommon.TypeUndefined { - var err error - - value, err = exifundefined.Decode(valueContext) - if err != nil { - if err == exifcommon.ErrUnhandledUndefinedTypedTag || err == exifundefined.ErrUnparseableValue { - return nil, err - } - - log.Panic(err) - } - } else { - var err error - - value, err = valueContext.Values() - log.PanicIf(err) - } - - return value, nil -} - -// Format returns the tag's value as a string. -func (ite *IfdTagEntry) Format() (phrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - value, err := ite.Value() - if err != nil { - if err == exifcommon.ErrUnhandledUndefinedTypedTag { - return exifundefined.UnparseableUnknownTagValuePlaceholder, nil - } else if err == exifundefined.ErrUnparseableValue { - return exifundefined.UnparseableHandledTagValuePlaceholder, nil - } - - log.Panic(err) - } - - phrase, err = exifcommon.FormatFromType(value, false) - log.PanicIf(err) - - return phrase, nil -} - -// FormatFirst returns the same as Format() but only the first item. -func (ite *IfdTagEntry) FormatFirst() (phrase string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): We should add a convenience type "timestamp", to simplify translating to and from the physical ASCII and provide validation. - - value, err := ite.Value() - if err != nil { - if err == exifcommon.ErrUnhandledUndefinedTypedTag { - return exifundefined.UnparseableUnknownTagValuePlaceholder, nil - } - - log.Panic(err) - } - - phrase, err = exifcommon.FormatFromType(value, true) - log.PanicIf(err) - - return phrase, nil -} - -func (ite *IfdTagEntry) setIsUnhandledUnknown(isUnhandledUnknown bool) { - ite.isUnhandledUnknown = isUnhandledUnknown -} - -// SetChildIfd sets child-IFD information (if we represent a child IFD). -func (ite *IfdTagEntry) SetChildIfd(ii *exifcommon.IfdIdentity) { - ite.childFqIfdPath = ii.String() - ite.childIfdPath = ii.UnindexedString() - ite.childIfdName = ii.Name() -} - -// ChildIfdName returns the name of the child IFD -func (ite *IfdTagEntry) ChildIfdName() string { - return ite.childIfdName -} - -// ChildIfdPath returns the path of the child IFD. -func (ite *IfdTagEntry) ChildIfdPath() string { - return ite.childIfdPath -} - -// ChildFqIfdPath returns the complete path of the child IFD along with the -// numeric suffixes differentiating sibling occurrences of the same type. "0" -// indices are omitted. -func (ite *IfdTagEntry) ChildFqIfdPath() string { - return ite.childFqIfdPath -} - -// IfdIdentity returns the IfdIdentity associated with this tag. -func (ite *IfdTagEntry) IfdIdentity() *exifcommon.IfdIdentity { - return ite.ifdIdentity -} - -func (ite *IfdTagEntry) getValueContext() *exifcommon.ValueContext { - return exifcommon.NewValueContext( - ite.ifdIdentity.String(), - ite.tagId, - ite.unitCount, - ite.valueOffset, - ite.rawValueOffset, - ite.addressableData, - ite.tagType, - ite.byteOrder) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/package.go b/vendor/github.com/dsoprea/go-exif/v2/package.go deleted file mode 100644 index 428f74e3a..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/package.go +++ /dev/null @@ -1,8 +0,0 @@ -// Package exif parses raw EXIF information given a block of raw EXIF data. It -// can also construct new EXIF information, and provides tools for doing so. -// This package is not involved with the parsing of particular file-formats. -// -// The EXIF data must first be extracted and then provided to us. Conversely, -// when constructing new EXIF data, the caller is responsible for packaging -// this in whichever format they require. -package exif diff --git a/vendor/github.com/dsoprea/go-exif/v2/tags.go b/vendor/github.com/dsoprea/go-exif/v2/tags.go deleted file mode 100644 index f53d9ce9c..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/tags.go +++ /dev/null @@ -1,411 +0,0 @@ -package exif - -import ( - "fmt" - - "github.com/dsoprea/go-logging" - "gopkg.in/yaml.v2" - - "github.com/dsoprea/go-exif/v2/common" -) - -const ( - // IFD1 - - // ThumbnailFqIfdPath is the fully-qualified IFD path that the thumbnail - // must be found in. - ThumbnailFqIfdPath = "IFD1" - - // ThumbnailOffsetTagId returns the tag-ID of the thumbnail offset. - ThumbnailOffsetTagId = 0x0201 - - // ThumbnailSizeTagId returns the tag-ID of the thumbnail size. - ThumbnailSizeTagId = 0x0202 -) - -const ( - // GPS - - // TagGpsVersionId is the ID of the GPS version tag. - TagGpsVersionId = 0x0000 - - // TagLatitudeId is the ID of the GPS latitude tag. - TagLatitudeId = 0x0002 - - // TagLatitudeRefId is the ID of the GPS latitude orientation tag. - TagLatitudeRefId = 0x0001 - - // TagLongitudeId is the ID of the GPS longitude tag. - TagLongitudeId = 0x0004 - - // TagLongitudeRefId is the ID of the GPS longitude-orientation tag. - TagLongitudeRefId = 0x0003 - - // TagTimestampId is the ID of the GPS time tag. - TagTimestampId = 0x0007 - - // TagDatestampId is the ID of the GPS date tag. - TagDatestampId = 0x001d - - // TagAltitudeId is the ID of the GPS altitude tag. - TagAltitudeId = 0x0006 - - // TagAltitudeRefId is the ID of the GPS altitude-orientation tag. - TagAltitudeRefId = 0x0005 -) - -var ( - // tagsWithoutAlignment is a tag-lookup for tags whose value size won't - // necessarily be a multiple of its tag-type. - tagsWithoutAlignment = map[uint16]struct{}{ - // The thumbnail offset is stored as a long, but its data is a binary - // blob (not a slice of longs). - ThumbnailOffsetTagId: {}, - } -) - -var ( - tagsLogger = log.NewLogger("exif.tags") -) - -// File structures. - -type encodedTag struct { - // id is signed, here, because YAML doesn't have enough information to - // support unsigned. - Id int `yaml:"id"` - Name string `yaml:"name"` - TypeName string `yaml:"type_name"` - TypeNames []string `yaml:"type_names"` -} - -// Indexing structures. - -// IndexedTag describes one index lookup result. -type IndexedTag struct { - // Id is the tag-ID. - Id uint16 - - // Name is the tag name. - Name string - - // IfdPath is the proper IFD path of this tag. This is not fully-qualified. - IfdPath string - - // SupportedTypes is an unsorted list of allowed tag-types. - SupportedTypes []exifcommon.TagTypePrimitive -} - -// String returns a descriptive string. -func (it *IndexedTag) String() string { - return fmt.Sprintf("TAG", it.Id, it.Name, it.IfdPath) -} - -// IsName returns true if this tag matches the given tag name. -func (it *IndexedTag) IsName(ifdPath, name string) bool { - return it.Name == name && it.IfdPath == ifdPath -} - -// Is returns true if this tag matched the given tag ID. -func (it *IndexedTag) Is(ifdPath string, id uint16) bool { - return it.Id == id && it.IfdPath == ifdPath -} - -// GetEncodingType returns the largest type that this tag's value can occupy. -func (it *IndexedTag) GetEncodingType(value interface{}) exifcommon.TagTypePrimitive { - // For convenience, we handle encoding a `time.Time` directly. - if IsTime(value) == true { - // Timestamps are encoded as ASCII. - value = "" - } - - if len(it.SupportedTypes) == 0 { - log.Panicf("IndexedTag [%s] (%d) has no supported types.", it.IfdPath, it.Id) - } else if len(it.SupportedTypes) == 1 { - return it.SupportedTypes[0] - } - - supportsLong := false - supportsShort := false - supportsRational := false - supportsSignedRational := false - for _, supportedType := range it.SupportedTypes { - if supportedType == exifcommon.TypeLong { - supportsLong = true - } else if supportedType == exifcommon.TypeShort { - supportsShort = true - } else if supportedType == exifcommon.TypeRational { - supportsRational = true - } else if supportedType == exifcommon.TypeSignedRational { - supportsSignedRational = true - } - } - - // We specifically check for the cases that we know to expect. - - if supportsLong == true && supportsShort == true { - return exifcommon.TypeLong - } else if supportsRational == true && supportsSignedRational == true { - if value == nil { - log.Panicf("GetEncodingType: require value to be given") - } - - if _, ok := value.(exifcommon.SignedRational); ok == true { - return exifcommon.TypeSignedRational - } - - return exifcommon.TypeRational - } - - log.Panicf("WidestSupportedType() case is not handled for tag [%s] (0x%04x): %v", it.IfdPath, it.Id, it.SupportedTypes) - return 0 -} - -// DoesSupportType returns true if this tag can be found/decoded with this type. -func (it *IndexedTag) DoesSupportType(tagType exifcommon.TagTypePrimitive) bool { - // This is always a very small collection. So, we keep it unsorted. - for _, thisTagType := range it.SupportedTypes { - if thisTagType == tagType { - return true - } - } - - return false -} - -// TagIndex is a tag-lookup facility. -type TagIndex struct { - tagsByIfd map[string]map[uint16]*IndexedTag - tagsByIfdR map[string]map[string]*IndexedTag -} - -// NewTagIndex returns a new TagIndex struct. -func NewTagIndex() *TagIndex { - ti := new(TagIndex) - - ti.tagsByIfd = make(map[string]map[uint16]*IndexedTag) - ti.tagsByIfdR = make(map[string]map[string]*IndexedTag) - - return ti -} - -// Add registers a new tag to be recognized during the parse. -func (ti *TagIndex) Add(it *IndexedTag) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Store by ID. - - family, found := ti.tagsByIfd[it.IfdPath] - if found == false { - family = make(map[uint16]*IndexedTag) - ti.tagsByIfd[it.IfdPath] = family - } - - if _, found := family[it.Id]; found == true { - log.Panicf("tag-ID defined more than once for IFD [%s]: (%02x)", it.IfdPath, it.Id) - } - - family[it.Id] = it - - // Store by name. - - familyR, found := ti.tagsByIfdR[it.IfdPath] - if found == false { - familyR = make(map[string]*IndexedTag) - ti.tagsByIfdR[it.IfdPath] = familyR - } - - if _, found := familyR[it.Name]; found == true { - log.Panicf("tag-name defined more than once for IFD [%s]: (%s)", it.IfdPath, it.Name) - } - - familyR[it.Name] = it - - return nil -} - -// Get returns information about the non-IFD tag given a tag ID. `ifdPath` must -// not be fully-qualified. -func (ti *TagIndex) Get(ii *exifcommon.IfdIdentity, id uint16) (it *IndexedTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(ti.tagsByIfd) == 0 { - err := LoadStandardTags(ti) - log.PanicIf(err) - } - - ifdPath := ii.UnindexedString() - - family, found := ti.tagsByIfd[ifdPath] - if found == false { - return nil, ErrTagNotFound - } - - it, found = family[id] - if found == false { - return nil, ErrTagNotFound - } - - return it, nil -} - -var ( - // tagGuessDefaultIfdIdentities describes which IFDs we'll look for a given - // tag-ID in, if it's not found where it's supposed to be. We suppose that - // Exif-IFD tags might be found in IFD0 or IFD1, or IFD0/IFD1 tags might be - // found in the Exif IFD. This is the only thing we've seen so far. So, this - // is the limit of our guessing. - tagGuessDefaultIfdIdentities = []*exifcommon.IfdIdentity{ - exifcommon.IfdExifStandardIfdIdentity, - exifcommon.IfdStandardIfdIdentity, - } -) - -// FindFirst looks for the given tag-ID in each of the given IFDs in the given -// order. If `fqIfdPaths` is `nil` then use a default search order. This defies -// the standard, which requires each tag to exist in certain IFDs. This is a -// contingency to make recommendations for malformed data. -// -// Things *can* end badly here, in that the same tag-ID in different IFDs might -// describe different data and different ata-types, and our decode might then -// produce binary and non-printable data. -func (ti *TagIndex) FindFirst(id uint16, typeId exifcommon.TagTypePrimitive, ifdIdentities []*exifcommon.IfdIdentity) (it *IndexedTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if ifdIdentities == nil { - ifdIdentities = tagGuessDefaultIfdIdentities - } - - for _, ii := range ifdIdentities { - it, err := ti.Get(ii, id) - if err != nil { - if err == ErrTagNotFound { - continue - } - - log.Panic(err) - } - - // Even though the tag might be mislocated, the type should still be the - // same. Check this so we don't accidentally end-up on a complete - // irrelevant tag with a totally different data type. This attempts to - // mitigate producing garbage. - for _, supportedType := range it.SupportedTypes { - if supportedType == typeId { - return it, nil - } - } - } - - return nil, ErrTagNotFound -} - -// GetWithName returns information about the non-IFD tag given a tag name. -func (ti *TagIndex) GetWithName(ii *exifcommon.IfdIdentity, name string) (it *IndexedTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(ti.tagsByIfdR) == 0 { - err := LoadStandardTags(ti) - log.PanicIf(err) - } - - ifdPath := ii.UnindexedString() - - it, found := ti.tagsByIfdR[ifdPath][name] - if found != true { - log.Panic(ErrTagNotFound) - } - - return it, nil -} - -// LoadStandardTags registers the tags that all devices/applications should -// support. -func LoadStandardTags(ti *TagIndex) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Read static data. - - encodedIfds := make(map[string][]encodedTag) - - err = yaml.Unmarshal([]byte(tagsYaml), encodedIfds) - log.PanicIf(err) - - // Load structure. - - count := 0 - for ifdPath, tags := range encodedIfds { - for _, tagInfo := range tags { - tagId := uint16(tagInfo.Id) - tagName := tagInfo.Name - tagTypeName := tagInfo.TypeName - tagTypeNames := tagInfo.TypeNames - - if tagTypeNames == nil { - if tagTypeName == "" { - log.Panicf("no tag-types were given when registering standard tag [%s] (0x%04x) [%s]", ifdPath, tagId, tagName) - } - - tagTypeNames = []string{ - tagTypeName, - } - } else if tagTypeName != "" { - log.Panicf("both 'type_names' and 'type_name' were given when registering standard tag [%s] (0x%04x) [%s]", ifdPath, tagId, tagName) - } - - tagTypes := make([]exifcommon.TagTypePrimitive, 0) - for _, tagTypeName := range tagTypeNames { - - // TODO(dustin): Discard unsupported types. This helps us with non-standard types that have actually been found in real data, that we ignore for right now. e.g. SSHORT, FLOAT, DOUBLE - tagTypeId, found := exifcommon.GetTypeByName(tagTypeName) - if found == false { - tagsLogger.Warningf(nil, "Type [%s] for tag [%s] being loaded is not valid and is being ignored.", tagTypeName, tagName) - continue - } - - tagTypes = append(tagTypes, tagTypeId) - } - - if len(tagTypes) == 0 { - tagsLogger.Warningf(nil, "Tag [%s] (0x%04x) [%s] being loaded does not have any supported types and will not be registered.", ifdPath, tagId, tagName) - continue - } - - it := &IndexedTag{ - IfdPath: ifdPath, - Id: tagId, - Name: tagName, - SupportedTypes: tagTypes, - } - - err = ti.Add(it) - log.PanicIf(err) - - count++ - } - } - - tagsLogger.Debugf(nil, "(%d) tags loaded.", count) - - return nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/tags_data.go b/vendor/github.com/dsoprea/go-exif/v2/tags_data.go deleted file mode 100644 index a770e5597..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/tags_data.go +++ /dev/null @@ -1,929 +0,0 @@ -package exif - -var ( - // From assets/tags.yaml . Needs to be here so it's embedded in the binary. - tagsYaml = ` -# Notes: -# -# This file was produced from http://www.exiv2.org/tags.html, using the included -# tool, though that document appears to have some duplicates when all IDs are -# supposed to be unique (EXIF information only has IDs, not IFDs; IFDs are -# determined by our pre-existing knowledge of those tags). -# -# The webpage that we've produced this file from appears to indicate that -# ImageWidth is represented by both 0x0100 and 0x0001 depending on whether the -# encoding is RGB or YCbCr. -IFD/Exif: -- id: 0x829a - name: ExposureTime - type_name: RATIONAL -- id: 0x829d - name: FNumber - type_name: RATIONAL -- id: 0x8822 - name: ExposureProgram - type_name: SHORT -- id: 0x8824 - name: SpectralSensitivity - type_name: ASCII -- id: 0x8827 - name: ISOSpeedRatings - type_name: SHORT -- id: 0x8828 - name: OECF - type_name: UNDEFINED -- id: 0x8830 - name: SensitivityType - type_name: SHORT -- id: 0x8831 - name: StandardOutputSensitivity - type_name: LONG -- id: 0x8832 - name: RecommendedExposureIndex - type_name: LONG -- id: 0x8833 - name: ISOSpeed - type_name: LONG -- id: 0x8834 - name: ISOSpeedLatitudeyyy - type_name: LONG -- id: 0x8835 - name: ISOSpeedLatitudezzz - type_name: LONG -- id: 0x9000 - name: ExifVersion - type_name: UNDEFINED -- id: 0x9003 - name: DateTimeOriginal - type_name: ASCII -- id: 0x9004 - name: DateTimeDigitized - type_name: ASCII -- id: 0x9101 - name: ComponentsConfiguration - type_name: UNDEFINED -- id: 0x9102 - name: CompressedBitsPerPixel - type_name: RATIONAL -- id: 0x9201 - name: ShutterSpeedValue - type_name: SRATIONAL -- id: 0x9202 - name: ApertureValue - type_name: RATIONAL -- id: 0x9203 - name: BrightnessValue - type_name: SRATIONAL -- id: 0x9204 - name: ExposureBiasValue - type_name: SRATIONAL -- id: 0x9205 - name: MaxApertureValue - type_name: RATIONAL -- id: 0x9206 - name: SubjectDistance - type_name: RATIONAL -- id: 0x9207 - name: MeteringMode - type_name: SHORT -- id: 0x9208 - name: LightSource - type_name: SHORT -- id: 0x9209 - name: Flash - type_name: SHORT -- id: 0x920a - name: FocalLength - type_name: RATIONAL -- id: 0x9214 - name: SubjectArea - type_name: SHORT -- id: 0x927c - name: MakerNote - type_name: UNDEFINED -- id: 0x9286 - name: UserComment - type_name: UNDEFINED -- id: 0x9290 - name: SubSecTime - type_name: ASCII -- id: 0x9291 - name: SubSecTimeOriginal - type_name: ASCII -- id: 0x9292 - name: SubSecTimeDigitized - type_name: ASCII -- id: 0xa000 - name: FlashpixVersion - type_name: UNDEFINED -- id: 0xa001 - name: ColorSpace - type_name: SHORT -- id: 0xa002 - name: PixelXDimension - type_names: [LONG, SHORT] -- id: 0xa003 - name: PixelYDimension - type_names: [LONG, SHORT] -- id: 0xa004 - name: RelatedSoundFile - type_name: ASCII -- id: 0xa005 - name: InteroperabilityTag - type_name: LONG -- id: 0xa20b - name: FlashEnergy - type_name: RATIONAL -- id: 0xa20c - name: SpatialFrequencyResponse - type_name: UNDEFINED -- id: 0xa20e - name: FocalPlaneXResolution - type_name: RATIONAL -- id: 0xa20f - name: FocalPlaneYResolution - type_name: RATIONAL -- id: 0xa210 - name: FocalPlaneResolutionUnit - type_name: SHORT -- id: 0xa214 - name: SubjectLocation - type_name: SHORT -- id: 0xa215 - name: ExposureIndex - type_name: RATIONAL -- id: 0xa217 - name: SensingMethod - type_name: SHORT -- id: 0xa300 - name: FileSource - type_name: UNDEFINED -- id: 0xa301 - name: SceneType - type_name: UNDEFINED -- id: 0xa302 - name: CFAPattern - type_name: UNDEFINED -- id: 0xa401 - name: CustomRendered - type_name: SHORT -- id: 0xa402 - name: ExposureMode - type_name: SHORT -- id: 0xa403 - name: WhiteBalance - type_name: SHORT -- id: 0xa404 - name: DigitalZoomRatio - type_name: RATIONAL -- id: 0xa405 - name: FocalLengthIn35mmFilm - type_name: SHORT -- id: 0xa406 - name: SceneCaptureType - type_name: SHORT -- id: 0xa407 - name: GainControl - type_name: SHORT -- id: 0xa408 - name: Contrast - type_name: SHORT -- id: 0xa409 - name: Saturation - type_name: SHORT -- id: 0xa40a - name: Sharpness - type_name: SHORT -- id: 0xa40b - name: DeviceSettingDescription - type_name: UNDEFINED -- id: 0xa40c - name: SubjectDistanceRange - type_name: SHORT -- id: 0xa420 - name: ImageUniqueID - type_name: ASCII -- id: 0xa430 - name: CameraOwnerName - type_name: ASCII -- id: 0xa431 - name: BodySerialNumber - type_name: ASCII -- id: 0xa432 - name: LensSpecification - type_name: RATIONAL -- id: 0xa433 - name: LensMake - type_name: ASCII -- id: 0xa434 - name: LensModel - type_name: ASCII -- id: 0xa435 - name: LensSerialNumber - type_name: ASCII -IFD/GPSInfo: -- id: 0x0000 - name: GPSVersionID - type_name: BYTE -- id: 0x0001 - name: GPSLatitudeRef - type_name: ASCII -- id: 0x0002 - name: GPSLatitude - type_name: RATIONAL -- id: 0x0003 - name: GPSLongitudeRef - type_name: ASCII -- id: 0x0004 - name: GPSLongitude - type_name: RATIONAL -- id: 0x0005 - name: GPSAltitudeRef - type_name: BYTE -- id: 0x0006 - name: GPSAltitude - type_name: RATIONAL -- id: 0x0007 - name: GPSTimeStamp - type_name: RATIONAL -- id: 0x0008 - name: GPSSatellites - type_name: ASCII -- id: 0x0009 - name: GPSStatus - type_name: ASCII -- id: 0x000a - name: GPSMeasureMode - type_name: ASCII -- id: 0x000b - name: GPSDOP - type_name: RATIONAL -- id: 0x000c - name: GPSSpeedRef - type_name: ASCII -- id: 0x000d - name: GPSSpeed - type_name: RATIONAL -- id: 0x000e - name: GPSTrackRef - type_name: ASCII -- id: 0x000f - name: GPSTrack - type_name: RATIONAL -- id: 0x0010 - name: GPSImgDirectionRef - type_name: ASCII -- id: 0x0011 - name: GPSImgDirection - type_name: RATIONAL -- id: 0x0012 - name: GPSMapDatum - type_name: ASCII -- id: 0x0013 - name: GPSDestLatitudeRef - type_name: ASCII -- id: 0x0014 - name: GPSDestLatitude - type_name: RATIONAL -- id: 0x0015 - name: GPSDestLongitudeRef - type_name: ASCII -- id: 0x0016 - name: GPSDestLongitude - type_name: RATIONAL -- id: 0x0017 - name: GPSDestBearingRef - type_name: ASCII -- id: 0x0018 - name: GPSDestBearing - type_name: RATIONAL -- id: 0x0019 - name: GPSDestDistanceRef - type_name: ASCII -- id: 0x001a - name: GPSDestDistance - type_name: RATIONAL -- id: 0x001b - name: GPSProcessingMethod - type_name: UNDEFINED -- id: 0x001c - name: GPSAreaInformation - type_name: UNDEFINED -- id: 0x001d - name: GPSDateStamp - type_name: ASCII -- id: 0x001e - name: GPSDifferential - type_name: SHORT -IFD: -- id: 0x000b - name: ProcessingSoftware - type_name: ASCII -- id: 0x00fe - name: NewSubfileType - type_name: LONG -- id: 0x00ff - name: SubfileType - type_name: SHORT -- id: 0x0100 - name: ImageWidth - type_names: [LONG, SHORT] -- id: 0x0101 - name: ImageLength - type_names: [LONG, SHORT] -- id: 0x0102 - name: BitsPerSample - type_name: SHORT -- id: 0x0103 - name: Compression - type_name: SHORT -- id: 0x0106 - name: PhotometricInterpretation - type_name: SHORT -- id: 0x0107 - name: Thresholding - type_name: SHORT -- id: 0x0108 - name: CellWidth - type_name: SHORT -- id: 0x0109 - name: CellLength - type_name: SHORT -- id: 0x010a - name: FillOrder - type_name: SHORT -- id: 0x010d - name: DocumentName - type_name: ASCII -- id: 0x010e - name: ImageDescription - type_name: ASCII -- id: 0x010f - name: Make - type_name: ASCII -- id: 0x0110 - name: Model - type_name: ASCII -- id: 0x0111 - name: StripOffsets - type_names: [LONG, SHORT] -- id: 0x0112 - name: Orientation - type_name: SHORT -- id: 0x0115 - name: SamplesPerPixel - type_name: SHORT -- id: 0x0116 - name: RowsPerStrip - type_names: [LONG, SHORT] -- id: 0x0117 - name: StripByteCounts - type_names: [LONG, SHORT] -- id: 0x011a - name: XResolution - type_name: RATIONAL -- id: 0x011b - name: YResolution - type_name: RATIONAL -- id: 0x011c - name: PlanarConfiguration - type_name: SHORT -- id: 0x0122 - name: GrayResponseUnit - type_name: SHORT -- id: 0x0123 - name: GrayResponseCurve - type_name: SHORT -- id: 0x0124 - name: T4Options - type_name: LONG -- id: 0x0125 - name: T6Options - type_name: LONG -- id: 0x0128 - name: ResolutionUnit - type_name: SHORT -- id: 0x0129 - name: PageNumber - type_name: SHORT -- id: 0x012d - name: TransferFunction - type_name: SHORT -- id: 0x0131 - name: Software - type_name: ASCII -- id: 0x0132 - name: DateTime - type_name: ASCII -- id: 0x013b - name: Artist - type_name: ASCII -- id: 0x013c - name: HostComputer - type_name: ASCII -- id: 0x013d - name: Predictor - type_name: SHORT -- id: 0x013e - name: WhitePoint - type_name: RATIONAL -- id: 0x013f - name: PrimaryChromaticities - type_name: RATIONAL -- id: 0x0140 - name: ColorMap - type_name: SHORT -- id: 0x0141 - name: HalftoneHints - type_name: SHORT -- id: 0x0142 - name: TileWidth - type_name: SHORT -- id: 0x0143 - name: TileLength - type_name: SHORT -- id: 0x0144 - name: TileOffsets - type_name: SHORT -- id: 0x0145 - name: TileByteCounts - type_name: SHORT -- id: 0x014a - name: SubIFDs - type_name: LONG -- id: 0x014c - name: InkSet - type_name: SHORT -- id: 0x014d - name: InkNames - type_name: ASCII -- id: 0x014e - name: NumberOfInks - type_name: SHORT -- id: 0x0150 - name: DotRange - type_name: BYTE -- id: 0x0151 - name: TargetPrinter - type_name: ASCII -- id: 0x0152 - name: ExtraSamples - type_name: SHORT -- id: 0x0153 - name: SampleFormat - type_name: SHORT -- id: 0x0154 - name: SMinSampleValue - type_name: SHORT -- id: 0x0155 - name: SMaxSampleValue - type_name: SHORT -- id: 0x0156 - name: TransferRange - type_name: SHORT -- id: 0x0157 - name: ClipPath - type_name: BYTE -- id: 0x015a - name: Indexed - type_name: SHORT -- id: 0x015b - name: JPEGTables - type_name: UNDEFINED -- id: 0x015f - name: OPIProxy - type_name: SHORT -- id: 0x0200 - name: JPEGProc - type_name: LONG -- id: 0x0201 - name: JPEGInterchangeFormat - type_name: LONG -- id: 0x0202 - name: JPEGInterchangeFormatLength - type_name: LONG -- id: 0x0203 - name: JPEGRestartInterval - type_name: SHORT -- id: 0x0205 - name: JPEGLosslessPredictors - type_name: SHORT -- id: 0x0206 - name: JPEGPointTransforms - type_name: SHORT -- id: 0x0207 - name: JPEGQTables - type_name: LONG -- id: 0x0208 - name: JPEGDCTables - type_name: LONG -- id: 0x0209 - name: JPEGACTables - type_name: LONG -- id: 0x0211 - name: YCbCrCoefficients - type_name: RATIONAL -- id: 0x0212 - name: YCbCrSubSampling - type_name: SHORT -- id: 0x0213 - name: YCbCrPositioning - type_name: SHORT -- id: 0x0214 - name: ReferenceBlackWhite - type_name: RATIONAL -- id: 0x02bc - name: XMLPacket - type_name: BYTE -- id: 0x4746 - name: Rating - type_name: SHORT -- id: 0x4749 - name: RatingPercent - type_name: SHORT -- id: 0x800d - name: ImageID - type_name: ASCII -- id: 0x828d - name: CFARepeatPatternDim - type_name: SHORT -- id: 0x828e - name: CFAPattern - type_name: BYTE -- id: 0x828f - name: BatteryLevel - type_name: RATIONAL -- id: 0x8298 - name: Copyright - type_name: ASCII -- id: 0x829a - name: ExposureTime -# NOTE(dustin): SRATIONAL isn't mentioned in the standard, but we have seen it in real data. - type_names: [RATIONAL, SRATIONAL] -- id: 0x829d - name: FNumber -# NOTE(dustin): SRATIONAL isn't mentioned in the standard, but we have seen it in real data. - type_names: [RATIONAL, SRATIONAL] -- id: 0x83bb - name: IPTCNAA - type_name: LONG -- id: 0x8649 - name: ImageResources - type_name: BYTE -- id: 0x8769 - name: ExifTag - type_name: LONG -- id: 0x8773 - name: InterColorProfile - type_name: UNDEFINED -- id: 0x8822 - name: ExposureProgram - type_name: SHORT -- id: 0x8824 - name: SpectralSensitivity - type_name: ASCII -- id: 0x8825 - name: GPSTag - type_name: LONG -- id: 0x8827 - name: ISOSpeedRatings - type_name: SHORT -- id: 0x8828 - name: OECF - type_name: UNDEFINED -- id: 0x8829 - name: Interlace - type_name: SHORT -- id: 0x882b - name: SelfTimerMode - type_name: SHORT -- id: 0x9003 - name: DateTimeOriginal - type_name: ASCII -- id: 0x9102 - name: CompressedBitsPerPixel - type_name: RATIONAL -- id: 0x9201 - name: ShutterSpeedValue - type_name: SRATIONAL -- id: 0x9202 - name: ApertureValue - type_name: RATIONAL -- id: 0x9203 - name: BrightnessValue - type_name: SRATIONAL -- id: 0x9204 - name: ExposureBiasValue - type_name: SRATIONAL -- id: 0x9205 - name: MaxApertureValue - type_name: RATIONAL -- id: 0x9206 - name: SubjectDistance - type_name: SRATIONAL -- id: 0x9207 - name: MeteringMode - type_name: SHORT -- id: 0x9208 - name: LightSource - type_name: SHORT -- id: 0x9209 - name: Flash - type_name: SHORT -- id: 0x920a - name: FocalLength - type_name: RATIONAL -- id: 0x920b - name: FlashEnergy - type_name: RATIONAL -- id: 0x920c - name: SpatialFrequencyResponse - type_name: UNDEFINED -- id: 0x920d - name: Noise - type_name: UNDEFINED -- id: 0x920e - name: FocalPlaneXResolution - type_name: RATIONAL -- id: 0x920f - name: FocalPlaneYResolution - type_name: RATIONAL -- id: 0x9210 - name: FocalPlaneResolutionUnit - type_name: SHORT -- id: 0x9211 - name: ImageNumber - type_name: LONG -- id: 0x9212 - name: SecurityClassification - type_name: ASCII -- id: 0x9213 - name: ImageHistory - type_name: ASCII -- id: 0x9214 - name: SubjectLocation - type_name: SHORT -- id: 0x9215 - name: ExposureIndex - type_name: RATIONAL -- id: 0x9216 - name: TIFFEPStandardID - type_name: BYTE -- id: 0x9217 - name: SensingMethod - type_name: SHORT -- id: 0x9c9b - name: XPTitle - type_name: BYTE -- id: 0x9c9c - name: XPComment - type_name: BYTE -- id: 0x9c9d - name: XPAuthor - type_name: BYTE -- id: 0x9c9e - name: XPKeywords - type_name: BYTE -- id: 0x9c9f - name: XPSubject - type_name: BYTE -- id: 0xc4a5 - name: PrintImageMatching - type_name: UNDEFINED -- id: 0xc612 - name: DNGVersion - type_name: BYTE -- id: 0xc613 - name: DNGBackwardVersion - type_name: BYTE -- id: 0xc614 - name: UniqueCameraModel - type_name: ASCII -- id: 0xc615 - name: LocalizedCameraModel - type_name: BYTE -- id: 0xc616 - name: CFAPlaneColor - type_name: BYTE -- id: 0xc617 - name: CFALayout - type_name: SHORT -- id: 0xc618 - name: LinearizationTable - type_name: SHORT -- id: 0xc619 - name: BlackLevelRepeatDim - type_name: SHORT -- id: 0xc61a - name: BlackLevel - type_name: RATIONAL -- id: 0xc61b - name: BlackLevelDeltaH - type_name: SRATIONAL -- id: 0xc61c - name: BlackLevelDeltaV - type_name: SRATIONAL -- id: 0xc61d - name: WhiteLevel - type_name: SHORT -- id: 0xc61e - name: DefaultScale - type_name: RATIONAL -- id: 0xc61f - name: DefaultCropOrigin - type_name: SHORT -- id: 0xc620 - name: DefaultCropSize - type_name: SHORT -- id: 0xc621 - name: ColorMatrix1 - type_name: SRATIONAL -- id: 0xc622 - name: ColorMatrix2 - type_name: SRATIONAL -- id: 0xc623 - name: CameraCalibration1 - type_name: SRATIONAL -- id: 0xc624 - name: CameraCalibration2 - type_name: SRATIONAL -- id: 0xc625 - name: ReductionMatrix1 - type_name: SRATIONAL -- id: 0xc626 - name: ReductionMatrix2 - type_name: SRATIONAL -- id: 0xc627 - name: AnalogBalance - type_name: RATIONAL -- id: 0xc628 - name: AsShotNeutral - type_name: SHORT -- id: 0xc629 - name: AsShotWhiteXY - type_name: RATIONAL -- id: 0xc62a - name: BaselineExposure - type_name: SRATIONAL -- id: 0xc62b - name: BaselineNoise - type_name: RATIONAL -- id: 0xc62c - name: BaselineSharpness - type_name: RATIONAL -- id: 0xc62d - name: BayerGreenSplit - type_name: LONG -- id: 0xc62e - name: LinearResponseLimit - type_name: RATIONAL -- id: 0xc62f - name: CameraSerialNumber - type_name: ASCII -- id: 0xc630 - name: LensInfo - type_name: RATIONAL -- id: 0xc631 - name: ChromaBlurRadius - type_name: RATIONAL -- id: 0xc632 - name: AntiAliasStrength - type_name: RATIONAL -- id: 0xc633 - name: ShadowScale - type_name: SRATIONAL -- id: 0xc634 - name: DNGPrivateData - type_name: BYTE -- id: 0xc635 - name: MakerNoteSafety - type_name: SHORT -- id: 0xc65a - name: CalibrationIlluminant1 - type_name: SHORT -- id: 0xc65b - name: CalibrationIlluminant2 - type_name: SHORT -- id: 0xc65c - name: BestQualityScale - type_name: RATIONAL -- id: 0xc65d - name: RawDataUniqueID - type_name: BYTE -- id: 0xc68b - name: OriginalRawFileName - type_name: BYTE -- id: 0xc68c - name: OriginalRawFileData - type_name: UNDEFINED -- id: 0xc68d - name: ActiveArea - type_name: SHORT -- id: 0xc68e - name: MaskedAreas - type_name: SHORT -- id: 0xc68f - name: AsShotICCProfile - type_name: UNDEFINED -- id: 0xc690 - name: AsShotPreProfileMatrix - type_name: SRATIONAL -- id: 0xc691 - name: CurrentICCProfile - type_name: UNDEFINED -- id: 0xc692 - name: CurrentPreProfileMatrix - type_name: SRATIONAL -- id: 0xc6bf - name: ColorimetricReference - type_name: SHORT -- id: 0xc6f3 - name: CameraCalibrationSignature - type_name: BYTE -- id: 0xc6f4 - name: ProfileCalibrationSignature - type_name: BYTE -- id: 0xc6f6 - name: AsShotProfileName - type_name: BYTE -- id: 0xc6f7 - name: NoiseReductionApplied - type_name: RATIONAL -- id: 0xc6f8 - name: ProfileName - type_name: BYTE -- id: 0xc6f9 - name: ProfileHueSatMapDims - type_name: LONG -- id: 0xc6fd - name: ProfileEmbedPolicy - type_name: LONG -- id: 0xc6fe - name: ProfileCopyright - type_name: BYTE -- id: 0xc714 - name: ForwardMatrix1 - type_name: SRATIONAL -- id: 0xc715 - name: ForwardMatrix2 - type_name: SRATIONAL -- id: 0xc716 - name: PreviewApplicationName - type_name: BYTE -- id: 0xc717 - name: PreviewApplicationVersion - type_name: BYTE -- id: 0xc718 - name: PreviewSettingsName - type_name: BYTE -- id: 0xc719 - name: PreviewSettingsDigest - type_name: BYTE -- id: 0xc71a - name: PreviewColorSpace - type_name: LONG -- id: 0xc71b - name: PreviewDateTime - type_name: ASCII -- id: 0xc71c - name: RawImageDigest - type_name: UNDEFINED -- id: 0xc71d - name: OriginalRawFileDigest - type_name: UNDEFINED -- id: 0xc71e - name: SubTileBlockSize - type_name: LONG -- id: 0xc71f - name: RowInterleaveFactor - type_name: LONG -- id: 0xc725 - name: ProfileLookTableDims - type_name: LONG -- id: 0xc740 - name: OpcodeList1 - type_name: UNDEFINED -- id: 0xc741 - name: OpcodeList2 - type_name: UNDEFINED -- id: 0xc74e - name: OpcodeList3 - type_name: UNDEFINED -IFD/Exif/Iop: -- id: 0x0001 - name: InteroperabilityIndex - type_name: ASCII -- id: 0x0002 - name: InteroperabilityVersion - type_name: UNDEFINED -- id: 0x1000 - name: RelatedImageFileFormat - type_name: ASCII -- id: 0x1001 - name: RelatedImageWidth - type_name: LONG -- id: 0x1002 - name: RelatedImageLength - type_name: LONG -` -) diff --git a/vendor/github.com/dsoprea/go-exif/v2/testing_common.go b/vendor/github.com/dsoprea/go-exif/v2/testing_common.go deleted file mode 100644 index fe69df936..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/testing_common.go +++ /dev/null @@ -1,182 +0,0 @@ -package exif - -import ( - "path" - "reflect" - "testing" - - "io/ioutil" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -var ( - testExifData []byte -) - -func getExifSimpleTestIb() *IfdBuilder { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - im := NewIfdMapping() - - err := LoadStandardIfds(im) - log.PanicIf(err) - - ti := NewTagIndex() - ib := NewIfdBuilder(im, ti, exifcommon.IfdStandardIfdIdentity, exifcommon.TestDefaultByteOrder) - - err = ib.AddStandard(0x000b, "asciivalue") - log.PanicIf(err) - - err = ib.AddStandard(0x00ff, []uint16{0x1122}) - log.PanicIf(err) - - err = ib.AddStandard(0x0100, []uint32{0x33445566}) - log.PanicIf(err) - - err = ib.AddStandard(0x013e, []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}) - log.PanicIf(err) - - return ib -} - -func getExifSimpleTestIbBytes() []byte { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - im := NewIfdMapping() - - err := LoadStandardIfds(im) - log.PanicIf(err) - - ti := NewTagIndex() - ib := NewIfdBuilder(im, ti, exifcommon.IfdStandardIfdIdentity, exifcommon.TestDefaultByteOrder) - - err = ib.AddStandard(0x000b, "asciivalue") - log.PanicIf(err) - - err = ib.AddStandard(0x00ff, []uint16{0x1122}) - log.PanicIf(err) - - err = ib.AddStandard(0x0100, []uint32{0x33445566}) - log.PanicIf(err) - - err = ib.AddStandard(0x013e, []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}) - log.PanicIf(err) - - ibe := NewIfdByteEncoder() - - exifData, err := ibe.EncodeToExif(ib) - log.PanicIf(err) - - return exifData -} - -func validateExifSimpleTestIb(exifData []byte, t *testing.T) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - im := NewIfdMapping() - - err := LoadStandardIfds(im) - log.PanicIf(err) - - ti := NewTagIndex() - - eh, index, err := Collect(im, ti, exifData) - log.PanicIf(err) - - if eh.ByteOrder != exifcommon.TestDefaultByteOrder { - t.Fatalf("EXIF byte-order is not correct: %v", eh.ByteOrder) - } else if eh.FirstIfdOffset != ExifDefaultFirstIfdOffset { - t.Fatalf("EXIF first IFD-offset not correct: (0x%02x)", eh.FirstIfdOffset) - } - - if len(index.Ifds) != 1 { - t.Fatalf("There wasn't exactly one IFD decoded: (%d)", len(index.Ifds)) - } - - ifd := index.RootIfd - - if ifd.ByteOrder != exifcommon.TestDefaultByteOrder { - t.Fatalf("IFD byte-order not correct.") - } else if ifd.ifdIdentity.UnindexedString() != exifcommon.IfdStandardIfdIdentity.UnindexedString() { - t.Fatalf("IFD name not correct.") - } else if ifd.ifdIdentity.Index() != 0 { - t.Fatalf("IFD index not zero: (%d)", ifd.ifdIdentity.Index()) - } else if ifd.Offset != uint32(0x0008) { - t.Fatalf("IFD offset not correct.") - } else if len(ifd.Entries) != 4 { - t.Fatalf("IFD number of entries not correct: (%d)", len(ifd.Entries)) - } else if ifd.NextIfdOffset != uint32(0) { - t.Fatalf("Next-IFD offset is non-zero.") - } else if ifd.NextIfd != nil { - t.Fatalf("Next-IFD pointer is non-nil.") - } - - // Verify the values by using the actual, original types (this is awesome). - - expected := []struct { - tagId uint16 - value interface{} - }{ - {tagId: 0x000b, value: "asciivalue"}, - {tagId: 0x00ff, value: []uint16{0x1122}}, - {tagId: 0x0100, value: []uint32{0x33445566}}, - {tagId: 0x013e, value: []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}}, - } - - for i, ite := range ifd.Entries { - if ite.TagId() != expected[i].tagId { - t.Fatalf("Tag-ID for entry (%d) not correct: (0x%02x) != (0x%02x)", i, ite.TagId(), expected[i].tagId) - } - - value, err := ite.Value() - log.PanicIf(err) - - if reflect.DeepEqual(value, expected[i].value) != true { - t.Fatalf("Value for entry (%d) not correct: [%v] != [%v]", i, value, expected[i].value) - } - } -} - -func getTestImageFilepath() string { - assetsPath := exifcommon.GetTestAssetsPath() - testImageFilepath := path.Join(assetsPath, "NDM_8901.jpg") - return testImageFilepath -} - -func getTestExifData() []byte { - if testExifData == nil { - assetsPath := exifcommon.GetTestAssetsPath() - filepath := path.Join(assetsPath, "NDM_8901.jpg.exif") - - var err error - - testExifData, err = ioutil.ReadFile(filepath) - log.PanicIf(err) - } - - return testExifData -} - -func getTestGpsImageFilepath() string { - assetsPath := exifcommon.GetTestAssetsPath() - testGpsImageFilepath := path.Join(assetsPath, "gps.jpg") - return testGpsImageFilepath -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/README.md b/vendor/github.com/dsoprea/go-exif/v2/undefined/README.md deleted file mode 100644 index d2caa6e51..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/README.md +++ /dev/null @@ -1,4 +0,0 @@ - -## 0xa40b - -The specification is not specific/clear enough to be handled. Without a working example ,we're deferring until some point in the future when either we or someone else has a better understanding. diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/accessor.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/accessor.go deleted file mode 100644 index 3e82c0f61..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/accessor.go +++ /dev/null @@ -1,62 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -// Encode encodes the given encodeable undefined value to bytes. -func Encode(value EncodeableValue, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - encoderName := value.EncoderName() - - encoder, found := encoders[encoderName] - if found == false { - log.Panicf("no encoder registered for type [%s]", encoderName) - } - - encoded, unitCount, err = encoder.Encode(value, byteOrder) - log.PanicIf(err) - - return encoded, unitCount, nil -} - -// Decode constructs a value from raw encoded bytes -func Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - uth := UndefinedTagHandle{ - IfdPath: valueContext.IfdPath(), - TagId: valueContext.TagId(), - } - - decoder, found := decoders[uth] - if found == false { - // We have no choice but to return the error. We have no way of knowing how - // much data there is without already knowing what data-type this tag is. - return nil, exifcommon.ErrUnhandledUndefinedTypedTag - } - - value, err = decoder.Decode(valueContext) - if err != nil { - if err == ErrUnparseableValue { - return nil, err - } - - log.Panic(err) - } - - return value, nil -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_8828_oecf.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_8828_oecf.go deleted file mode 100644 index 796d17ca7..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_8828_oecf.go +++ /dev/null @@ -1,148 +0,0 @@ -package exifundefined - -import ( - "bytes" - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag8828Oecf struct { - Columns uint16 - Rows uint16 - ColumnNames []string - Values []exifcommon.SignedRational -} - -func (oecf Tag8828Oecf) String() string { - return fmt.Sprintf("Tag8828Oecf", oecf.Columns, oecf.Rows) -} - -func (oecf Tag8828Oecf) EncoderName() string { - return "Codec8828Oecf" -} - -type Codec8828Oecf struct { -} - -func (Codec8828Oecf) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - oecf, ok := value.(Tag8828Oecf) - if ok == false { - log.Panicf("can only encode a Tag8828Oecf") - } - - b := new(bytes.Buffer) - - err = binary.Write(b, byteOrder, oecf.Columns) - log.PanicIf(err) - - err = binary.Write(b, byteOrder, oecf.Rows) - log.PanicIf(err) - - for _, name := range oecf.ColumnNames { - _, err := b.Write([]byte(name)) - log.PanicIf(err) - - _, err = b.Write([]byte{0}) - log.PanicIf(err) - } - - ve := exifcommon.NewValueEncoder(byteOrder) - - ed, err := ve.Encode(oecf.Values) - log.PanicIf(err) - - _, err = b.Write(ed.Encoded) - log.PanicIf(err) - - return b.Bytes(), uint32(b.Len()), nil -} - -func (Codec8828Oecf) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test using known good data. - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - oecf := Tag8828Oecf{} - - oecf.Columns = valueContext.ByteOrder().Uint16(valueBytes[0:2]) - oecf.Rows = valueContext.ByteOrder().Uint16(valueBytes[2:4]) - - columnNames := make([]string, oecf.Columns) - - // startAt is where the current column name starts. - startAt := 4 - - // offset is our current position. - offset := startAt - - currentColumnNumber := uint16(0) - - for currentColumnNumber < oecf.Columns { - if valueBytes[offset] == 0 { - columnName := string(valueBytes[startAt:offset]) - if len(columnName) == 0 { - log.Panicf("SFR column (%d) has zero length", currentColumnNumber) - } - - columnNames[currentColumnNumber] = columnName - currentColumnNumber++ - - offset++ - startAt = offset - continue - } - - offset++ - } - - oecf.ColumnNames = columnNames - - rawRationalBytes := valueBytes[offset:] - - rationalSize := exifcommon.TypeSignedRational.Size() - if len(rawRationalBytes)%rationalSize > 0 { - log.Panicf("OECF signed-rationals not aligned: (%d) %% (%d) > 0", len(rawRationalBytes), rationalSize) - } - - rationalCount := len(rawRationalBytes) / rationalSize - - parser := new(exifcommon.Parser) - - byteOrder := valueContext.ByteOrder() - - items, err := parser.ParseSignedRationals(rawRationalBytes, uint32(rationalCount), byteOrder) - log.PanicIf(err) - - oecf.Values = items - - return oecf, nil -} - -func init() { - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0x8828, - Codec8828Oecf{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9000_exif_version.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9000_exif_version.go deleted file mode 100644 index 19cfcc906..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9000_exif_version.go +++ /dev/null @@ -1,69 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag9000ExifVersion struct { - ExifVersion string -} - -func (Tag9000ExifVersion) EncoderName() string { - return "Codec9000ExifVersion" -} - -func (ev Tag9000ExifVersion) String() string { - return ev.ExifVersion -} - -type Codec9000ExifVersion struct { -} - -func (Codec9000ExifVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s, ok := value.(Tag9000ExifVersion) - if ok == false { - log.Panicf("can only encode a Tag9000ExifVersion") - } - - return []byte(s.ExifVersion), uint32(len(s.ExifVersion)), nil -} - -func (Codec9000ExifVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) - - valueString, err := valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - ev := Tag9000ExifVersion{ - ExifVersion: valueString, - } - - return ev, nil -} - -func init() { - registerEncoder( - Tag9000ExifVersion{}, - Codec9000ExifVersion{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0x9000, - Codec9000ExifVersion{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9101_components_configuration.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9101_components_configuration.go deleted file mode 100644 index 16a4b38e4..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9101_components_configuration.go +++ /dev/null @@ -1,124 +0,0 @@ -package exifundefined - -import ( - "bytes" - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -const ( - TagUndefinedType_9101_ComponentsConfiguration_Channel_Y = 0x1 - TagUndefinedType_9101_ComponentsConfiguration_Channel_Cb = 0x2 - TagUndefinedType_9101_ComponentsConfiguration_Channel_Cr = 0x3 - TagUndefinedType_9101_ComponentsConfiguration_Channel_R = 0x4 - TagUndefinedType_9101_ComponentsConfiguration_Channel_G = 0x5 - TagUndefinedType_9101_ComponentsConfiguration_Channel_B = 0x6 -) - -const ( - TagUndefinedType_9101_ComponentsConfiguration_OTHER = iota - TagUndefinedType_9101_ComponentsConfiguration_RGB = iota - TagUndefinedType_9101_ComponentsConfiguration_YCBCR = iota -) - -var ( - TagUndefinedType_9101_ComponentsConfiguration_Names = map[int]string{ - TagUndefinedType_9101_ComponentsConfiguration_OTHER: "OTHER", - TagUndefinedType_9101_ComponentsConfiguration_RGB: "RGB", - TagUndefinedType_9101_ComponentsConfiguration_YCBCR: "YCBCR", - } - - TagUndefinedType_9101_ComponentsConfiguration_Configurations = map[int][]byte{ - TagUndefinedType_9101_ComponentsConfiguration_RGB: { - TagUndefinedType_9101_ComponentsConfiguration_Channel_R, - TagUndefinedType_9101_ComponentsConfiguration_Channel_G, - TagUndefinedType_9101_ComponentsConfiguration_Channel_B, - 0, - }, - - TagUndefinedType_9101_ComponentsConfiguration_YCBCR: { - TagUndefinedType_9101_ComponentsConfiguration_Channel_Y, - TagUndefinedType_9101_ComponentsConfiguration_Channel_Cb, - TagUndefinedType_9101_ComponentsConfiguration_Channel_Cr, - 0, - }, - } -) - -type TagExif9101ComponentsConfiguration struct { - ConfigurationId int - ConfigurationBytes []byte -} - -func (TagExif9101ComponentsConfiguration) EncoderName() string { - return "CodecExif9101ComponentsConfiguration" -} - -func (cc TagExif9101ComponentsConfiguration) String() string { - return fmt.Sprintf("Exif9101ComponentsConfiguration", TagUndefinedType_9101_ComponentsConfiguration_Names[cc.ConfigurationId], cc.ConfigurationBytes) -} - -type CodecExif9101ComponentsConfiguration struct { -} - -func (CodecExif9101ComponentsConfiguration) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - cc, ok := value.(TagExif9101ComponentsConfiguration) - if ok == false { - log.Panicf("can only encode a TagExif9101ComponentsConfiguration") - } - - return cc.ConfigurationBytes, uint32(len(cc.ConfigurationBytes)), nil -} - -func (CodecExif9101ComponentsConfiguration) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - for configurationId, configurationBytes := range TagUndefinedType_9101_ComponentsConfiguration_Configurations { - if bytes.Equal(configurationBytes, valueBytes) == true { - cc := TagExif9101ComponentsConfiguration{ - ConfigurationId: configurationId, - ConfigurationBytes: valueBytes, - } - - return cc, nil - } - } - - cc := TagExif9101ComponentsConfiguration{ - ConfigurationId: TagUndefinedType_9101_ComponentsConfiguration_OTHER, - ConfigurationBytes: valueBytes, - } - - return cc, nil -} - -func init() { - registerEncoder( - TagExif9101ComponentsConfiguration{}, - CodecExif9101ComponentsConfiguration{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0x9101, - CodecExif9101ComponentsConfiguration{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_927C_maker_note.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_927C_maker_note.go deleted file mode 100644 index e0a52db2a..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_927C_maker_note.go +++ /dev/null @@ -1,114 +0,0 @@ -package exifundefined - -import ( - "fmt" - "strings" - - "crypto/sha1" - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag927CMakerNote struct { - MakerNoteType []byte - MakerNoteBytes []byte -} - -func (Tag927CMakerNote) EncoderName() string { - return "Codec927CMakerNote" -} - -func (mn Tag927CMakerNote) String() string { - parts := make([]string, len(mn.MakerNoteType)) - - for i, c := range mn.MakerNoteType { - parts[i] = fmt.Sprintf("%02x", c) - } - - h := sha1.New() - - _, err := h.Write(mn.MakerNoteBytes) - log.PanicIf(err) - - digest := h.Sum(nil) - - return fmt.Sprintf("MakerNote", strings.Join(parts, " "), len(mn.MakerNoteBytes), digest) -} - -type Codec927CMakerNote struct { -} - -func (Codec927CMakerNote) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - mn, ok := value.(Tag927CMakerNote) - if ok == false { - log.Panicf("can only encode a Tag927CMakerNote") - } - - // TODO(dustin): Confirm this size against the specification. - - return mn.MakerNoteBytes, uint32(len(mn.MakerNoteBytes)), nil -} - -func (Codec927CMakerNote) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // MakerNote - // TODO(dustin): !! This is the Wild Wild West. This very well might be a child IFD, but any and all OEM's define their own formats. If we're going to be writing changes and this is complete EXIF (which may not have the first eight bytes), it might be fine. However, if these are just IFDs they'll be relative to the main EXIF, this will invalidate the MakerNote data for IFDs and any other implementations that use offsets unless we can interpret them all. It be best to return to this later and just exclude this from being written for now, though means a loss of a wealth of image metadata. - // -> We can also just blindly try to interpret as an IFD and just validate that it's looks good (maybe it will even have a 'next ifd' pointer that we can validate is 0x0). - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - // TODO(dustin): Doesn't work, but here as an example. - // ie := NewIfdEnumerate(valueBytes, byteOrder) - - // // TODO(dustin): !! Validate types (might have proprietary types, but it might be worth splitting the list between valid and not valid; maybe fail if a certain proportion are invalid, or maybe aren't less then a certain small integer)? - // ii, err := ie.Collect(0x0) - - // for _, entry := range ii.RootIfd.Entries { - // fmt.Printf("ENTRY: 0x%02x %d\n", entry.TagId, entry.TagType) - // } - - var makerNoteType []byte - if len(valueBytes) >= 20 { - makerNoteType = valueBytes[:20] - } else { - makerNoteType = valueBytes - } - - mn := Tag927CMakerNote{ - MakerNoteType: makerNoteType, - - // MakerNoteBytes has the whole length of bytes. There's always - // the chance that the first 20 bytes includes actual data. - MakerNoteBytes: valueBytes, - } - - return mn, nil -} - -func init() { - registerEncoder( - Tag927CMakerNote{}, - Codec927CMakerNote{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0x927c, - Codec927CMakerNote{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9286_user_comment.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9286_user_comment.go deleted file mode 100644 index de07fe19e..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_9286_user_comment.go +++ /dev/null @@ -1,142 +0,0 @@ -package exifundefined - -import ( - "bytes" - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -var ( - exif9286Logger = log.NewLogger("exifundefined.exif_9286_user_comment") -) - -const ( - TagUndefinedType_9286_UserComment_Encoding_ASCII = iota - TagUndefinedType_9286_UserComment_Encoding_JIS = iota - TagUndefinedType_9286_UserComment_Encoding_UNICODE = iota - TagUndefinedType_9286_UserComment_Encoding_UNDEFINED = iota -) - -var ( - TagUndefinedType_9286_UserComment_Encoding_Names = map[int]string{ - TagUndefinedType_9286_UserComment_Encoding_ASCII: "ASCII", - TagUndefinedType_9286_UserComment_Encoding_JIS: "JIS", - TagUndefinedType_9286_UserComment_Encoding_UNICODE: "UNICODE", - TagUndefinedType_9286_UserComment_Encoding_UNDEFINED: "UNDEFINED", - } - - TagUndefinedType_9286_UserComment_Encodings = map[int][]byte{ - TagUndefinedType_9286_UserComment_Encoding_ASCII: {'A', 'S', 'C', 'I', 'I', 0, 0, 0}, - TagUndefinedType_9286_UserComment_Encoding_JIS: {'J', 'I', 'S', 0, 0, 0, 0, 0}, - TagUndefinedType_9286_UserComment_Encoding_UNICODE: {'U', 'n', 'i', 'c', 'o', 'd', 'e', 0}, - TagUndefinedType_9286_UserComment_Encoding_UNDEFINED: {0, 0, 0, 0, 0, 0, 0, 0}, - } -) - -type Tag9286UserComment struct { - EncodingType int - EncodingBytes []byte -} - -func (Tag9286UserComment) EncoderName() string { - return "Codec9286UserComment" -} - -func (uc Tag9286UserComment) String() string { - var valuePhrase string - - if uc.EncodingType == TagUndefinedType_9286_UserComment_Encoding_ASCII { - return fmt.Sprintf("[ASCII] %s", string(uc.EncodingBytes)) - } else { - if len(uc.EncodingBytes) <= 8 { - valuePhrase = fmt.Sprintf("%v", uc.EncodingBytes) - } else { - valuePhrase = fmt.Sprintf("%v...", uc.EncodingBytes[:8]) - } - } - - return fmt.Sprintf("UserComment", len(uc.EncodingBytes), TagUndefinedType_9286_UserComment_Encoding_Names[uc.EncodingType], valuePhrase, len(uc.EncodingBytes)) -} - -type Codec9286UserComment struct { -} - -func (Codec9286UserComment) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - uc, ok := value.(Tag9286UserComment) - if ok == false { - log.Panicf("can only encode a Tag9286UserComment") - } - - encodingTypeBytes, found := TagUndefinedType_9286_UserComment_Encodings[uc.EncodingType] - if found == false { - log.Panicf("encoding-type not valid for unknown-type tag 9286 (UserComment): (%d)", uc.EncodingType) - } - - encoded = make([]byte, len(uc.EncodingBytes)+8) - - copy(encoded[:8], encodingTypeBytes) - copy(encoded[8:], uc.EncodingBytes) - - // TODO(dustin): Confirm this size against the specification. - - return encoded, uint32(len(encoded)), nil -} - -func (Codec9286UserComment) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - if len(valueBytes) < 8 { - return nil, ErrUnparseableValue - } - - unknownUc := Tag9286UserComment{ - EncodingType: TagUndefinedType_9286_UserComment_Encoding_UNDEFINED, - EncodingBytes: []byte{}, - } - - encoding := valueBytes[:8] - for encodingIndex, encodingBytes := range TagUndefinedType_9286_UserComment_Encodings { - if bytes.Compare(encoding, encodingBytes) == 0 { - uc := Tag9286UserComment{ - EncodingType: encodingIndex, - EncodingBytes: valueBytes[8:], - } - - return uc, nil - } - } - - exif9286Logger.Warningf(nil, "User-comment encoding not valid. Returning 'unknown' type (the default).") - return unknownUc, nil -} - -func init() { - registerEncoder( - Tag9286UserComment{}, - Codec9286UserComment{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0x9286, - Codec9286UserComment{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A000_flashpix_version.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A000_flashpix_version.go deleted file mode 100644 index 28849cde5..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A000_flashpix_version.go +++ /dev/null @@ -1,69 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type TagA000FlashpixVersion struct { - FlashpixVersion string -} - -func (TagA000FlashpixVersion) EncoderName() string { - return "CodecA000FlashpixVersion" -} - -func (fv TagA000FlashpixVersion) String() string { - return fv.FlashpixVersion -} - -type CodecA000FlashpixVersion struct { -} - -func (CodecA000FlashpixVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s, ok := value.(TagA000FlashpixVersion) - if ok == false { - log.Panicf("can only encode a TagA000FlashpixVersion") - } - - return []byte(s.FlashpixVersion), uint32(len(s.FlashpixVersion)), nil -} - -func (CodecA000FlashpixVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) - - valueString, err := valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - fv := TagA000FlashpixVersion{ - FlashpixVersion: valueString, - } - - return fv, nil -} - -func init() { - registerEncoder( - TagA000FlashpixVersion{}, - CodecA000FlashpixVersion{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0xa000, - CodecA000FlashpixVersion{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A20C_spatial_frequency_response.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A20C_spatial_frequency_response.go deleted file mode 100644 index d49c8c52d..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A20C_spatial_frequency_response.go +++ /dev/null @@ -1,160 +0,0 @@ -package exifundefined - -import ( - "bytes" - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type TagA20CSpatialFrequencyResponse struct { - Columns uint16 - Rows uint16 - ColumnNames []string - Values []exifcommon.Rational -} - -func (TagA20CSpatialFrequencyResponse) EncoderName() string { - return "CodecA20CSpatialFrequencyResponse" -} - -func (sfr TagA20CSpatialFrequencyResponse) String() string { - return fmt.Sprintf("CodecA20CSpatialFrequencyResponse", sfr.Columns, sfr.Rows) -} - -type CodecA20CSpatialFrequencyResponse struct { -} - -func (CodecA20CSpatialFrequencyResponse) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test. - - sfr, ok := value.(TagA20CSpatialFrequencyResponse) - if ok == false { - log.Panicf("can only encode a TagA20CSpatialFrequencyResponse") - } - - b := new(bytes.Buffer) - - err = binary.Write(b, byteOrder, sfr.Columns) - log.PanicIf(err) - - err = binary.Write(b, byteOrder, sfr.Rows) - log.PanicIf(err) - - // Write columns. - - for _, name := range sfr.ColumnNames { - _, err := b.WriteString(name) - log.PanicIf(err) - - err = b.WriteByte(0) - log.PanicIf(err) - } - - // Write values. - - ve := exifcommon.NewValueEncoder(byteOrder) - - ed, err := ve.Encode(sfr.Values) - log.PanicIf(err) - - _, err = b.Write(ed.Encoded) - log.PanicIf(err) - - encoded = b.Bytes() - - // TODO(dustin): Confirm this size against the specification. - - return encoded, uint32(len(encoded)), nil -} - -func (CodecA20CSpatialFrequencyResponse) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test using known good data. - - byteOrder := valueContext.ByteOrder() - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - sfr := TagA20CSpatialFrequencyResponse{} - - sfr.Columns = byteOrder.Uint16(valueBytes[0:2]) - sfr.Rows = byteOrder.Uint16(valueBytes[2:4]) - - columnNames := make([]string, sfr.Columns) - - // startAt is where the current column name starts. - startAt := 4 - - // offset is our current position. - offset := 4 - - currentColumnNumber := uint16(0) - - for currentColumnNumber < sfr.Columns { - if valueBytes[offset] == 0 { - columnName := string(valueBytes[startAt:offset]) - if len(columnName) == 0 { - log.Panicf("SFR column (%d) has zero length", currentColumnNumber) - } - - columnNames[currentColumnNumber] = columnName - currentColumnNumber++ - - offset++ - startAt = offset - continue - } - - offset++ - } - - sfr.ColumnNames = columnNames - - rawRationalBytes := valueBytes[offset:] - - rationalSize := exifcommon.TypeRational.Size() - if len(rawRationalBytes)%rationalSize > 0 { - log.Panicf("SFR rationals not aligned: (%d) %% (%d) > 0", len(rawRationalBytes), rationalSize) - } - - rationalCount := len(rawRationalBytes) / rationalSize - - parser := new(exifcommon.Parser) - - items, err := parser.ParseRationals(rawRationalBytes, uint32(rationalCount), byteOrder) - log.PanicIf(err) - - sfr.Values = items - - return sfr, nil -} - -func init() { - registerEncoder( - TagA20CSpatialFrequencyResponse{}, - CodecA20CSpatialFrequencyResponse{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0xa20c, - CodecA20CSpatialFrequencyResponse{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A300_file_source.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A300_file_source.go deleted file mode 100644 index 18a7cdf63..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A300_file_source.go +++ /dev/null @@ -1,79 +0,0 @@ -package exifundefined - -import ( - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type TagExifA300FileSource uint32 - -func (TagExifA300FileSource) EncoderName() string { - return "CodecExifA300FileSource" -} - -func (af TagExifA300FileSource) String() string { - return fmt.Sprintf("0x%08x", uint32(af)) -} - -const ( - TagUndefinedType_A300_SceneType_Others TagExifA300FileSource = 0 - TagUndefinedType_A300_SceneType_ScannerOfTransparentType TagExifA300FileSource = 1 - TagUndefinedType_A300_SceneType_ScannerOfReflexType TagExifA300FileSource = 2 - TagUndefinedType_A300_SceneType_Dsc TagExifA300FileSource = 3 -) - -type CodecExifA300FileSource struct { -} - -func (CodecExifA300FileSource) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - st, ok := value.(TagExifA300FileSource) - if ok == false { - log.Panicf("can only encode a TagExifA300FileSource") - } - - ve := exifcommon.NewValueEncoder(byteOrder) - - ed, err := ve.Encode([]uint32{uint32(st)}) - log.PanicIf(err) - - // TODO(dustin): Confirm this size against the specification. It's non-specific about what type it is, but it looks to be no more than a single integer scalar. So, we're assuming it's a LONG. - - return ed.Encoded, 1, nil -} - -func (CodecExifA300FileSource) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeLong) - - valueLongs, err := valueContext.ReadLongs() - log.PanicIf(err) - - return TagExifA300FileSource(valueLongs[0]), nil -} - -func init() { - registerEncoder( - TagExifA300FileSource(0), - CodecExifA300FileSource{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0xa300, - CodecExifA300FileSource{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A301_scene_type.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A301_scene_type.go deleted file mode 100644 index b4246da18..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A301_scene_type.go +++ /dev/null @@ -1,76 +0,0 @@ -package exifundefined - -import ( - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type TagExifA301SceneType uint32 - -func (TagExifA301SceneType) EncoderName() string { - return "CodecExifA301SceneType" -} - -func (st TagExifA301SceneType) String() string { - return fmt.Sprintf("0x%08x", uint32(st)) -} - -const ( - TagUndefinedType_A301_SceneType_DirectlyPhotographedImage TagExifA301SceneType = 1 -) - -type CodecExifA301SceneType struct { -} - -func (CodecExifA301SceneType) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - st, ok := value.(TagExifA301SceneType) - if ok == false { - log.Panicf("can only encode a TagExif9101ComponentsConfiguration") - } - - ve := exifcommon.NewValueEncoder(byteOrder) - - ed, err := ve.Encode([]uint32{uint32(st)}) - log.PanicIf(err) - - // TODO(dustin): Confirm this size against the specification. It's non-specific about what type it is, but it looks to be no more than a single integer scalar. So, we're assuming it's a LONG. - - return ed.Encoded, uint32(int(ed.UnitCount)), nil -} - -func (CodecExifA301SceneType) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeLong) - - valueLongs, err := valueContext.ReadLongs() - log.PanicIf(err) - - return TagExifA301SceneType(valueLongs[0]), nil -} - -func init() { - registerEncoder( - TagExifA301SceneType(0), - CodecExifA301SceneType{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0xa301, - CodecExifA301SceneType{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A302_cfa_pattern.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A302_cfa_pattern.go deleted file mode 100644 index beca78c23..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_A302_cfa_pattern.go +++ /dev/null @@ -1,97 +0,0 @@ -package exifundefined - -import ( - "bytes" - "fmt" - - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type TagA302CfaPattern struct { - HorizontalRepeat uint16 - VerticalRepeat uint16 - CfaValue []byte -} - -func (TagA302CfaPattern) EncoderName() string { - return "CodecA302CfaPattern" -} - -func (cp TagA302CfaPattern) String() string { - return fmt.Sprintf("TagA302CfaPattern", cp.HorizontalRepeat, cp.VerticalRepeat, len(cp.CfaValue)) -} - -type CodecA302CfaPattern struct { -} - -func (CodecA302CfaPattern) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test. - - cp, ok := value.(TagA302CfaPattern) - if ok == false { - log.Panicf("can only encode a TagA302CfaPattern") - } - - b := new(bytes.Buffer) - - err = binary.Write(b, byteOrder, cp.HorizontalRepeat) - log.PanicIf(err) - - err = binary.Write(b, byteOrder, cp.VerticalRepeat) - log.PanicIf(err) - - _, err = b.Write(cp.CfaValue) - log.PanicIf(err) - - encoded = b.Bytes() - - // TODO(dustin): Confirm this size against the specification. - - return encoded, uint32(len(encoded)), nil -} - -func (CodecA302CfaPattern) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test using known good data. - - valueContext.SetUndefinedValueType(exifcommon.TypeByte) - - valueBytes, err := valueContext.ReadBytes() - log.PanicIf(err) - - cp := TagA302CfaPattern{} - - cp.HorizontalRepeat = valueContext.ByteOrder().Uint16(valueBytes[0:2]) - cp.VerticalRepeat = valueContext.ByteOrder().Uint16(valueBytes[2:4]) - - expectedLength := int(cp.HorizontalRepeat * cp.VerticalRepeat) - cp.CfaValue = valueBytes[4 : 4+expectedLength] - - return cp, nil -} - -func init() { - registerEncoder( - TagA302CfaPattern{}, - CodecA302CfaPattern{}) - - registerDecoder( - exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), - 0xa302, - CodecA302CfaPattern{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_iop_0002_interop_version.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_iop_0002_interop_version.go deleted file mode 100644 index eca046b05..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/exif_iop_0002_interop_version.go +++ /dev/null @@ -1,69 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag0002InteropVersion struct { - InteropVersion string -} - -func (Tag0002InteropVersion) EncoderName() string { - return "Codec0002InteropVersion" -} - -func (iv Tag0002InteropVersion) String() string { - return iv.InteropVersion -} - -type Codec0002InteropVersion struct { -} - -func (Codec0002InteropVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s, ok := value.(Tag0002InteropVersion) - if ok == false { - log.Panicf("can only encode a Tag0002InteropVersion") - } - - return []byte(s.InteropVersion), uint32(len(s.InteropVersion)), nil -} - -func (Codec0002InteropVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) - - valueString, err := valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - iv := Tag0002InteropVersion{ - InteropVersion: valueString, - } - - return iv, nil -} - -func init() { - registerEncoder( - Tag0002InteropVersion{}, - Codec0002InteropVersion{}) - - registerDecoder( - exifcommon.IfdExifIopStandardIfdIdentity.UnindexedString(), - 0x0002, - Codec0002InteropVersion{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001B_gps_processing_method.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001B_gps_processing_method.go deleted file mode 100644 index 8583bfb27..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001B_gps_processing_method.go +++ /dev/null @@ -1,65 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag001BGPSProcessingMethod struct { - string -} - -func (Tag001BGPSProcessingMethod) EncoderName() string { - return "Codec001BGPSProcessingMethod" -} - -func (gpm Tag001BGPSProcessingMethod) String() string { - return gpm.string -} - -type Codec001BGPSProcessingMethod struct { -} - -func (Codec001BGPSProcessingMethod) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s, ok := value.(Tag001BGPSProcessingMethod) - if ok == false { - log.Panicf("can only encode a Tag001BGPSProcessingMethod") - } - - return []byte(s.string), uint32(len(s.string)), nil -} - -func (Codec001BGPSProcessingMethod) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) - - valueString, err := valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - return Tag001BGPSProcessingMethod{valueString}, nil -} - -func init() { - registerEncoder( - Tag001BGPSProcessingMethod{}, - Codec001BGPSProcessingMethod{}) - - registerDecoder( - exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString(), - 0x001b, - Codec001BGPSProcessingMethod{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001C_gps_area_information.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001C_gps_area_information.go deleted file mode 100644 index 67acceb65..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/gps_001C_gps_area_information.go +++ /dev/null @@ -1,65 +0,0 @@ -package exifundefined - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" -) - -type Tag001CGPSAreaInformation struct { - string -} - -func (Tag001CGPSAreaInformation) EncoderName() string { - return "Codec001CGPSAreaInformation" -} - -func (gai Tag001CGPSAreaInformation) String() string { - return gai.string -} - -type Codec001CGPSAreaInformation struct { -} - -func (Codec001CGPSAreaInformation) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s, ok := value.(Tag001CGPSAreaInformation) - if ok == false { - log.Panicf("can only encode a Tag001CGPSAreaInformation") - } - - return []byte(s.string), uint32(len(s.string)), nil -} - -func (Codec001CGPSAreaInformation) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) - - valueString, err := valueContext.ReadAsciiNoNul() - log.PanicIf(err) - - return Tag001CGPSAreaInformation{valueString}, nil -} - -func init() { - registerEncoder( - Tag001CGPSAreaInformation{}, - Codec001CGPSAreaInformation{}) - - registerDecoder( - exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString(), - 0x001c, - Codec001CGPSAreaInformation{}) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/registration.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/registration.go deleted file mode 100644 index cccc20a82..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/registration.go +++ /dev/null @@ -1,42 +0,0 @@ -package exifundefined - -import ( - "github.com/dsoprea/go-logging" -) - -// UndefinedTagHandle defines one undefined-type tag with a corresponding -// decoder. -type UndefinedTagHandle struct { - IfdPath string - TagId uint16 -} - -func registerEncoder(entity EncodeableValue, encoder UndefinedValueEncoder) { - typeName := entity.EncoderName() - - _, found := encoders[typeName] - if found == true { - log.Panicf("encoder already registered: %v", typeName) - } - - encoders[typeName] = encoder -} - -func registerDecoder(ifdPath string, tagId uint16, decoder UndefinedValueDecoder) { - uth := UndefinedTagHandle{ - IfdPath: ifdPath, - TagId: tagId, - } - - _, found := decoders[uth] - if found == true { - log.Panicf("decoder already registered: %v", uth) - } - - decoders[uth] = decoder -} - -var ( - encoders = make(map[string]UndefinedValueEncoder) - decoders = make(map[UndefinedTagHandle]UndefinedValueDecoder) -) diff --git a/vendor/github.com/dsoprea/go-exif/v2/undefined/type.go b/vendor/github.com/dsoprea/go-exif/v2/undefined/type.go deleted file mode 100644 index 29890ef86..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/undefined/type.go +++ /dev/null @@ -1,44 +0,0 @@ -package exifundefined - -import ( - "errors" - - "encoding/binary" - - "github.com/dsoprea/go-exif/v2/common" -) - -const ( - // UnparseableUnknownTagValuePlaceholder is the string to use for an unknown - // undefined tag. - UnparseableUnknownTagValuePlaceholder = "!UNKNOWN" - - // UnparseableHandledTagValuePlaceholder is the string to use for a known - // value that is not parseable. - UnparseableHandledTagValuePlaceholder = "!MALFORMED" -) - -var ( - // ErrUnparseableValue is the error for a value that we should have been - // able to parse but were not able to. - ErrUnparseableValue = errors.New("unparseable undefined tag") -) - -// UndefinedValueEncoder knows how to encode an undefined-type tag's value to -// bytes. -type UndefinedValueEncoder interface { - Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) -} - -// EncodeableValue wraps a value with the information that will be needed to re- -// encode it later. -type EncodeableValue interface { - EncoderName() string - String() string -} - -// UndefinedValueDecoder knows how to decode an undefined-type tag's value from -// bytes. -type UndefinedValueDecoder interface { - Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) -} diff --git a/vendor/github.com/dsoprea/go-exif/v2/utility.go b/vendor/github.com/dsoprea/go-exif/v2/utility.go deleted file mode 100644 index ad692477e..000000000 --- a/vendor/github.com/dsoprea/go-exif/v2/utility.go +++ /dev/null @@ -1,233 +0,0 @@ -package exif - -import ( - "fmt" - "math" - "reflect" - "strconv" - "strings" - "time" - - "github.com/dsoprea/go-logging" - - "github.com/dsoprea/go-exif/v2/common" - "github.com/dsoprea/go-exif/v2/undefined" -) - -var ( - utilityLogger = log.NewLogger("exif.utility") -) - -var ( - timeType = reflect.TypeOf(time.Time{}) -) - -// ParseExifFullTimestamp parses dates like "2018:11:30 13:01:49" into a UTC -// `time.Time` struct. -func ParseExifFullTimestamp(fullTimestampPhrase string) (timestamp time.Time, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - parts := strings.Split(fullTimestampPhrase, " ") - datestampValue, timestampValue := parts[0], parts[1] - - // Normalize the separators. - datestampValue = strings.ReplaceAll(datestampValue, "-", ":") - timestampValue = strings.ReplaceAll(timestampValue, "-", ":") - - dateParts := strings.Split(datestampValue, ":") - - year, err := strconv.ParseUint(dateParts[0], 10, 16) - if err != nil { - log.Panicf("could not parse year") - } - - month, err := strconv.ParseUint(dateParts[1], 10, 8) - if err != nil { - log.Panicf("could not parse month") - } - - day, err := strconv.ParseUint(dateParts[2], 10, 8) - if err != nil { - log.Panicf("could not parse day") - } - - timeParts := strings.Split(timestampValue, ":") - - hour, err := strconv.ParseUint(timeParts[0], 10, 8) - if err != nil { - log.Panicf("could not parse hour") - } - - minute, err := strconv.ParseUint(timeParts[1], 10, 8) - if err != nil { - log.Panicf("could not parse minute") - } - - second, err := strconv.ParseUint(timeParts[2], 10, 8) - if err != nil { - log.Panicf("could not parse second") - } - - timestamp = time.Date(int(year), time.Month(month), int(day), int(hour), int(minute), int(second), 0, time.UTC) - return timestamp, nil -} - -// ExifFullTimestampString produces a string like "2018:11:30 13:01:49" from a -// `time.Time` struct. It will attempt to convert to UTC first. -func ExifFullTimestampString(t time.Time) (fullTimestampPhrase string) { - return exifcommon.ExifFullTimestampString(t) -} - -// ExifTag is one simple representation of a tag in a flat list of all of them. -type ExifTag struct { - // IfdPath is the fully-qualified IFD path (even though it is not named as - // such). - IfdPath string `json:"ifd_path"` - - // TagId is the tag-ID. - TagId uint16 `json:"id"` - - // TagName is the tag-name. This is never empty. - TagName string `json:"name"` - - // UnitCount is the recorded number of units constution of the value. - UnitCount uint32 `json:"unit_count"` - - // TagTypeId is the type-ID. - TagTypeId exifcommon.TagTypePrimitive `json:"type_id"` - - // TagTypeName is the type name. - TagTypeName string `json:"type_name"` - - // Value is the decoded value. - Value interface{} `json:"value"` - - // ValueBytes is the raw, encoded value. - ValueBytes []byte `json:"value_bytes"` - - // Formatted is the human representation of the first value (tag values are - // always an array). - FormattedFirst string `json:"formatted_first"` - - // Formatted is the human representation of the complete value. - Formatted string `json:"formatted"` - - // ChildIfdPath is the name of the child IFD this tag represents (if it - // represents any). Otherwise, this is empty. - ChildIfdPath string `json:"child_ifd_path"` -} - -// String returns a string representation. -func (et ExifTag) String() string { - return fmt.Sprintf( - "ExifTag<"+ - "IFD-PATH=[%s] "+ - "TAG-ID=(0x%02x) "+ - "TAG-NAME=[%s] "+ - "TAG-TYPE=[%s] "+ - "VALUE=[%v] "+ - "VALUE-BYTES=(%d) "+ - "CHILD-IFD-PATH=[%s]", - et.IfdPath, et.TagId, et.TagName, et.TagTypeName, et.FormattedFirst, - len(et.ValueBytes), et.ChildIfdPath) -} - -// GetFlatExifData returns a simple, flat representation of all tags. -func GetFlatExifData(exifData []byte) (exifTags []ExifTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - eh, err := ParseExifHeader(exifData) - log.PanicIf(err) - - im := NewIfdMappingWithStandard() - ti := NewTagIndex() - - ie := NewIfdEnumerate(im, ti, exifData, eh.ByteOrder) - - exifTags = make([]ExifTag, 0) - - visitor := func(fqIfdPath string, ifdIndex int, ite *IfdTagEntry) (err error) { - // This encodes down to base64. Since this an example tool and we do not - // expect to ever decode the output, we are not worried about - // specifically base64-encoding it in order to have a measure of - // control. - valueBytes, err := ite.GetRawBytes() - if err != nil { - if err == exifundefined.ErrUnparseableValue { - return nil - } - - log.Panic(err) - } - - value, err := ite.Value() - if err != nil { - if err == exifcommon.ErrUnhandledUndefinedTypedTag { - value = exifundefined.UnparseableUnknownTagValuePlaceholder - } else { - log.Panic(err) - } - } - - et := ExifTag{ - IfdPath: fqIfdPath, - TagId: ite.TagId(), - TagName: ite.TagName(), - UnitCount: ite.UnitCount(), - TagTypeId: ite.TagType(), - TagTypeName: ite.TagType().String(), - Value: value, - ValueBytes: valueBytes, - ChildIfdPath: ite.ChildIfdPath(), - } - - et.Formatted, err = ite.Format() - log.PanicIf(err) - - et.FormattedFirst, err = ite.FormatFirst() - log.PanicIf(err) - - exifTags = append(exifTags, et) - - return nil - } - - _, err = ie.Scan(exifcommon.IfdStandardIfdIdentity, eh.FirstIfdOffset, visitor) - log.PanicIf(err) - - return exifTags, nil -} - -// GpsDegreesEquals returns true if the two `GpsDegrees` are identical. -func GpsDegreesEquals(gi1, gi2 GpsDegrees) bool { - if gi2.Orientation != gi1.Orientation { - return false - } - - degreesRightBound := math.Nextafter(gi1.Degrees, gi1.Degrees+1) - minutesRightBound := math.Nextafter(gi1.Minutes, gi1.Minutes+1) - secondsRightBound := math.Nextafter(gi1.Seconds, gi1.Seconds+1) - - if gi2.Degrees < gi1.Degrees || gi2.Degrees >= degreesRightBound { - return false - } else if gi2.Minutes < gi1.Minutes || gi2.Minutes >= minutesRightBound { - return false - } else if gi2.Seconds < gi1.Seconds || gi2.Seconds >= secondsRightBound { - return false - } - - return true -} - -// IsTime returns true if the value is a `time.Time`. -func IsTime(v interface{}) bool { - return reflect.TypeOf(v) == timeType -} diff --git a/vendor/github.com/dsoprea/go-exif/v3/.MODULE_ROOT b/vendor/github.com/dsoprea/go-exif/v3/.MODULE_ROOT new file mode 100644 index 000000000..e69de29bb diff --git a/vendor/github.com/dsoprea/go-exif/v3/LICENSE b/vendor/github.com/dsoprea/go-exif/v3/LICENSE new file mode 100644 index 000000000..0b9358a3a --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/LICENSE @@ -0,0 +1,9 @@ +MIT LICENSE + +Copyright 2019 Dustin Oprea + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/ifd.go b/vendor/github.com/dsoprea/go-exif/v3/common/ifd.go new file mode 100644 index 000000000..01886e966 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/ifd.go @@ -0,0 +1,651 @@ +package exifcommon + +import ( + "errors" + "fmt" + "strings" + + "github.com/dsoprea/go-logging" +) + +var ( + ifdLogger = log.NewLogger("exifcommon.ifd") +) + +var ( + ErrChildIfdNotMapped = errors.New("no child-IFD for that tag-ID under parent") +) + +// MappedIfd is one node in the IFD-mapping. +type MappedIfd struct { + ParentTagId uint16 + Placement []uint16 + Path []string + + Name string + TagId uint16 + Children map[uint16]*MappedIfd +} + +// String returns a descriptive string. +func (mi *MappedIfd) String() string { + pathPhrase := mi.PathPhrase() + return fmt.Sprintf("MappedIfd<(0x%04X) [%s] PATH=[%s]>", mi.TagId, mi.Name, pathPhrase) +} + +// PathPhrase returns a non-fully-qualified IFD path. +func (mi *MappedIfd) PathPhrase() string { + return strings.Join(mi.Path, "/") +} + +// TODO(dustin): Refactor this to use IfdIdentity structs. + +// IfdMapping describes all of the IFDs that we currently recognize. +type IfdMapping struct { + rootNode *MappedIfd +} + +// NewIfdMapping returns a new IfdMapping struct. +func NewIfdMapping() (ifdMapping *IfdMapping) { + rootNode := &MappedIfd{ + Path: make([]string, 0), + Children: make(map[uint16]*MappedIfd), + } + + return &IfdMapping{ + rootNode: rootNode, + } +} + +// NewIfdMappingWithStandard retruns a new IfdMapping struct preloaded with the +// standard IFDs. +func NewIfdMappingWithStandard() (ifdMapping *IfdMapping, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + im := NewIfdMapping() + + err = LoadStandardIfds(im) + log.PanicIf(err) + + return im, nil +} + +// Get returns the node given the path slice. +func (im *IfdMapping) Get(parentPlacement []uint16) (childIfd *MappedIfd, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ptr := im.rootNode + for _, tagId := range parentPlacement { + if descendantPtr, found := ptr.Children[tagId]; found == false { + log.Panicf("ifd child with tag-ID (%04x) not registered: [%s]", tagId, ptr.PathPhrase()) + } else { + ptr = descendantPtr + } + } + + return ptr, nil +} + +// GetWithPath returns the node given the path string. +func (im *IfdMapping) GetWithPath(pathPhrase string) (mi *MappedIfd, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if pathPhrase == "" { + log.Panicf("path-phrase is empty") + } + + path := strings.Split(pathPhrase, "/") + ptr := im.rootNode + + for _, name := range path { + var hit *MappedIfd + for _, mi := range ptr.Children { + if mi.Name == name { + hit = mi + break + } + } + + if hit == nil { + log.Panicf("ifd child with name [%s] not registered: [%s]", name, ptr.PathPhrase()) + } + + ptr = hit + } + + return ptr, nil +} + +// GetChild is a convenience function to get the child path for a given parent +// placement and child tag-ID. +func (im *IfdMapping) GetChild(parentPathPhrase string, tagId uint16) (mi *MappedIfd, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + mi, err = im.GetWithPath(parentPathPhrase) + log.PanicIf(err) + + for _, childMi := range mi.Children { + if childMi.TagId == tagId { + return childMi, nil + } + } + + // Whether or not an IFD is defined in data, such an IFD is not registered + // and would be unknown. + log.Panic(ErrChildIfdNotMapped) + return nil, nil +} + +// IfdTagIdAndIndex represents a specific part of the IFD path. +// +// This is a legacy type. +type IfdTagIdAndIndex struct { + Name string + TagId uint16 + Index int +} + +// String returns a descriptive string. +func (itii IfdTagIdAndIndex) String() string { + return fmt.Sprintf("IfdTagIdAndIndex", itii.Name, itii.TagId, itii.Index) +} + +// ResolvePath takes a list of names, which can also be suffixed with indices +// (to identify the second, third, etc.. sibling IFD) and returns a list of +// tag-IDs and those indices. +// +// Example: +// +// - IFD/Exif/Iop +// - IFD0/Exif/Iop +// +// This is the only call that supports adding the numeric indices. +func (im *IfdMapping) ResolvePath(pathPhrase string) (lineage []IfdTagIdAndIndex, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + pathPhrase = strings.TrimSpace(pathPhrase) + + if pathPhrase == "" { + log.Panicf("can not resolve empty path-phrase") + } + + path := strings.Split(pathPhrase, "/") + lineage = make([]IfdTagIdAndIndex, len(path)) + + ptr := im.rootNode + empty := IfdTagIdAndIndex{} + for i, name := range path { + indexByte := name[len(name)-1] + index := 0 + if indexByte >= '0' && indexByte <= '9' { + index = int(indexByte - '0') + name = name[:len(name)-1] + } + + itii := IfdTagIdAndIndex{} + for _, mi := range ptr.Children { + if mi.Name != name { + continue + } + + itii.Name = name + itii.TagId = mi.TagId + itii.Index = index + + ptr = mi + + break + } + + if itii == empty { + log.Panicf("ifd child with name [%s] not registered: [%s]", name, pathPhrase) + } + + lineage[i] = itii + } + + return lineage, nil +} + +// FqPathPhraseFromLineage returns the fully-qualified IFD path from the slice. +func (im *IfdMapping) FqPathPhraseFromLineage(lineage []IfdTagIdAndIndex) (fqPathPhrase string) { + fqPathParts := make([]string, len(lineage)) + for i, itii := range lineage { + if itii.Index > 0 { + fqPathParts[i] = fmt.Sprintf("%s%d", itii.Name, itii.Index) + } else { + fqPathParts[i] = itii.Name + } + } + + return strings.Join(fqPathParts, "/") +} + +// PathPhraseFromLineage returns the non-fully-qualified IFD path from the +// slice. +func (im *IfdMapping) PathPhraseFromLineage(lineage []IfdTagIdAndIndex) (pathPhrase string) { + pathParts := make([]string, len(lineage)) + for i, itii := range lineage { + pathParts[i] = itii.Name + } + + return strings.Join(pathParts, "/") +} + +// StripPathPhraseIndices returns a non-fully-qualified path-phrase (no +// indices). +func (im *IfdMapping) StripPathPhraseIndices(pathPhrase string) (strippedPathPhrase string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + lineage, err := im.ResolvePath(pathPhrase) + log.PanicIf(err) + + strippedPathPhrase = im.PathPhraseFromLineage(lineage) + return strippedPathPhrase, nil +} + +// Add puts the given IFD at the given position of the tree. The position of the +// tree is referred to as the placement and is represented by a set of tag-IDs, +// where the leftmost is the root tag and the tags going to the right are +// progressive descendants. +func (im *IfdMapping) Add(parentPlacement []uint16, tagId uint16, name string) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! It would be nicer to provide a list of names in the placement rather than tag-IDs. + + ptr, err := im.Get(parentPlacement) + log.PanicIf(err) + + path := make([]string, len(parentPlacement)+1) + if len(parentPlacement) > 0 { + copy(path, ptr.Path) + } + + path[len(path)-1] = name + + placement := make([]uint16, len(parentPlacement)+1) + if len(placement) > 0 { + copy(placement, ptr.Placement) + } + + placement[len(placement)-1] = tagId + + childIfd := &MappedIfd{ + ParentTagId: ptr.TagId, + Path: path, + Placement: placement, + Name: name, + TagId: tagId, + Children: make(map[uint16]*MappedIfd), + } + + if _, found := ptr.Children[tagId]; found == true { + log.Panicf("child IFD with tag-ID (%04x) already registered under IFD [%s] with tag-ID (%04x)", tagId, ptr.Name, ptr.TagId) + } + + ptr.Children[tagId] = childIfd + + return nil +} + +func (im *IfdMapping) dumpLineages(stack []*MappedIfd, input []string) (output []string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + currentIfd := stack[len(stack)-1] + + output = input + for _, childIfd := range currentIfd.Children { + stackCopy := make([]*MappedIfd, len(stack)+1) + + copy(stackCopy, stack) + stackCopy[len(stack)] = childIfd + + // Add to output, but don't include the obligatory root node. + parts := make([]string, len(stackCopy)-1) + for i, mi := range stackCopy[1:] { + parts[i] = mi.Name + } + + output = append(output, strings.Join(parts, "/")) + + output, err = im.dumpLineages(stackCopy, output) + log.PanicIf(err) + } + + return output, nil +} + +// DumpLineages returns a slice of strings representing all mappings. +func (im *IfdMapping) DumpLineages() (output []string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + stack := []*MappedIfd{im.rootNode} + output = make([]string, 0) + + output, err = im.dumpLineages(stack, output) + log.PanicIf(err) + + return output, nil +} + +// LoadStandardIfds loads the standard IFDs into the mapping. +func LoadStandardIfds(im *IfdMapping) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = im.Add( + []uint16{}, + IfdStandardIfdIdentity.TagId(), IfdStandardIfdIdentity.Name()) + + log.PanicIf(err) + + err = im.Add( + []uint16{IfdStandardIfdIdentity.TagId()}, + IfdExifStandardIfdIdentity.TagId(), IfdExifStandardIfdIdentity.Name()) + + log.PanicIf(err) + + err = im.Add( + []uint16{IfdStandardIfdIdentity.TagId(), IfdExifStandardIfdIdentity.TagId()}, + IfdExifIopStandardIfdIdentity.TagId(), IfdExifIopStandardIfdIdentity.Name()) + + log.PanicIf(err) + + err = im.Add( + []uint16{IfdStandardIfdIdentity.TagId()}, + IfdGpsInfoStandardIfdIdentity.TagId(), IfdGpsInfoStandardIfdIdentity.Name()) + + log.PanicIf(err) + + return nil +} + +// IfdTag describes a single IFD tag and its parent (if any). +type IfdTag struct { + parentIfdTag *IfdTag + tagId uint16 + name string +} + +func NewIfdTag(parentIfdTag *IfdTag, tagId uint16, name string) IfdTag { + return IfdTag{ + parentIfdTag: parentIfdTag, + tagId: tagId, + name: name, + } +} + +// ParentIfd returns the IfdTag of this IFD's parent. +func (it IfdTag) ParentIfd() *IfdTag { + return it.parentIfdTag +} + +// TagId returns the tag-ID of this IFD. +func (it IfdTag) TagId() uint16 { + return it.tagId +} + +// Name returns the simple name of this IFD. +func (it IfdTag) Name() string { + return it.name +} + +// String returns a descriptive string. +func (it IfdTag) String() string { + parentIfdPhrase := "" + if it.parentIfdTag != nil { + parentIfdPhrase = fmt.Sprintf(" PARENT=(0x%04x)[%s]", it.parentIfdTag.tagId, it.parentIfdTag.name) + } + + return fmt.Sprintf("IfdTag", it.tagId, it.name, parentIfdPhrase) +} + +var ( + // rootStandardIfd is the standard root IFD. + rootStandardIfd = NewIfdTag(nil, 0x0000, "IFD") // IFD + + // exifStandardIfd is the standard "Exif" IFD. + exifStandardIfd = NewIfdTag(&rootStandardIfd, 0x8769, "Exif") // IFD/Exif + + // iopStandardIfd is the standard "Iop" IFD. + iopStandardIfd = NewIfdTag(&exifStandardIfd, 0xA005, "Iop") // IFD/Exif/Iop + + // gpsInfoStandardIfd is the standard "GPS" IFD. + gpsInfoStandardIfd = NewIfdTag(&rootStandardIfd, 0x8825, "GPSInfo") // IFD/GPSInfo +) + +// IfdIdentityPart represents one component in an IFD path. +type IfdIdentityPart struct { + Name string + Index int +} + +// String returns a fully-qualified IFD path. +func (iip IfdIdentityPart) String() string { + if iip.Index > 0 { + return fmt.Sprintf("%s%d", iip.Name, iip.Index) + } else { + return iip.Name + } +} + +// UnindexedString returned a non-fully-qualified IFD path. +func (iip IfdIdentityPart) UnindexedString() string { + return iip.Name +} + +// IfdIdentity represents a single IFD path and provides access to various +// information and representations. +// +// Only global instances can be used for equality checks. +type IfdIdentity struct { + ifdTag IfdTag + parts []IfdIdentityPart + ifdPath string + fqIfdPath string +} + +// NewIfdIdentity returns a new IfdIdentity struct. +func NewIfdIdentity(ifdTag IfdTag, parts ...IfdIdentityPart) (ii *IfdIdentity) { + ii = &IfdIdentity{ + ifdTag: ifdTag, + parts: parts, + } + + ii.ifdPath = ii.getIfdPath() + ii.fqIfdPath = ii.getFqIfdPath() + + return ii +} + +// NewIfdIdentityFromString parses a string like "IFD/Exif" or "IFD1" or +// something more exotic with custom IFDs ("SomeIFD4/SomeChildIFD6"). Note that +// this will valid the unindexed IFD structure (because the standard tags from +// the specification are unindexed), but not, obviously, any indices (e.g. +// the numbers in "IFD0", "IFD1", "SomeIFD4/SomeChildIFD6"). It is +// required for the caller to check whether these specific instances +// were actually parsed out of the stream. +func NewIfdIdentityFromString(im *IfdMapping, fqIfdPath string) (ii *IfdIdentity, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + lineage, err := im.ResolvePath(fqIfdPath) + log.PanicIf(err) + + var lastIt *IfdTag + identityParts := make([]IfdIdentityPart, len(lineage)) + for i, itii := range lineage { + // Build out the tag that will eventually point to the IFD represented + // by the right-most part in the IFD path. + + it := &IfdTag{ + parentIfdTag: lastIt, + tagId: itii.TagId, + name: itii.Name, + } + + lastIt = it + + // Create the next IfdIdentity part. + + iip := IfdIdentityPart{ + Name: itii.Name, + Index: itii.Index, + } + + identityParts[i] = iip + } + + ii = NewIfdIdentity(*lastIt, identityParts...) + return ii, nil +} + +func (ii *IfdIdentity) getFqIfdPath() string { + partPhrases := make([]string, len(ii.parts)) + for i, iip := range ii.parts { + partPhrases[i] = iip.String() + } + + return strings.Join(partPhrases, "/") +} + +func (ii *IfdIdentity) getIfdPath() string { + partPhrases := make([]string, len(ii.parts)) + for i, iip := range ii.parts { + partPhrases[i] = iip.UnindexedString() + } + + return strings.Join(partPhrases, "/") +} + +// String returns a fully-qualified IFD path. +func (ii *IfdIdentity) String() string { + return ii.fqIfdPath +} + +// UnindexedString returns a non-fully-qualified IFD path. +func (ii *IfdIdentity) UnindexedString() string { + return ii.ifdPath +} + +// IfdTag returns the tag struct behind this IFD. +func (ii *IfdIdentity) IfdTag() IfdTag { + return ii.ifdTag +} + +// TagId returns the tag-ID of the IFD. +func (ii *IfdIdentity) TagId() uint16 { + return ii.ifdTag.TagId() +} + +// LeafPathPart returns the last right-most path-part, which represents the +// current IFD. +func (ii *IfdIdentity) LeafPathPart() IfdIdentityPart { + return ii.parts[len(ii.parts)-1] +} + +// Name returns the simple name of this IFD. +func (ii *IfdIdentity) Name() string { + return ii.LeafPathPart().Name +} + +// Index returns the index of this IFD (more then one IFD under a parent IFD +// will be numbered [0..n]). +func (ii *IfdIdentity) Index() int { + return ii.LeafPathPart().Index +} + +// Equals returns true if the two IfdIdentity instances are effectively +// identical. +// +// Since there's no way to get a specific fully-qualified IFD path without a +// certain slice of parts and all other fields are also derived from this, +// checking that the fully-qualified IFD path is equals is sufficient. +func (ii *IfdIdentity) Equals(ii2 *IfdIdentity) bool { + return ii.String() == ii2.String() +} + +// NewChild creates an IfdIdentity for an IFD that is a child of the current +// IFD. +func (ii *IfdIdentity) NewChild(childIfdTag IfdTag, index int) (iiChild *IfdIdentity) { + if *childIfdTag.parentIfdTag != ii.ifdTag { + log.Panicf("can not add child; we are not the parent:\nUS=%v\nCHILD=%v", ii.ifdTag, childIfdTag) + } + + childPart := IfdIdentityPart{childIfdTag.name, index} + childParts := append(ii.parts, childPart) + + iiChild = NewIfdIdentity(childIfdTag, childParts...) + return iiChild +} + +// NewSibling creates an IfdIdentity for an IFD that is a sibling to the current +// one. +func (ii *IfdIdentity) NewSibling(index int) (iiSibling *IfdIdentity) { + parts := make([]IfdIdentityPart, len(ii.parts)) + + copy(parts, ii.parts) + parts[len(parts)-1].Index = index + + iiSibling = NewIfdIdentity(ii.ifdTag, parts...) + return iiSibling +} + +var ( + // IfdStandardIfdIdentity represents the IFD path for IFD0. + IfdStandardIfdIdentity = NewIfdIdentity(rootStandardIfd, IfdIdentityPart{"IFD", 0}) + + // IfdExifStandardIfdIdentity represents the IFD path for IFD0/Exif0. + IfdExifStandardIfdIdentity = IfdStandardIfdIdentity.NewChild(exifStandardIfd, 0) + + // IfdExifIopStandardIfdIdentity represents the IFD path for IFD0/Exif0/Iop0. + IfdExifIopStandardIfdIdentity = IfdExifStandardIfdIdentity.NewChild(iopStandardIfd, 0) + + // IfdGPSInfoStandardIfdIdentity represents the IFD path for IFD0/GPSInfo0. + IfdGpsInfoStandardIfdIdentity = IfdStandardIfdIdentity.NewChild(gpsInfoStandardIfd, 0) + + // Ifd1StandardIfdIdentity represents the IFD path for IFD1. + Ifd1StandardIfdIdentity = NewIfdIdentity(rootStandardIfd, IfdIdentityPart{"IFD", 1}) +) diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/parser.go b/vendor/github.com/dsoprea/go-exif/v3/common/parser.go new file mode 100644 index 000000000..76e8ef425 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/parser.go @@ -0,0 +1,280 @@ +package exifcommon + +import ( + "bytes" + "errors" + "math" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +var ( + parserLogger = log.NewLogger("exifcommon.parser") +) + +var ( + ErrParseFail = errors.New("parse failure") +) + +// Parser knows how to parse all well-defined, encoded EXIF types. +type Parser struct { +} + +// ParseBytesknows how to parse a byte-type value. +func (p *Parser) ParseBytes(data []byte, unitCount uint32) (value []uint8, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeByte.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = []uint8(data[:count]) + + return value, nil +} + +// ParseAscii returns a string and auto-strips the trailing NUL character that +// should be at the end of the encoding. +func (p *Parser) ParseAscii(data []byte, unitCount uint32) (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeAscii.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + if len(data) == 0 || data[count-1] != 0 { + s := string(data[:count]) + parserLogger.Warningf(nil, "ASCII not terminated with NUL as expected: [%v]", s) + + for i, c := range s { + if c > 127 { + // Binary + + t := s[:i] + parserLogger.Warningf(nil, "ASCII also had binary characters. Truncating: [%v]->[%s]", s, t) + + return t, nil + } + } + + return s, nil + } + + // Auto-strip the NUL from the end. It serves no purpose outside of + // encoding semantics. + + return string(data[:count-1]), nil +} + +// ParseAsciiNoNul returns a string without any consideration for a trailing NUL +// character. +func (p *Parser) ParseAsciiNoNul(data []byte, unitCount uint32) (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeAscii.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + return string(data[:count]), nil +} + +// ParseShorts knows how to parse an encoded list of shorts. +func (p *Parser) ParseShorts(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint16, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeShort.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = make([]uint16, count) + for i := 0; i < count; i++ { + value[i] = byteOrder.Uint16(data[i*2:]) + } + + return value, nil +} + +// ParseLongs knows how to encode an encoded list of unsigned longs. +func (p *Parser) ParseLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeLong.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = make([]uint32, count) + for i := 0; i < count; i++ { + value[i] = byteOrder.Uint32(data[i*4:]) + } + + return value, nil +} + +// ParseFloats knows how to encode an encoded list of floats. +func (p *Parser) ParseFloats(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []float32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + count := int(unitCount) + + if len(data) != (TypeFloat.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = make([]float32, count) + for i := 0; i < count; i++ { + value[i] = math.Float32frombits(byteOrder.Uint32(data[i*4 : (i+1)*4])) + } + + return value, nil +} + +// ParseDoubles knows how to encode an encoded list of doubles. +func (p *Parser) ParseDoubles(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []float64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + count := int(unitCount) + + if len(data) != (TypeDouble.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = make([]float64, count) + for i := 0; i < count; i++ { + value[i] = math.Float64frombits(byteOrder.Uint64(data[i*8 : (i+1)*8])) + } + + return value, nil +} + +// ParseRationals knows how to parse an encoded list of unsigned rationals. +func (p *Parser) ParseRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []Rational, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeRational.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + value = make([]Rational, count) + for i := 0; i < count; i++ { + value[i].Numerator = byteOrder.Uint32(data[i*8:]) + value[i].Denominator = byteOrder.Uint32(data[i*8+4:]) + } + + return value, nil +} + +// ParseSignedLongs knows how to parse an encoded list of signed longs. +func (p *Parser) ParseSignedLongs(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []int32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeSignedLong.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + b := bytes.NewBuffer(data) + + value = make([]int32, count) + for i := 0; i < count; i++ { + err := binary.Read(b, byteOrder, &value[i]) + log.PanicIf(err) + } + + return value, nil +} + +// ParseSignedRationals knows how to parse an encoded list of signed +// rationals. +func (p *Parser) ParseSignedRationals(data []byte, unitCount uint32, byteOrder binary.ByteOrder) (value []SignedRational, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + count := int(unitCount) + + if len(data) < (TypeSignedRational.Size() * count) { + log.Panic(ErrNotEnoughData) + } + + b := bytes.NewBuffer(data) + + value = make([]SignedRational, count) + for i := 0; i < count; i++ { + err = binary.Read(b, byteOrder, &value[i].Numerator) + log.PanicIf(err) + + err = binary.Read(b, byteOrder, &value[i].Denominator) + log.PanicIf(err) + } + + return value, nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/testing_common.go b/vendor/github.com/dsoprea/go-exif/v3/common/testing_common.go new file mode 100644 index 000000000..f04fa22b6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/testing_common.go @@ -0,0 +1,88 @@ +package exifcommon + +import ( + "os" + "path" + + "encoding/binary" + "io/ioutil" + + "github.com/dsoprea/go-logging" +) + +var ( + moduleRootPath = "" + + testExifData []byte = nil + + // EncodeDefaultByteOrder is the default byte-order for encoding operations. + EncodeDefaultByteOrder = binary.BigEndian + + // Default byte order for tests. + TestDefaultByteOrder = binary.BigEndian +) + +func GetModuleRootPath() string { + if moduleRootPath == "" { + moduleRootPath = os.Getenv("EXIF_MODULE_ROOT_PATH") + if moduleRootPath != "" { + return moduleRootPath + } + + currentWd, err := os.Getwd() + log.PanicIf(err) + + currentPath := currentWd + + visited := make([]string, 0) + + for { + tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") + + _, err := os.Stat(tryStampFilepath) + if err != nil && os.IsNotExist(err) != true { + log.Panic(err) + } else if err == nil { + break + } + + visited = append(visited, tryStampFilepath) + + currentPath = path.Dir(currentPath) + if currentPath == "/" { + log.Panicf("could not find module-root: %v", visited) + } + } + + moduleRootPath = currentPath + } + + return moduleRootPath +} + +func GetTestAssetsPath() string { + moduleRootPath := GetModuleRootPath() + assetsPath := path.Join(moduleRootPath, "assets") + + return assetsPath +} + +func getTestImageFilepath() string { + assetsPath := GetTestAssetsPath() + testImageFilepath := path.Join(assetsPath, "NDM_8901.jpg") + return testImageFilepath +} + +func getTestExifData() []byte { + if testExifData == nil { + assetsPath := GetTestAssetsPath() + filepath := path.Join(assetsPath, "NDM_8901.jpg.exif") + + var err error + + testExifData, err = ioutil.ReadFile(filepath) + log.PanicIf(err) + } + + return testExifData +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/type.go b/vendor/github.com/dsoprea/go-exif/v3/common/type.go new file mode 100644 index 000000000..e79bcb9a1 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/type.go @@ -0,0 +1,482 @@ +package exifcommon + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "unicode" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +var ( + typeLogger = log.NewLogger("exif.type") +) + +var ( + // ErrNotEnoughData is used when there isn't enough data to accommodate what + // we're trying to parse (sizeof(type) * unit_count). + ErrNotEnoughData = errors.New("not enough data for type") + + // ErrWrongType is used when we try to parse anything other than the + // current type. + ErrWrongType = errors.New("wrong type, can not parse") + + // ErrUnhandledUndefinedTypedTag is used when we try to parse a tag that's + // recorded as an "unknown" type but not a documented tag (therefore + // leaving us not knowning how to read it). + ErrUnhandledUndefinedTypedTag = errors.New("not a standard unknown-typed tag") +) + +// TagTypePrimitive is a type-alias that let's us easily lookup type properties. +type TagTypePrimitive uint16 + +const ( + // TypeByte describes an encoded list of bytes. + TypeByte TagTypePrimitive = 1 + + // TypeAscii describes an encoded list of characters that is terminated + // with a NUL in its encoded form. + TypeAscii TagTypePrimitive = 2 + + // TypeShort describes an encoded list of shorts. + TypeShort TagTypePrimitive = 3 + + // TypeLong describes an encoded list of longs. + TypeLong TagTypePrimitive = 4 + + // TypeRational describes an encoded list of rationals. + TypeRational TagTypePrimitive = 5 + + // TypeUndefined describes an encoded value that has a complex/non-clearcut + // interpretation. + TypeUndefined TagTypePrimitive = 7 + + // We've seen type-8, but have no documentation on it. + + // TypeSignedLong describes an encoded list of signed longs. + TypeSignedLong TagTypePrimitive = 9 + + // TypeSignedRational describes an encoded list of signed rationals. + TypeSignedRational TagTypePrimitive = 10 + + // TypeFloat describes an encoded list of floats + TypeFloat TagTypePrimitive = 11 + + // TypeDouble describes an encoded list of doubles. + TypeDouble TagTypePrimitive = 12 + + // TypeAsciiNoNul is just a pseudo-type, for our own purposes. + TypeAsciiNoNul TagTypePrimitive = 0xf0 +) + +// String returns the name of the type +func (typeType TagTypePrimitive) String() string { + return TypeNames[typeType] +} + +// Size returns the size of one atomic unit of the type. +func (tagType TagTypePrimitive) Size() int { + switch tagType { + case TypeByte, TypeAscii, TypeAsciiNoNul: + return 1 + case TypeShort: + return 2 + case TypeLong, TypeSignedLong, TypeFloat: + return 4 + case TypeRational, TypeSignedRational, TypeDouble: + return 8 + default: + log.Panicf("can not determine tag-value size for type (%d): [%s]", + tagType, + TypeNames[tagType]) + // Never called. + return 0 + } +} + +// IsValid returns true if tagType is a valid type. +func (tagType TagTypePrimitive) IsValid() bool { + + // TODO(dustin): Add test + + return tagType == TypeByte || + tagType == TypeAscii || + tagType == TypeAsciiNoNul || + tagType == TypeShort || + tagType == TypeLong || + tagType == TypeRational || + tagType == TypeSignedLong || + tagType == TypeSignedRational || + tagType == TypeFloat || + tagType == TypeDouble || + tagType == TypeUndefined +} + +var ( + // TODO(dustin): Rename TypeNames() to typeNames() and add getter. + TypeNames = map[TagTypePrimitive]string{ + TypeByte: "BYTE", + TypeAscii: "ASCII", + TypeShort: "SHORT", + TypeLong: "LONG", + TypeRational: "RATIONAL", + TypeUndefined: "UNDEFINED", + TypeSignedLong: "SLONG", + TypeSignedRational: "SRATIONAL", + TypeFloat: "FLOAT", + TypeDouble: "DOUBLE", + + TypeAsciiNoNul: "_ASCII_NO_NUL", + } + + typeNamesR = map[string]TagTypePrimitive{} +) + +// Rational describes an unsigned rational value. +type Rational struct { + // Numerator is the numerator of the rational value. + Numerator uint32 + + // Denominator is the numerator of the rational value. + Denominator uint32 +} + +// SignedRational describes a signed rational value. +type SignedRational struct { + // Numerator is the numerator of the rational value. + Numerator int32 + + // Denominator is the numerator of the rational value. + Denominator int32 +} + +func isPrintableText(s string) bool { + for _, c := range s { + // unicode.IsPrint() returns false for newline characters. + if c == 0x0d || c == 0x0a { + continue + } else if unicode.IsPrint(rune(c)) == false { + return false + } + } + + return true +} + +// Format returns a stringified value for the given encoding. Automatically +// parses. Automatically calculates count based on type size. This function +// also supports undefined-type values (the ones that we support, anyway) by +// way of the String() method that they all require. We can't be more specific +// because we're a base package and we can't refer to it. +func FormatFromType(value interface{}, justFirst bool) (phrase string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test + + switch t := value.(type) { + case []byte: + return DumpBytesToString(t), nil + case string: + for i, c := range t { + if c == 0 { + t = t[:i] + break + } + } + + if isPrintableText(t) == false { + phrase = fmt.Sprintf("string with binary data (%d bytes)", len(t)) + return phrase, nil + } + + return t, nil + case []uint16, []uint32, []int32, []float64, []float32: + val := reflect.ValueOf(t) + + if val.Len() == 0 { + return "", nil + } + + if justFirst == true { + var valueSuffix string + if val.Len() > 1 { + valueSuffix = "..." + } + + return fmt.Sprintf("%v%s", val.Index(0), valueSuffix), nil + } + + return fmt.Sprintf("%v", val), nil + case []Rational: + if len(t) == 0 { + return "", nil + } + + parts := make([]string, len(t)) + for i, r := range t { + parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) + + if justFirst == true { + break + } + } + + if justFirst == true { + var valueSuffix string + if len(t) > 1 { + valueSuffix = "..." + } + + return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil + } + + return fmt.Sprintf("%v", parts), nil + case []SignedRational: + if len(t) == 0 { + return "", nil + } + + parts := make([]string, len(t)) + for i, r := range t { + parts[i] = fmt.Sprintf("%d/%d", r.Numerator, r.Denominator) + + if justFirst == true { + break + } + } + + if justFirst == true { + var valueSuffix string + if len(t) > 1 { + valueSuffix = "..." + } + + return fmt.Sprintf("%v%s", parts[0], valueSuffix), nil + } + + return fmt.Sprintf("%v", parts), nil + case fmt.Stringer: + s := t.String() + if isPrintableText(s) == false { + phrase = fmt.Sprintf("stringable with binary data (%d bytes)", len(s)) + return phrase, nil + } + + // An undefined value that is documented (or that we otherwise support). + return s, nil + default: + // Affects only "unknown" values, in general. + log.Panicf("type can not be formatted into string: %v", reflect.TypeOf(value).Name()) + + // Never called. + return "", nil + } +} + +// Format returns a stringified value for the given encoding. Automatically +// parses. Automatically calculates count based on type size. +func FormatFromBytes(rawBytes []byte, tagType TagTypePrimitive, justFirst bool, byteOrder binary.ByteOrder) (phrase string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test + + typeSize := tagType.Size() + + if len(rawBytes)%typeSize != 0 { + log.Panicf("byte-count (%d) does not align for [%s] type with a size of (%d) bytes", len(rawBytes), TypeNames[tagType], typeSize) + } + + // unitCount is the calculated unit-count. This should equal the original + // value from the tag (pre-resolution). + unitCount := uint32(len(rawBytes) / typeSize) + + // Truncate the items if it's not bytes or a string and we just want the first. + + var value interface{} + + switch tagType { + case TypeByte: + var err error + + value, err = parser.ParseBytes(rawBytes, unitCount) + log.PanicIf(err) + case TypeAscii: + var err error + + value, err = parser.ParseAscii(rawBytes, unitCount) + log.PanicIf(err) + case TypeAsciiNoNul: + var err error + + value, err = parser.ParseAsciiNoNul(rawBytes, unitCount) + log.PanicIf(err) + case TypeShort: + var err error + + value, err = parser.ParseShorts(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeLong: + var err error + + value, err = parser.ParseLongs(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeFloat: + var err error + + value, err = parser.ParseFloats(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeDouble: + var err error + + value, err = parser.ParseDoubles(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeRational: + var err error + + value, err = parser.ParseRationals(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeSignedLong: + var err error + + value, err = parser.ParseSignedLongs(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + case TypeSignedRational: + var err error + + value, err = parser.ParseSignedRationals(rawBytes, unitCount, byteOrder) + log.PanicIf(err) + default: + // Affects only "unknown" values, in general. + log.Panicf("value of type [%s] can not be formatted into string", tagType.String()) + + // Never called. + return "", nil + } + + phrase, err = FormatFromType(value, justFirst) + log.PanicIf(err) + + return phrase, nil +} + +// TranslateStringToType converts user-provided strings to properly-typed +// values. If a string, returns a string. Else, assumes that it's a single +// number. If a list needs to be processed, it is the caller's responsibility to +// split it (according to whichever convention has been established). +func TranslateStringToType(tagType TagTypePrimitive, valueString string) (value interface{}, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if tagType == TypeUndefined { + // The caller should just call String() on the decoded type. + log.Panicf("undefined-type values are not supported") + } + + if tagType == TypeByte { + wide, err := strconv.ParseInt(valueString, 16, 8) + log.PanicIf(err) + + return byte(wide), nil + } else if tagType == TypeAscii || tagType == TypeAsciiNoNul { + // Whether or not we're putting an NUL on the end is only relevant for + // byte-level encoding. This function really just supports a user + // interface. + + return valueString, nil + } else if tagType == TypeShort { + n, err := strconv.ParseUint(valueString, 10, 16) + log.PanicIf(err) + + return uint16(n), nil + } else if tagType == TypeLong { + n, err := strconv.ParseUint(valueString, 10, 32) + log.PanicIf(err) + + return uint32(n), nil + } else if tagType == TypeRational { + parts := strings.SplitN(valueString, "/", 2) + + numerator, err := strconv.ParseUint(parts[0], 10, 32) + log.PanicIf(err) + + denominator, err := strconv.ParseUint(parts[1], 10, 32) + log.PanicIf(err) + + return Rational{ + Numerator: uint32(numerator), + Denominator: uint32(denominator), + }, nil + } else if tagType == TypeSignedLong { + n, err := strconv.ParseInt(valueString, 10, 32) + log.PanicIf(err) + + return int32(n), nil + } else if tagType == TypeFloat { + n, err := strconv.ParseFloat(valueString, 32) + log.PanicIf(err) + + return float32(n), nil + } else if tagType == TypeDouble { + n, err := strconv.ParseFloat(valueString, 64) + log.PanicIf(err) + + return float64(n), nil + } else if tagType == TypeSignedRational { + parts := strings.SplitN(valueString, "/", 2) + + numerator, err := strconv.ParseInt(parts[0], 10, 32) + log.PanicIf(err) + + denominator, err := strconv.ParseInt(parts[1], 10, 32) + log.PanicIf(err) + + return SignedRational{ + Numerator: int32(numerator), + Denominator: int32(denominator), + }, nil + } + + log.Panicf("from-string encoding for type not supported; this shouldn't happen: [%s]", tagType.String()) + return nil, nil +} + +// GetTypeByName returns the `TagTypePrimitive` for the given type name. +// Returns (0) if not valid. +func GetTypeByName(typeName string) (tagType TagTypePrimitive, found bool) { + tagType, found = typeNamesR[typeName] + return tagType, found +} + +// BasicTag describes a single tag for any purpose. +type BasicTag struct { + // FqIfdPath is the fully-qualified IFD-path. + FqIfdPath string + + // IfdPath is the unindexed IFD-path. + IfdPath string + + // TagId is the tag-ID. + TagId uint16 +} + +func init() { + for typeId, typeName := range TypeNames { + typeNamesR[typeName] = typeId + } +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/utility.go b/vendor/github.com/dsoprea/go-exif/v3/common/utility.go new file mode 100644 index 000000000..575049706 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/utility.go @@ -0,0 +1,148 @@ +package exifcommon + +import ( + "bytes" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "github.com/dsoprea/go-logging" +) + +var ( + timeType = reflect.TypeOf(time.Time{}) +) + +// DumpBytes prints a list of hex-encoded bytes. +func DumpBytes(data []byte) { + fmt.Printf("DUMP: ") + for _, x := range data { + fmt.Printf("%02x ", x) + } + + fmt.Printf("\n") +} + +// DumpBytesClause prints a list like DumpBytes(), but encapsulated in +// "[]byte { ... }". +func DumpBytesClause(data []byte) { + fmt.Printf("DUMP: ") + + fmt.Printf("[]byte { ") + + for i, x := range data { + fmt.Printf("0x%02x", x) + + if i < len(data)-1 { + fmt.Printf(", ") + } + } + + fmt.Printf(" }\n") +} + +// DumpBytesToString returns a stringified list of hex-encoded bytes. +func DumpBytesToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteRune(' ') + log.PanicIf(err) + } + } + + return b.String() +} + +// DumpBytesClauseToString returns a comma-separated list of hex-encoded bytes. +func DumpBytesClauseToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteString(", ") + log.PanicIf(err) + } + } + + return b.String() +} + +// ExifFullTimestampString produces a string like "2018:11:30 13:01:49" from a +// `time.Time` struct. It will attempt to convert to UTC first. +func ExifFullTimestampString(t time.Time) (fullTimestampPhrase string) { + t = t.UTC() + + return fmt.Sprintf("%04d:%02d:%02d %02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) +} + +// ParseExifFullTimestamp parses dates like "2018:11:30 13:01:49" into a UTC +// `time.Time` struct. +func ParseExifFullTimestamp(fullTimestampPhrase string) (timestamp time.Time, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + parts := strings.Split(fullTimestampPhrase, " ") + datestampValue, timestampValue := parts[0], parts[1] + + // Normalize the separators. + datestampValue = strings.ReplaceAll(datestampValue, "-", ":") + timestampValue = strings.ReplaceAll(timestampValue, "-", ":") + + dateParts := strings.Split(datestampValue, ":") + + year, err := strconv.ParseUint(dateParts[0], 10, 16) + if err != nil { + log.Panicf("could not parse year") + } + + month, err := strconv.ParseUint(dateParts[1], 10, 8) + if err != nil { + log.Panicf("could not parse month") + } + + day, err := strconv.ParseUint(dateParts[2], 10, 8) + if err != nil { + log.Panicf("could not parse day") + } + + timeParts := strings.Split(timestampValue, ":") + + hour, err := strconv.ParseUint(timeParts[0], 10, 8) + if err != nil { + log.Panicf("could not parse hour") + } + + minute, err := strconv.ParseUint(timeParts[1], 10, 8) + if err != nil { + log.Panicf("could not parse minute") + } + + second, err := strconv.ParseUint(timeParts[2], 10, 8) + if err != nil { + log.Panicf("could not parse second") + } + + timestamp = time.Date(int(year), time.Month(month), int(day), int(hour), int(minute), int(second), 0, time.UTC) + return timestamp, nil +} + +// IsTime returns true if the value is a `time.Time`. +func IsTime(v interface{}) bool { + + // TODO(dustin): Add test + + return reflect.TypeOf(v) == timeType +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/value_context.go b/vendor/github.com/dsoprea/go-exif/v3/common/value_context.go new file mode 100644 index 000000000..b9e634106 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/value_context.go @@ -0,0 +1,464 @@ +package exifcommon + +import ( + "errors" + "io" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +var ( + parser *Parser +) + +var ( + // ErrNotFarValue indicates that an offset-based lookup was attempted for a + // non-offset-based (embedded) value. + ErrNotFarValue = errors.New("not a far value") +) + +// ValueContext embeds all of the parameters required to find and extract the +// actual tag value. +type ValueContext struct { + unitCount uint32 + valueOffset uint32 + rawValueOffset []byte + rs io.ReadSeeker + + tagType TagTypePrimitive + byteOrder binary.ByteOrder + + // undefinedValueTagType is the effective type to use if this is an + // "undefined" value. + undefinedValueTagType TagTypePrimitive + + ifdPath string + tagId uint16 +} + +// TODO(dustin): We can update newValueContext() to derive `valueOffset` itself (from `rawValueOffset`). + +// NewValueContext returns a new ValueContext struct. +func NewValueContext(ifdPath string, tagId uint16, unitCount, valueOffset uint32, rawValueOffset []byte, rs io.ReadSeeker, tagType TagTypePrimitive, byteOrder binary.ByteOrder) *ValueContext { + return &ValueContext{ + unitCount: unitCount, + valueOffset: valueOffset, + rawValueOffset: rawValueOffset, + rs: rs, + + tagType: tagType, + byteOrder: byteOrder, + + ifdPath: ifdPath, + tagId: tagId, + } +} + +// SetUndefinedValueType sets the effective type if this is an unknown-type tag. +func (vc *ValueContext) SetUndefinedValueType(tagType TagTypePrimitive) { + if vc.tagType != TypeUndefined { + log.Panicf("can not set effective type for unknown-type tag because this is *not* an unknown-type tag") + } + + vc.undefinedValueTagType = tagType +} + +// UnitCount returns the embedded unit-count. +func (vc *ValueContext) UnitCount() uint32 { + return vc.unitCount +} + +// ValueOffset returns the value-offset decoded as a `uint32`. +func (vc *ValueContext) ValueOffset() uint32 { + return vc.valueOffset +} + +// RawValueOffset returns the uninterpreted value-offset. This is used for +// embedded values (values small enough to fit within the offset bytes rather +// than needing to be stored elsewhere and referred to by an actual offset). +func (vc *ValueContext) RawValueOffset() []byte { + return vc.rawValueOffset +} + +// AddressableData returns the block of data that we can dereference into. +func (vc *ValueContext) AddressableData() io.ReadSeeker { + + // RELEASE)dustin): Rename from AddressableData() to ReadSeeker() + + return vc.rs +} + +// ByteOrder returns the byte-order of numbers. +func (vc *ValueContext) ByteOrder() binary.ByteOrder { + return vc.byteOrder +} + +// IfdPath returns the path of the IFD containing this tag. +func (vc *ValueContext) IfdPath() string { + return vc.ifdPath +} + +// TagId returns the ID of the tag that we represent. +func (vc *ValueContext) TagId() uint16 { + return vc.tagId +} + +// isEmbedded returns whether the value is embedded or a reference. This can't +// be precalculated since the size is not defined for all types (namely the +// "undefined" types). +func (vc *ValueContext) isEmbedded() bool { + tagType := vc.effectiveValueType() + + return (tagType.Size() * int(vc.unitCount)) <= 4 +} + +// SizeInBytes returns the number of bytes that this value requires. The +// underlying call will panic if the type is UNDEFINED. It is the +// responsibility of the caller to preemptively check that. +func (vc *ValueContext) SizeInBytes() int { + tagType := vc.effectiveValueType() + + return tagType.Size() * int(vc.unitCount) +} + +// effectiveValueType returns the effective type of the unknown-type tag or, if +// not unknown, the actual type. +func (vc *ValueContext) effectiveValueType() (tagType TagTypePrimitive) { + if vc.tagType == TypeUndefined { + tagType = vc.undefinedValueTagType + + if tagType == 0 { + log.Panicf("undefined-value type not set") + } + } else { + tagType = vc.tagType + } + + return tagType +} + +// readRawEncoded returns the encoded bytes for the value that we represent. +func (vc *ValueContext) readRawEncoded() (rawBytes []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + tagType := vc.effectiveValueType() + + unitSizeRaw := uint32(tagType.Size()) + + if vc.isEmbedded() == true { + byteLength := unitSizeRaw * vc.unitCount + return vc.rawValueOffset[:byteLength], nil + } + + _, err = vc.rs.Seek(int64(vc.valueOffset), io.SeekStart) + log.PanicIf(err) + + rawBytes = make([]byte, vc.unitCount*unitSizeRaw) + + _, err = io.ReadFull(vc.rs, rawBytes) + log.PanicIf(err) + + return rawBytes, nil +} + +// GetFarOffset returns the offset if the value is not embedded [within the +// pointer itself] or an error if an embedded value. +func (vc *ValueContext) GetFarOffset() (offset uint32, err error) { + if vc.isEmbedded() == true { + return 0, ErrNotFarValue + } + + return vc.valueOffset, nil +} + +// ReadRawEncoded returns the encoded bytes for the value that we represent. +func (vc *ValueContext) ReadRawEncoded() (rawBytes []byte, err error) { + + // TODO(dustin): Remove this method and rename readRawEncoded in its place. + + return vc.readRawEncoded() +} + +// Format returns a string representation for the value. +// +// Where the type is not ASCII, `justFirst` indicates whether to just stringify +// the first item in the slice (or return an empty string if the slice is +// empty). +// +// Since this method lacks the information to process undefined-type tags (e.g. +// byte-order, tag-ID, IFD type), it will return an error if attempted. See +// `Undefined()`. +func (vc *ValueContext) Format() (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawBytes, err := vc.readRawEncoded() + log.PanicIf(err) + + phrase, err := FormatFromBytes(rawBytes, vc.effectiveValueType(), false, vc.byteOrder) + log.PanicIf(err) + + return phrase, nil +} + +// FormatFirst is similar to `Format` but only gets and stringifies the first +// item. +func (vc *ValueContext) FormatFirst() (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawBytes, err := vc.readRawEncoded() + log.PanicIf(err) + + phrase, err := FormatFromBytes(rawBytes, vc.tagType, true, vc.byteOrder) + log.PanicIf(err) + + return phrase, nil +} + +// ReadBytes parses the encoded byte-array from the value-context. +func (vc *ValueContext) ReadBytes() (value []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseBytes(rawValue, vc.unitCount) + log.PanicIf(err) + + return value, nil +} + +// ReadAscii parses the encoded NUL-terminated ASCII string from the value- +// context. +func (vc *ValueContext) ReadAscii() (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseAscii(rawValue, vc.unitCount) + log.PanicIf(err) + + return value, nil +} + +// ReadAsciiNoNul parses the non-NUL-terminated encoded ASCII string from the +// value-context. +func (vc *ValueContext) ReadAsciiNoNul() (value string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseAsciiNoNul(rawValue, vc.unitCount) + log.PanicIf(err) + + return value, nil +} + +// ReadShorts parses the list of encoded shorts from the value-context. +func (vc *ValueContext) ReadShorts() (value []uint16, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseShorts(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadLongs parses the list of encoded, unsigned longs from the value-context. +func (vc *ValueContext) ReadLongs() (value []uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseLongs(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadFloats parses the list of encoded, floats from the value-context. +func (vc *ValueContext) ReadFloats() (value []float32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseFloats(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadDoubles parses the list of encoded, doubles from the value-context. +func (vc *ValueContext) ReadDoubles() (value []float64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseDoubles(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadRationals parses the list of encoded, unsigned rationals from the value- +// context. +func (vc *ValueContext) ReadRationals() (value []Rational, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseRationals(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadSignedLongs parses the list of encoded, signed longs from the value-context. +func (vc *ValueContext) ReadSignedLongs() (value []int32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseSignedLongs(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// ReadSignedRationals parses the list of encoded, signed rationals from the +// value-context. +func (vc *ValueContext) ReadSignedRationals() (value []SignedRational, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawValue, err := vc.readRawEncoded() + log.PanicIf(err) + + value, err = parser.ParseSignedRationals(rawValue, vc.unitCount, vc.byteOrder) + log.PanicIf(err) + + return value, nil +} + +// Values knows how to resolve the given value. This value is always a list +// (undefined-values aside), so we're named accordingly. +// +// Since this method lacks the information to process unknown-type tags (e.g. +// byte-order, tag-ID, IFD type), it will return an error if attempted. See +// `Undefined()`. +func (vc *ValueContext) Values() (values interface{}, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if vc.tagType == TypeByte { + values, err = vc.ReadBytes() + log.PanicIf(err) + } else if vc.tagType == TypeAscii { + values, err = vc.ReadAscii() + log.PanicIf(err) + } else if vc.tagType == TypeAsciiNoNul { + values, err = vc.ReadAsciiNoNul() + log.PanicIf(err) + } else if vc.tagType == TypeShort { + values, err = vc.ReadShorts() + log.PanicIf(err) + } else if vc.tagType == TypeLong { + values, err = vc.ReadLongs() + log.PanicIf(err) + } else if vc.tagType == TypeRational { + values, err = vc.ReadRationals() + log.PanicIf(err) + } else if vc.tagType == TypeSignedLong { + values, err = vc.ReadSignedLongs() + log.PanicIf(err) + } else if vc.tagType == TypeSignedRational { + values, err = vc.ReadSignedRationals() + log.PanicIf(err) + } else if vc.tagType == TypeFloat { + values, err = vc.ReadFloats() + log.PanicIf(err) + } else if vc.tagType == TypeDouble { + values, err = vc.ReadDoubles() + log.PanicIf(err) + } else if vc.tagType == TypeUndefined { + log.Panicf("will not parse undefined-type value") + + // Never called. + return nil, nil + } else { + log.Panicf("value of type [%s] is unparseable", vc.tagType) + // Never called. + return nil, nil + } + + return values, nil +} + +func init() { + parser = new(Parser) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/common/value_encoder.go b/vendor/github.com/dsoprea/go-exif/v3/common/value_encoder.go new file mode 100644 index 000000000..2cd26cc7b --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/common/value_encoder.go @@ -0,0 +1,273 @@ +package exifcommon + +import ( + "bytes" + "math" + "reflect" + "time" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +var ( + typeEncodeLogger = log.NewLogger("exif.type_encode") +) + +// EncodedData encapsulates the compound output of an encoding operation. +type EncodedData struct { + Type TagTypePrimitive + Encoded []byte + + // TODO(dustin): Is this really necessary? We might have this just to correlate to the incoming stream format (raw bytes and a unit-count both for incoming and outgoing). + UnitCount uint32 +} + +// ValueEncoder knows how to encode values of every type to bytes. +type ValueEncoder struct { + byteOrder binary.ByteOrder +} + +// NewValueEncoder returns a new ValueEncoder. +func NewValueEncoder(byteOrder binary.ByteOrder) *ValueEncoder { + return &ValueEncoder{ + byteOrder: byteOrder, + } +} + +func (ve *ValueEncoder) encodeBytes(value []uint8) (ed EncodedData, err error) { + ed.Type = TypeByte + ed.Encoded = []byte(value) + ed.UnitCount = uint32(len(value)) + + return ed, nil +} + +func (ve *ValueEncoder) encodeAscii(value string) (ed EncodedData, err error) { + ed.Type = TypeAscii + + ed.Encoded = []byte(value) + ed.Encoded = append(ed.Encoded, 0) + + ed.UnitCount = uint32(len(ed.Encoded)) + + return ed, nil +} + +// encodeAsciiNoNul returns a string encoded as a byte-string without a trailing +// NUL byte. +// +// Note that: +// +// 1. This type can not be automatically encoded using `Encode()`. The default +// mode is to encode *with* a trailing NUL byte using `encodeAscii`. Only +// certain undefined-type tags using an unterminated ASCII string and these +// are exceptional in nature. +// +// 2. The presence of this method allows us to completely test the complimentary +// no-nul parser. +// +func (ve *ValueEncoder) encodeAsciiNoNul(value string) (ed EncodedData, err error) { + ed.Type = TypeAsciiNoNul + ed.Encoded = []byte(value) + ed.UnitCount = uint32(len(ed.Encoded)) + + return ed, nil +} + +func (ve *ValueEncoder) encodeShorts(value []uint16) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + ed.Encoded = make([]byte, ed.UnitCount*2) + + for i := uint32(0); i < ed.UnitCount; i++ { + ve.byteOrder.PutUint16(ed.Encoded[i*2:(i+1)*2], value[i]) + } + + ed.Type = TypeShort + + return ed, nil +} + +func (ve *ValueEncoder) encodeLongs(value []uint32) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + ed.Encoded = make([]byte, ed.UnitCount*4) + + for i := uint32(0); i < ed.UnitCount; i++ { + ve.byteOrder.PutUint32(ed.Encoded[i*4:(i+1)*4], value[i]) + } + + ed.Type = TypeLong + + return ed, nil +} + +func (ve *ValueEncoder) encodeFloats(value []float32) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + ed.Encoded = make([]byte, ed.UnitCount*4) + + for i := uint32(0); i < ed.UnitCount; i++ { + ve.byteOrder.PutUint32(ed.Encoded[i*4:(i+1)*4], math.Float32bits(value[i])) + } + + ed.Type = TypeFloat + + return ed, nil +} + +func (ve *ValueEncoder) encodeDoubles(value []float64) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + ed.Encoded = make([]byte, ed.UnitCount*8) + + for i := uint32(0); i < ed.UnitCount; i++ { + ve.byteOrder.PutUint64(ed.Encoded[i*8:(i+1)*8], math.Float64bits(value[i])) + } + + ed.Type = TypeDouble + + return ed, nil +} + +func (ve *ValueEncoder) encodeRationals(value []Rational) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + ed.Encoded = make([]byte, ed.UnitCount*8) + + for i := uint32(0); i < ed.UnitCount; i++ { + ve.byteOrder.PutUint32(ed.Encoded[i*8+0:i*8+4], value[i].Numerator) + ve.byteOrder.PutUint32(ed.Encoded[i*8+4:i*8+8], value[i].Denominator) + } + + ed.Type = TypeRational + + return ed, nil +} + +func (ve *ValueEncoder) encodeSignedLongs(value []int32) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + + b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) + + for i := uint32(0); i < ed.UnitCount; i++ { + err := binary.Write(b, ve.byteOrder, value[i]) + log.PanicIf(err) + } + + ed.Type = TypeSignedLong + ed.Encoded = b.Bytes() + + return ed, nil +} + +func (ve *ValueEncoder) encodeSignedRationals(value []SignedRational) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ed.UnitCount = uint32(len(value)) + + b := bytes.NewBuffer(make([]byte, 0, 8*ed.UnitCount)) + + for i := uint32(0); i < ed.UnitCount; i++ { + err := binary.Write(b, ve.byteOrder, value[i].Numerator) + log.PanicIf(err) + + err = binary.Write(b, ve.byteOrder, value[i].Denominator) + log.PanicIf(err) + } + + ed.Type = TypeSignedRational + ed.Encoded = b.Bytes() + + return ed, nil +} + +// Encode returns bytes for the given value, infering type from the actual +// value. This does not support `TypeAsciiNoNull` (all strings are encoded as +// `TypeAscii`). +func (ve *ValueEncoder) Encode(value interface{}) (ed EncodedData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + switch t := value.(type) { + case []byte: + ed, err = ve.encodeBytes(t) + log.PanicIf(err) + case string: + ed, err = ve.encodeAscii(t) + log.PanicIf(err) + case []uint16: + ed, err = ve.encodeShorts(t) + log.PanicIf(err) + case []uint32: + ed, err = ve.encodeLongs(t) + log.PanicIf(err) + case []float32: + ed, err = ve.encodeFloats(t) + log.PanicIf(err) + case []float64: + ed, err = ve.encodeDoubles(t) + log.PanicIf(err) + case []Rational: + ed, err = ve.encodeRationals(t) + log.PanicIf(err) + case []int32: + ed, err = ve.encodeSignedLongs(t) + log.PanicIf(err) + case []SignedRational: + ed, err = ve.encodeSignedRationals(t) + log.PanicIf(err) + case time.Time: + // For convenience, if the user doesn't want to deal with translation + // semantics with timestamps. + + s := ExifFullTimestampString(t) + + ed, err = ve.encodeAscii(s) + log.PanicIf(err) + default: + log.Panicf("value not encodable: [%s] [%v]", reflect.TypeOf(value), value) + } + + return ed, nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/data_layer.go b/vendor/github.com/dsoprea/go-exif/v3/data_layer.go new file mode 100644 index 000000000..7883752cc --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/data_layer.go @@ -0,0 +1,50 @@ +package exif + +import ( + "io" + + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-utility/v2/filesystem" +) + +type ExifBlobSeeker interface { + GetReadSeeker(initialOffset int64) (rs io.ReadSeeker, err error) +} + +// ExifReadSeeker knows how to retrieve data from the EXIF blob relative to the +// beginning of the blob (so, absolute position (0) is the first byte of the +// EXIF data). +type ExifReadSeeker struct { + rs io.ReadSeeker +} + +func NewExifReadSeeker(rs io.ReadSeeker) *ExifReadSeeker { + return &ExifReadSeeker{ + rs: rs, + } +} + +func NewExifReadSeekerWithBytes(exifData []byte) *ExifReadSeeker { + sb := rifs.NewSeekableBufferWithBytes(exifData) + edbs := NewExifReadSeeker(sb) + + return edbs +} + +// Fork creates a new ReadSeeker instead that wraps a BouncebackReader to +// maintain its own position in the stream. +func (edbs *ExifReadSeeker) GetReadSeeker(initialOffset int64) (rs io.ReadSeeker, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + br, err := rifs.NewBouncebackReader(edbs.rs) + log.PanicIf(err) + + _, err = br.Seek(initialOffset, io.SeekStart) + log.PanicIf(err) + + return br, nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/error.go b/vendor/github.com/dsoprea/go-exif/v3/error.go new file mode 100644 index 000000000..2f00b08a4 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/error.go @@ -0,0 +1,14 @@ +package exif + +import ( + "errors" +) + +var ( + // ErrTagNotFound indicates that the tag was not found. + ErrTagNotFound = errors.New("tag not found") + + // ErrTagNotKnown indicates that the tag is not registered with us as a + // known tag. + ErrTagNotKnown = errors.New("tag is not known") +) diff --git a/vendor/github.com/dsoprea/go-exif/v3/exif.go b/vendor/github.com/dsoprea/go-exif/v3/exif.go new file mode 100644 index 000000000..f66e839d9 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/exif.go @@ -0,0 +1,333 @@ +package exif + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + + "encoding/binary" + "io/ioutil" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +const ( + // ExifAddressableAreaStart is the absolute offset in the file that all + // offsets are relative to. + ExifAddressableAreaStart = uint32(0x0) + + // ExifDefaultFirstIfdOffset is essentially the number of bytes in addition + // to `ExifAddressableAreaStart` that you have to move in order to escape + // the rest of the header and get to the earliest point where we can put + // stuff (which has to be the first IFD). This is the size of the header + // sequence containing the two-character byte-order, two-character fixed- + // bytes, and the four bytes describing the first-IFD offset. + ExifDefaultFirstIfdOffset = uint32(2 + 2 + 4) +) + +const ( + // ExifSignatureLength is the number of bytes in the EXIF signature (which + // customarily includes the first IFD offset). + ExifSignatureLength = 8 +) + +var ( + exifLogger = log.NewLogger("exif.exif") + + ExifBigEndianSignature = [4]byte{'M', 'M', 0x00, 0x2a} + ExifLittleEndianSignature = [4]byte{'I', 'I', 0x2a, 0x00} +) + +var ( + ErrNoExif = errors.New("no exif data") + ErrExifHeaderError = errors.New("exif header error") +) + +// SearchAndExtractExif searches for an EXIF blob in the byte-slice. +func SearchAndExtractExif(data []byte) (rawExif []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + b := bytes.NewBuffer(data) + + rawExif, err = SearchAndExtractExifWithReader(b) + if err != nil { + if err == ErrNoExif { + return nil, err + } + + log.Panic(err) + } + + return rawExif, nil +} + +// SearchAndExtractExifN searches for an EXIF blob in the byte-slice, but skips +// the given number of EXIF blocks first. This is a forensics tool that helps +// identify multiple EXIF blocks in a file. +func SearchAndExtractExifN(data []byte, n int) (rawExif []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + skips := 0 + totalDiscarded := 0 + for { + b := bytes.NewBuffer(data) + + var discarded int + + rawExif, discarded, err = searchAndExtractExifWithReaderWithDiscarded(b) + if err != nil { + if err == ErrNoExif { + return nil, err + } + + log.Panic(err) + } + + exifLogger.Debugf(nil, "Read EXIF block (%d).", skips) + + totalDiscarded += discarded + + if skips >= n { + exifLogger.Debugf(nil, "Reached requested EXIF block (%d).", n) + break + } + + nextOffset := discarded + 1 + exifLogger.Debugf(nil, "Skipping EXIF block (%d) by seeking to position (%d).", skips, nextOffset) + + data = data[nextOffset:] + skips++ + } + + exifLogger.Debugf(nil, "Found EXIF blob (%d) bytes from initial position.", totalDiscarded) + return rawExif, nil +} + +// searchAndExtractExifWithReaderWithDiscarded searches for an EXIF blob using +// an `io.Reader`. We can't know how much long the EXIF data is without parsing +// it, so this will likely grab up a lot of the image-data, too. +// +// This function returned the count of preceding bytes. +func searchAndExtractExifWithReaderWithDiscarded(r io.Reader) (rawExif []byte, discarded int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Search for the beginning of the EXIF information. The EXIF is near the + // beginning of most JPEGs, so this likely doesn't have a high cost (at + // least, again, with JPEGs). + + br := bufio.NewReader(r) + + for { + window, err := br.Peek(ExifSignatureLength) + if err != nil { + if err == io.EOF { + return nil, 0, ErrNoExif + } + + log.Panic(err) + } + + _, err = ParseExifHeader(window) + if err != nil { + if log.Is(err, ErrNoExif) == true { + // No EXIF. Move forward by one byte. + + _, err := br.Discard(1) + log.PanicIf(err) + + discarded++ + + continue + } + + // Some other error. + log.Panic(err) + } + + break + } + + exifLogger.Debugf(nil, "Found EXIF blob (%d) bytes from initial position.", discarded) + + rawExif, err = ioutil.ReadAll(br) + log.PanicIf(err) + + return rawExif, discarded, nil +} + +// RELEASE(dustin): We should replace the implementation of SearchAndExtractExifWithReader with searchAndExtractExifWithReaderWithDiscarded and drop the latter. + +// SearchAndExtractExifWithReader searches for an EXIF blob using an +// `io.Reader`. We can't know how much long the EXIF data is without parsing it, +// so this will likely grab up a lot of the image-data, too. +func SearchAndExtractExifWithReader(r io.Reader) (rawExif []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rawExif, _, err = searchAndExtractExifWithReaderWithDiscarded(r) + if err != nil { + if err == ErrNoExif { + return nil, err + } + + log.Panic(err) + } + + return rawExif, nil +} + +// SearchFileAndExtractExif returns a slice from the beginning of the EXIF data +// to the end of the file (it's not practical to try and calculate where the +// data actually ends). +func SearchFileAndExtractExif(filepath string) (rawExif []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Open the file. + + f, err := os.Open(filepath) + log.PanicIf(err) + + defer f.Close() + + rawExif, err = SearchAndExtractExifWithReader(f) + log.PanicIf(err) + + return rawExif, nil +} + +type ExifHeader struct { + ByteOrder binary.ByteOrder + FirstIfdOffset uint32 +} + +func (eh ExifHeader) String() string { + return fmt.Sprintf("ExifHeader", eh.ByteOrder, eh.FirstIfdOffset) +} + +// ParseExifHeader parses the bytes at the very top of the header. +// +// This will panic with ErrNoExif on any data errors so that we can double as +// an EXIF-detection routine. +func ParseExifHeader(data []byte) (eh ExifHeader, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Good reference: + // + // CIPA DC-008-2016; JEITA CP-3451D + // -> http://www.cipa.jp/std/documents/e/DC-008-Translation-2016-E.pdf + + if len(data) < ExifSignatureLength { + exifLogger.Warningf(nil, "Not enough data for EXIF header: (%d)", len(data)) + return eh, ErrNoExif + } + + if bytes.Equal(data[:4], ExifBigEndianSignature[:]) == true { + exifLogger.Debugf(nil, "Byte-order is big-endian.") + eh.ByteOrder = binary.BigEndian + } else if bytes.Equal(data[:4], ExifLittleEndianSignature[:]) == true { + eh.ByteOrder = binary.LittleEndian + exifLogger.Debugf(nil, "Byte-order is little-endian.") + } else { + return eh, ErrNoExif + } + + eh.FirstIfdOffset = eh.ByteOrder.Uint32(data[4:8]) + + return eh, nil +} + +// Visit recursively invokes a callback for every tag. +func Visit(rootIfdIdentity *exifcommon.IfdIdentity, ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, exifData []byte, visitor TagVisitorFn, so *ScanOptions) (eh ExifHeader, furthestOffset uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + eh, err = ParseExifHeader(exifData) + log.PanicIf(err) + + ebs := NewExifReadSeekerWithBytes(exifData) + ie := NewIfdEnumerate(ifdMapping, tagIndex, ebs, eh.ByteOrder) + + _, err = ie.Scan(rootIfdIdentity, eh.FirstIfdOffset, visitor, so) + log.PanicIf(err) + + furthestOffset = ie.FurthestOffset() + + return eh, furthestOffset, nil +} + +// Collect recursively builds a static structure of all IFDs and tags. +func Collect(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, exifData []byte) (eh ExifHeader, index IfdIndex, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + eh, err = ParseExifHeader(exifData) + log.PanicIf(err) + + ebs := NewExifReadSeekerWithBytes(exifData) + ie := NewIfdEnumerate(ifdMapping, tagIndex, ebs, eh.ByteOrder) + + index, err = ie.Collect(eh.FirstIfdOffset) + log.PanicIf(err) + + return eh, index, nil +} + +// BuildExifHeader constructs the bytes that go at the front of the stream. +func BuildExifHeader(byteOrder binary.ByteOrder, firstIfdOffset uint32) (headerBytes []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + b := new(bytes.Buffer) + + var signatureBytes []byte + if byteOrder == binary.BigEndian { + signatureBytes = ExifBigEndianSignature[:] + } else { + signatureBytes = ExifLittleEndianSignature[:] + } + + _, err = b.Write(signatureBytes) + log.PanicIf(err) + + err = binary.Write(b, byteOrder, firstIfdOffset) + log.PanicIf(err) + + return b.Bytes(), nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/gps.go b/vendor/github.com/dsoprea/go-exif/v3/gps.go new file mode 100644 index 000000000..7a61cd94d --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/gps.go @@ -0,0 +1,117 @@ +package exif + +import ( + "errors" + "fmt" + "time" + + "github.com/dsoprea/go-logging" + "github.com/golang/geo/s2" + + "github.com/dsoprea/go-exif/v3/common" +) + +var ( + // ErrGpsCoordinatesNotValid means that some part of the geographic data was + // unparseable. + ErrGpsCoordinatesNotValid = errors.New("GPS coordinates not valid") +) + +// GpsDegrees is a high-level struct representing geographic data. +type GpsDegrees struct { + // Orientation describes the N/E/S/W direction that this position is + // relative to. + Orientation byte + + // Degrees is a simple float representing the underlying rational degrees + // amount. + Degrees float64 + + // Minutes is a simple float representing the underlying rational minutes + // amount. + Minutes float64 + + // Seconds is a simple float representing the underlying ration seconds + // amount. + Seconds float64 +} + +// NewGpsDegreesFromRationals returns a GpsDegrees struct given the EXIF-encoded +// information. The refValue is the N/E/S/W direction that this position is +// relative to. +func NewGpsDegreesFromRationals(refValue string, rawCoordinate []exifcommon.Rational) (gd GpsDegrees, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if len(rawCoordinate) != 3 { + log.Panicf("new GpsDegrees struct requires a raw-coordinate with exactly three rationals") + } + + gd = GpsDegrees{ + Orientation: refValue[0], + Degrees: float64(rawCoordinate[0].Numerator) / float64(rawCoordinate[0].Denominator), + Minutes: float64(rawCoordinate[1].Numerator) / float64(rawCoordinate[1].Denominator), + Seconds: float64(rawCoordinate[2].Numerator) / float64(rawCoordinate[2].Denominator), + } + + return gd, nil +} + +// String provides returns a descriptive string. +func (d GpsDegrees) String() string { + return fmt.Sprintf("Degrees", string([]byte{d.Orientation}), d.Degrees, d.Minutes, d.Seconds) +} + +// Decimal calculates and returns the simplified float representation of the +// component degrees. +func (d GpsDegrees) Decimal() float64 { + decimal := float64(d.Degrees) + float64(d.Minutes)/60.0 + float64(d.Seconds)/3600.0 + + if d.Orientation == 'S' || d.Orientation == 'W' { + return -decimal + } + + return decimal +} + +// Raw returns a Rational struct that can be used to *write* coordinates. In +// practice, the denominator are typically (1) in the original EXIF data, and, +// that being the case, this will best preserve precision. +func (d GpsDegrees) Raw() []exifcommon.Rational { + return []exifcommon.Rational{ + {Numerator: uint32(d.Degrees), Denominator: 1}, + {Numerator: uint32(d.Minutes), Denominator: 1}, + {Numerator: uint32(d.Seconds), Denominator: 1}, + } +} + +// GpsInfo encapsulates all of the geographic information in one place. +type GpsInfo struct { + Latitude, Longitude GpsDegrees + Altitude int + Timestamp time.Time +} + +// String returns a descriptive string. +func (gi *GpsInfo) String() string { + return fmt.Sprintf("GpsInfo", + gi.Latitude.Decimal(), gi.Longitude.Decimal(), gi.Altitude, gi.Timestamp) +} + +// S2CellId returns the cell-ID of the geographic location on the earth. +func (gi *GpsInfo) S2CellId() s2.CellID { + latitude := gi.Latitude.Decimal() + longitude := gi.Longitude.Decimal() + + ll := s2.LatLngFromDegrees(latitude, longitude) + cellId := s2.CellIDFromLatLng(ll) + + if cellId.IsValid() == false { + panic(ErrGpsCoordinatesNotValid) + } + + return cellId +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/ifd_builder.go b/vendor/github.com/dsoprea/go-exif/v3/ifd_builder.go new file mode 100644 index 000000000..a404b362a --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/ifd_builder.go @@ -0,0 +1,1199 @@ +package exif + +// NOTES: +// +// The thumbnail offset and length tags shouldn't be set directly. Use the +// (*IfdBuilder).SetThumbnail() method instead. + +import ( + "errors" + "fmt" + "strings" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-exif/v3/undefined" +) + +var ( + ifdBuilderLogger = log.NewLogger("exif.ifd_builder") +) + +var ( + ErrTagEntryNotFound = errors.New("tag entry not found") + ErrChildIbNotFound = errors.New("child IB not found") +) + +type IfdBuilderTagValue struct { + valueBytes []byte + ib *IfdBuilder +} + +func (ibtv IfdBuilderTagValue) String() string { + if ibtv.IsBytes() == true { + var valuePhrase string + if len(ibtv.valueBytes) <= 8 { + valuePhrase = fmt.Sprintf("%v", ibtv.valueBytes) + } else { + valuePhrase = fmt.Sprintf("%v...", ibtv.valueBytes[:8]) + } + + return fmt.Sprintf("IfdBuilderTagValue", valuePhrase, len(ibtv.valueBytes)) + } else if ibtv.IsIb() == true { + return fmt.Sprintf("IfdBuilderTagValue", ibtv.ib) + } else { + log.Panicf("IBTV state undefined") + return "" + } +} + +func NewIfdBuilderTagValueFromBytes(valueBytes []byte) *IfdBuilderTagValue { + return &IfdBuilderTagValue{ + valueBytes: valueBytes, + } +} + +func NewIfdBuilderTagValueFromIfdBuilder(ib *IfdBuilder) *IfdBuilderTagValue { + return &IfdBuilderTagValue{ + ib: ib, + } +} + +// IsBytes returns true if the bytes are populated. This is always the case +// when we're loaded from a tag in an existing IFD. +func (ibtv IfdBuilderTagValue) IsBytes() bool { + return ibtv.valueBytes != nil +} + +func (ibtv IfdBuilderTagValue) Bytes() []byte { + if ibtv.IsBytes() == false { + log.Panicf("this tag is not a byte-slice value") + } else if ibtv.IsIb() == true { + log.Panicf("this tag is an IFD-builder value not a byte-slice") + } + + return ibtv.valueBytes +} + +func (ibtv IfdBuilderTagValue) IsIb() bool { + return ibtv.ib != nil +} + +func (ibtv IfdBuilderTagValue) Ib() *IfdBuilder { + if ibtv.IsIb() == false { + log.Panicf("this tag is not an IFD-builder value") + } else if ibtv.IsBytes() == true { + log.Panicf("this tag is a byte-slice, not a IFD-builder") + } + + return ibtv.ib +} + +type BuilderTag struct { + // ifdPath is the path of the IFD that hosts this tag. + ifdPath string + + tagId uint16 + typeId exifcommon.TagTypePrimitive + + // value is either a value that can be encoded, an IfdBuilder instance (for + // child IFDs), or an IfdTagEntry instance representing an existing, + // previously-stored tag. + value *IfdBuilderTagValue + + // byteOrder is the byte order. It's chiefly/originally here to support + // printing the value. + byteOrder binary.ByteOrder +} + +func NewBuilderTag(ifdPath string, tagId uint16, typeId exifcommon.TagTypePrimitive, value *IfdBuilderTagValue, byteOrder binary.ByteOrder) *BuilderTag { + return &BuilderTag{ + ifdPath: ifdPath, + tagId: tagId, + typeId: typeId, + value: value, + byteOrder: byteOrder, + } +} + +func NewChildIfdBuilderTag(ifdPath string, tagId uint16, value *IfdBuilderTagValue) *BuilderTag { + return &BuilderTag{ + ifdPath: ifdPath, + tagId: tagId, + typeId: exifcommon.TypeLong, + value: value, + } +} + +func (bt *BuilderTag) Value() (value *IfdBuilderTagValue) { + return bt.value +} + +func (bt *BuilderTag) String() string { + var valueString string + + if bt.value.IsBytes() == true { + var err error + + valueString, err = exifcommon.FormatFromBytes(bt.value.Bytes(), bt.typeId, false, bt.byteOrder) + log.PanicIf(err) + } else { + valueString = fmt.Sprintf("%v", bt.value) + } + + return fmt.Sprintf("BuilderTag", bt.ifdPath, bt.tagId, bt.typeId.String(), valueString) +} + +func (bt *BuilderTag) SetValue(byteOrder binary.ByteOrder, value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test. + + var ed exifcommon.EncodedData + if bt.typeId == exifcommon.TypeUndefined { + encodeable := value.(exifundefined.EncodeableValue) + + encoded, unitCount, err := exifundefined.Encode(encodeable, byteOrder) + log.PanicIf(err) + + ed = exifcommon.EncodedData{ + Type: exifcommon.TypeUndefined, + Encoded: encoded, + UnitCount: unitCount, + } + } else { + ve := exifcommon.NewValueEncoder(byteOrder) + + var err error + + ed, err = ve.Encode(value) + log.PanicIf(err) + } + + bt.value = NewIfdBuilderTagValueFromBytes(ed.Encoded) + + return nil +} + +// NewStandardBuilderTag constructs a `BuilderTag` instance. The type is looked +// up. `ii` is the type of IFD that owns this tag. +func NewStandardBuilderTag(ifdPath string, it *IndexedTag, byteOrder binary.ByteOrder, value interface{}) *BuilderTag { + // If there is more than one supported type, we'll go with the larger to + // encode with. It'll use the same amount of fixed-space, and we'll + // eliminate unnecessary overflows/issues. + tagType := it.GetEncodingType(value) + + var rawBytes []byte + if it.DoesSupportType(exifcommon.TypeUndefined) == true { + encodeable := value.(exifundefined.EncodeableValue) + + var err error + + rawBytes, _, err = exifundefined.Encode(encodeable, byteOrder) + log.PanicIf(err) + } else { + ve := exifcommon.NewValueEncoder(byteOrder) + + ed, err := ve.Encode(value) + log.PanicIf(err) + + rawBytes = ed.Encoded + } + + tagValue := NewIfdBuilderTagValueFromBytes(rawBytes) + + return NewBuilderTag( + ifdPath, + it.Id, + tagType, + tagValue, + byteOrder) +} + +type IfdBuilder struct { + ifdIdentity *exifcommon.IfdIdentity + + byteOrder binary.ByteOrder + + // Includes both normal tags and IFD tags (which point to child IFDs). + // TODO(dustin): Keep a separate list of children like with `Ifd`. + // TODO(dustin): Either rename this or `Entries` in `Ifd` to be the same thing. + tags []*BuilderTag + + // existingOffset will be the offset that this IFD is currently found at if + // it represents an IFD that has previously been stored (or 0 if not). + existingOffset uint32 + + // nextIb represents the next link if we're chaining to another. + nextIb *IfdBuilder + + // thumbnailData is populated with thumbnail data if there was thumbnail + // data. Otherwise, it's nil. + thumbnailData []byte + + ifdMapping *exifcommon.IfdMapping + tagIndex *TagIndex +} + +func NewIfdBuilder(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder) (ib *IfdBuilder) { + ib = &IfdBuilder{ + ifdIdentity: ii, + + byteOrder: byteOrder, + tags: make([]*BuilderTag, 0), + + ifdMapping: ifdMapping, + tagIndex: tagIndex, + } + + return ib +} + +// NewIfdBuilderWithExistingIfd creates a new IB using the same header type +// information as the given IFD. +func NewIfdBuilderWithExistingIfd(ifd *Ifd) (ib *IfdBuilder) { + ib = &IfdBuilder{ + ifdIdentity: ifd.IfdIdentity(), + + byteOrder: ifd.ByteOrder(), + existingOffset: ifd.Offset(), + ifdMapping: ifd.ifdMapping, + tagIndex: ifd.tagIndex, + } + + return ib +} + +// NewIfdBuilderFromExistingChain creates a chain of IB instances from an +// IFD chain generated from real data. +func NewIfdBuilderFromExistingChain(rootIfd *Ifd) (firstIb *IfdBuilder) { + var lastIb *IfdBuilder + i := 0 + for thisExistingIfd := rootIfd; thisExistingIfd != nil; thisExistingIfd = thisExistingIfd.nextIfd { + newIb := NewIfdBuilder( + rootIfd.ifdMapping, + rootIfd.tagIndex, + rootIfd.ifdIdentity, + thisExistingIfd.ByteOrder()) + + if firstIb == nil { + firstIb = newIb + } else { + lastIb.SetNextIb(newIb) + } + + err := newIb.AddTagsFromExisting(thisExistingIfd, nil, nil) + log.PanicIf(err) + + lastIb = newIb + i++ + } + + return firstIb +} + +func (ib *IfdBuilder) IfdIdentity() *exifcommon.IfdIdentity { + return ib.ifdIdentity +} + +func (ib *IfdBuilder) NextIb() (nextIb *IfdBuilder, err error) { + return ib.nextIb, nil +} + +func (ib *IfdBuilder) ChildWithTagId(childIfdTagId uint16) (childIb *IfdBuilder, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for _, bt := range ib.tags { + if bt.value.IsIb() == false { + continue + } + + childIbThis := bt.value.Ib() + + if childIbThis.IfdIdentity().TagId() == childIfdTagId { + return childIbThis, nil + } + } + + log.Panic(ErrChildIbNotFound) + + // Never reached. + return nil, nil +} + +func getOrCreateIbFromRootIbInner(rootIb *IfdBuilder, parentIb *IfdBuilder, currentLineage []exifcommon.IfdTagIdAndIndex) (ib *IfdBuilder, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test. + + thisIb := rootIb + + // Since we're calling ourselves recursively with incrementally different + // paths, the FQ IFD-path of the parent that called us needs to be passed + // in, in order for us to know it. + var parentLineage []exifcommon.IfdTagIdAndIndex + if parentIb != nil { + var err error + + parentLineage, err = thisIb.ifdMapping.ResolvePath(parentIb.IfdIdentity().String()) + log.PanicIf(err) + } + + // Process the current path part. + currentItIi := currentLineage[0] + + // Make sure the leftmost part of the FQ IFD-path agrees with the IB we + // were given. + + expectedFqRootIfdPath := "" + if parentLineage != nil { + expectedLineage := append(parentLineage, currentItIi) + expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(expectedLineage) + } else { + expectedFqRootIfdPath = thisIb.ifdMapping.PathPhraseFromLineage(currentLineage[:1]) + } + + if expectedFqRootIfdPath != thisIb.IfdIdentity().String() { + log.Panicf("the FQ IFD-path [%s] we were given does not match the builder's FQ IFD-path [%s]", expectedFqRootIfdPath, thisIb.IfdIdentity().String()) + } + + // If we actually wanted a sibling (currentItIi.Index > 0) then seek to it, + // appending new siblings, as required, until we get there. + for i := 0; i < currentItIi.Index; i++ { + if thisIb.nextIb == nil { + // Generate an FQ IFD-path for the sibling. It'll use the same + // non-FQ IFD-path as the current IB. + + iiSibling := thisIb.IfdIdentity().NewSibling(i + 1) + thisIb.nextIb = NewIfdBuilder(thisIb.ifdMapping, thisIb.tagIndex, iiSibling, thisIb.byteOrder) + } + + thisIb = thisIb.nextIb + } + + // There is no child IFD to process. We're done. + if len(currentLineage) == 1 { + return thisIb, nil + } + + // Establish the next child to be processed. + + childItii := currentLineage[1] + + var foundChild *IfdBuilder + for _, bt := range thisIb.tags { + if bt.value.IsIb() == false { + continue + } + + childIb := bt.value.Ib() + + if childIb.IfdIdentity().TagId() == childItii.TagId { + foundChild = childIb + break + } + } + + // If we didn't find the child, add it. + + if foundChild == nil { + currentIfdTag := thisIb.IfdIdentity().IfdTag() + + childIfdTag := + exifcommon.NewIfdTag( + ¤tIfdTag, + childItii.TagId, + childItii.Name) + + iiChild := thisIb.IfdIdentity().NewChild(childIfdTag, 0) + + foundChild = + NewIfdBuilder( + thisIb.ifdMapping, + thisIb.tagIndex, + iiChild, + thisIb.byteOrder) + + err = thisIb.AddChildIb(foundChild) + log.PanicIf(err) + } + + finalIb, err := getOrCreateIbFromRootIbInner(foundChild, thisIb, currentLineage[1:]) + log.PanicIf(err) + + return finalIb, nil +} + +// GetOrCreateIbFromRootIb returns an IB representing the requested IFD, even if +// an IB doesn't already exist for it. This function may call itself +// recursively. +func GetOrCreateIbFromRootIb(rootIb *IfdBuilder, fqIfdPath string) (ib *IfdBuilder, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // lineage is a necessity of our recursion process. It doesn't include any + // parent IFDs on its left-side; it starts with the current IB only. + lineage, err := rootIb.ifdMapping.ResolvePath(fqIfdPath) + log.PanicIf(err) + + ib, err = getOrCreateIbFromRootIbInner(rootIb, nil, lineage) + log.PanicIf(err) + + return ib, nil +} + +func (ib *IfdBuilder) String() string { + nextIfdPhrase := "" + if ib.nextIb != nil { + // TODO(dustin): We were setting this to ii.String(), but we were getting hex-data when printing this after building from an existing chain. + nextIfdPhrase = ib.nextIb.IfdIdentity().UnindexedString() + } + + return fmt.Sprintf("IfdBuilder", ib.IfdIdentity().UnindexedString(), ib.IfdIdentity().TagId(), len(ib.tags), ib.existingOffset, nextIfdPhrase) +} + +func (ib *IfdBuilder) Tags() (tags []*BuilderTag) { + return ib.tags +} + +// SetThumbnail sets thumbnail data. +// +// NOTES: +// +// - We don't manage any facet of the thumbnail data. This is the +// responsibility of the user/developer. +// - This method will fail unless the thumbnail is set on a the root IFD. +// However, in order to be valid, it must be set on the second one, linked to +// by the first, as per the EXIF/TIFF specification. +// - We set the offset to (0) now but will allocate the data and properly assign +// the offset when the IB is encoded (later). +func (ib *IfdBuilder) SetThumbnail(data []byte) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if ib.IfdIdentity().UnindexedString() != exifcommon.IfdStandardIfdIdentity.UnindexedString() { + log.Panicf("thumbnails can only go into a root Ifd (and only the second one)") + } + + // TODO(dustin): !! Add a test for this function. + + if data == nil || len(data) == 0 { + log.Panic("thumbnail is empty") + } + + ib.thumbnailData = data + + ibtvfb := NewIfdBuilderTagValueFromBytes(ib.thumbnailData) + offsetBt := + NewBuilderTag( + ib.IfdIdentity().UnindexedString(), + ThumbnailOffsetTagId, + exifcommon.TypeLong, + ibtvfb, + ib.byteOrder) + + err = ib.Set(offsetBt) + log.PanicIf(err) + + thumbnailSizeIt, err := ib.tagIndex.Get(ib.IfdIdentity(), ThumbnailSizeTagId) + log.PanicIf(err) + + sizeBt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), thumbnailSizeIt, ib.byteOrder, []uint32{uint32(len(ib.thumbnailData))}) + + err = ib.Set(sizeBt) + log.PanicIf(err) + + return nil +} + +func (ib *IfdBuilder) Thumbnail() []byte { + return ib.thumbnailData +} + +func (ib *IfdBuilder) printTagTree(levels int) { + indent := strings.Repeat(" ", levels*2) + + i := 0 + for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { + prefix := " " + if i > 0 { + prefix = ">" + } + + if levels == 0 { + fmt.Printf("%s%sIFD: %s INDEX=(%d)\n", indent, prefix, currentIb, i) + } else { + fmt.Printf("%s%sChild IFD: %s\n", indent, prefix, currentIb) + } + + if len(currentIb.tags) > 0 { + fmt.Printf("\n") + + for i, tag := range currentIb.tags { + isChildIb := false + _, err := ib.ifdMapping.GetChild(currentIb.IfdIdentity().UnindexedString(), tag.tagId) + if err == nil { + isChildIb = true + } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { + log.Panic(err) + } + + tagName := "" + + // If a normal tag (not a child IFD) get the name. + if isChildIb == true { + tagName = "" + } else { + it, err := ib.tagIndex.Get(ib.ifdIdentity, tag.tagId) + if log.Is(err, ErrTagNotFound) == true { + tagName = "" + } else if err != nil { + log.Panic(err) + } else { + tagName = it.Name + } + } + + value := tag.Value() + + if value.IsIb() == true { + fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, value.Ib()) + } else { + fmt.Printf("%s (%d): [%s] %s\n", indent, i, tagName, tag) + } + + if isChildIb == true { + if tag.value.IsIb() == false { + log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) + } + + fmt.Printf("\n") + + childIb := tag.value.Ib() + childIb.printTagTree(levels + 1) + } + } + + fmt.Printf("\n") + } + + i++ + } +} + +func (ib *IfdBuilder) PrintTagTree() { + ib.printTagTree(0) +} + +func (ib *IfdBuilder) printIfdTree(levels int) { + indent := strings.Repeat(" ", levels*2) + + i := 0 + for currentIb := ib; currentIb != nil; currentIb = currentIb.nextIb { + prefix := " " + if i > 0 { + prefix = ">" + } + + fmt.Printf("%s%s%s\n", indent, prefix, currentIb) + + if len(currentIb.tags) > 0 { + for _, tag := range currentIb.tags { + isChildIb := false + _, err := ib.ifdMapping.GetChild(currentIb.IfdIdentity().UnindexedString(), tag.tagId) + if err == nil { + isChildIb = true + } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { + log.Panic(err) + } + + if isChildIb == true { + if tag.value.IsIb() == false { + log.Panicf("tag-ID (0x%04x) is an IFD but the tag value is not an IB instance: %v", tag.tagId, tag) + } + + childIb := tag.value.Ib() + childIb.printIfdTree(levels + 1) + } + } + } + + i++ + } +} + +func (ib *IfdBuilder) PrintIfdTree() { + ib.printIfdTree(0) +} + +func (ib *IfdBuilder) dumpToStrings(thisIb *IfdBuilder, prefix string, tagId uint16, lines []string) (linesOutput []string) { + if lines == nil { + linesOutput = make([]string, 0) + } else { + linesOutput = lines + } + + siblingIfdIndex := 0 + for ; thisIb != nil; thisIb = thisIb.nextIb { + line := fmt.Sprintf("IFD", prefix, thisIb.IfdIdentity().String(), siblingIfdIndex, thisIb.IfdIdentity().TagId(), tagId) + linesOutput = append(linesOutput, line) + + for i, tag := range thisIb.tags { + var childIb *IfdBuilder + childIfdName := "" + if tag.value.IsIb() == true { + childIb = tag.value.Ib() + childIfdName = childIb.IfdIdentity().UnindexedString() + } + + line := fmt.Sprintf("TAG", prefix, thisIb.IfdIdentity().String(), thisIb.IfdIdentity().TagId(), childIfdName, i, tag.tagId) + linesOutput = append(linesOutput, line) + + if childIb == nil { + continue + } + + childPrefix := "" + if prefix == "" { + childPrefix = fmt.Sprintf("%s", thisIb.IfdIdentity().UnindexedString()) + } else { + childPrefix = fmt.Sprintf("%s->%s", prefix, thisIb.IfdIdentity().UnindexedString()) + } + + linesOutput = thisIb.dumpToStrings(childIb, childPrefix, tag.tagId, linesOutput) + } + + siblingIfdIndex++ + } + + return linesOutput +} + +func (ib *IfdBuilder) DumpToStrings() (lines []string) { + return ib.dumpToStrings(ib, "", 0, lines) +} + +func (ib *IfdBuilder) SetNextIb(nextIb *IfdBuilder) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ib.nextIb = nextIb + + return nil +} + +func (ib *IfdBuilder) DeleteN(tagId uint16, n int) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if n < 1 { + log.Panicf("N must be at least 1: (%d)", n) + } + + for n > 0 { + j := -1 + for i, bt := range ib.tags { + if bt.tagId == tagId { + j = i + break + } + } + + if j == -1 { + log.Panic(ErrTagEntryNotFound) + } + + ib.tags = append(ib.tags[:j], ib.tags[j+1:]...) + n-- + } + + return nil +} + +func (ib *IfdBuilder) DeleteFirst(tagId uint16) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = ib.DeleteN(tagId, 1) + log.PanicIf(err) + + return nil +} + +func (ib *IfdBuilder) DeleteAll(tagId uint16) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for { + err = ib.DeleteN(tagId, 1) + if log.Is(err, ErrTagEntryNotFound) == true { + break + } else if err != nil { + log.Panic(err) + } + + n++ + } + + return n, nil +} + +func (ib *IfdBuilder) ReplaceAt(position int, bt *BuilderTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if position < 0 { + log.Panicf("replacement position must be 0 or greater") + } else if position >= len(ib.tags) { + log.Panicf("replacement position does not exist") + } + + ib.tags[position] = bt + + return nil +} + +func (ib *IfdBuilder) Replace(tagId uint16, bt *BuilderTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + position, err := ib.Find(tagId) + log.PanicIf(err) + + ib.tags[position] = bt + + return nil +} + +// Set will add a new entry or update an existing entry. +func (ib *IfdBuilder) Set(bt *BuilderTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + position, err := ib.Find(bt.tagId) + if err == nil { + ib.tags[position] = bt + } else if log.Is(err, ErrTagEntryNotFound) == true { + err = ib.add(bt) + log.PanicIf(err) + } else { + log.Panic(err) + } + + return nil +} + +func (ib *IfdBuilder) FindN(tagId uint16, maxFound int) (found []int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + found = make([]int, 0) + + for i, bt := range ib.tags { + if bt.tagId == tagId { + found = append(found, i) + if maxFound == 0 || len(found) >= maxFound { + break + } + } + } + + return found, nil +} + +func (ib *IfdBuilder) Find(tagId uint16) (position int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + found, err := ib.FindN(tagId, 1) + log.PanicIf(err) + + if len(found) == 0 { + log.Panic(ErrTagEntryNotFound) + } + + return found[0], nil +} + +func (ib *IfdBuilder) FindTag(tagId uint16) (bt *BuilderTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + found, err := ib.FindN(tagId, 1) + log.PanicIf(err) + + if len(found) == 0 { + log.Panic(ErrTagEntryNotFound) + } + + position := found[0] + + return ib.tags[position], nil +} + +func (ib *IfdBuilder) FindTagWithName(tagName string) (bt *BuilderTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) + log.PanicIf(err) + + found, err := ib.FindN(it.Id, 1) + log.PanicIf(err) + + if len(found) == 0 { + log.Panic(ErrTagEntryNotFound) + } + + position := found[0] + + return ib.tags[position], nil +} + +func (ib *IfdBuilder) add(bt *BuilderTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if bt.ifdPath == "" { + log.Panicf("BuilderTag ifdPath is not set: %s", bt) + } else if bt.typeId == 0x0 { + log.Panicf("BuilderTag type-ID is not set: %s", bt) + } else if bt.value == nil { + log.Panicf("BuilderTag value is not set: %s", bt) + } + + ib.tags = append(ib.tags, bt) + return nil +} + +func (ib *IfdBuilder) Add(bt *BuilderTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if bt.value.IsIb() == true { + log.Panicf("child IfdBuilders must be added via AddChildIb() or AddTagsFromExisting(), not Add()") + } + + err = ib.add(bt) + log.PanicIf(err) + + return nil +} + +// AddChildIb adds a tag that branches to a new IFD. +func (ib *IfdBuilder) AddChildIb(childIb *IfdBuilder) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if childIb.IfdIdentity().TagId() == 0 { + log.Panicf("IFD can not be used as a child IFD (not associated with a tag-ID): %v", childIb) + } else if childIb.byteOrder != ib.byteOrder { + log.Panicf("Child IFD does not have the same byte-order: [%s] != [%s]", childIb.byteOrder, ib.byteOrder) + } + + // Since no standard IFDs supports occur`ring more than once, check that a + // tag of this type has not been previously added. Note that we just search + // the current IFD and *not every* IFD. + for _, bt := range childIb.tags { + if bt.tagId == childIb.IfdIdentity().TagId() { + log.Panicf("child-IFD already added: %v", childIb.IfdIdentity().UnindexedString()) + } + } + + bt := ib.NewBuilderTagFromBuilder(childIb) + ib.tags = append(ib.tags, bt) + + return nil +} + +func (ib *IfdBuilder) NewBuilderTagFromBuilder(childIb *IfdBuilder) (bt *BuilderTag) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + value := NewIfdBuilderTagValueFromIfdBuilder(childIb) + + bt = NewChildIfdBuilderTag( + ib.IfdIdentity().UnindexedString(), + childIb.IfdIdentity().TagId(), + value) + + return bt +} + +// AddTagsFromExisting does a verbatim copy of the entries in `ifd` to this +// builder. It excludes child IFDs. These must be added explicitly via +// `AddChildIb()`. +func (ib *IfdBuilder) AddTagsFromExisting(ifd *Ifd, includeTagIds []uint16, excludeTagIds []uint16) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + thumbnailData, err := ifd.Thumbnail() + if err == nil { + err = ib.SetThumbnail(thumbnailData) + log.PanicIf(err) + } else if log.Is(err, ErrNoThumbnail) == false { + log.Panic(err) + } + + for i, ite := range ifd.Entries() { + if ite.IsThumbnailOffset() == true || ite.IsThumbnailSize() { + // These will be added on-the-fly when we encode. + continue + } + + if excludeTagIds != nil && len(excludeTagIds) > 0 { + found := false + for _, excludedTagId := range excludeTagIds { + if excludedTagId == ite.TagId() { + found = true + } + } + + if found == true { + continue + } + } + + if includeTagIds != nil && len(includeTagIds) > 0 { + // Whether or not there was a list of excludes, if there is a list + // of includes than the current tag has to be in it. + + found := false + for _, includedTagId := range includeTagIds { + if includedTagId == ite.TagId() { + found = true + break + } + } + + if found == false { + continue + } + } + + var bt *BuilderTag + + if ite.ChildIfdPath() != "" { + // If we want to add an IFD tag, we'll have to build it first and + // *then* add it via a different method. + + // Figure out which of the child-IFDs that are associated with + // this IFD represents this specific child IFD. + + var childIfd *Ifd + for _, thisChildIfd := range ifd.Children() { + if thisChildIfd.ParentTagIndex() != i { + continue + } else if thisChildIfd.ifdIdentity.TagId() != 0xffff && thisChildIfd.ifdIdentity.TagId() != ite.TagId() { + log.Panicf("child-IFD tag is not correct: TAG-POSITION=(%d) ITE=%s CHILD-IFD=%s", thisChildIfd.ParentTagIndex(), ite, thisChildIfd) + } + + childIfd = thisChildIfd + break + } + + if childIfd == nil { + childTagIds := make([]string, len(ifd.Children())) + for j, childIfd := range ifd.Children() { + childTagIds[j] = fmt.Sprintf("0x%04x (parent tag-position %d)", childIfd.ifdIdentity.TagId(), childIfd.ParentTagIndex()) + } + + log.Panicf("could not find child IFD for child ITE: IFD-PATH=[%s] TAG-ID=(0x%04x) CURRENT-TAG-POSITION=(%d) CHILDREN=%v", ite.IfdPath(), ite.TagId(), i, childTagIds) + } + + childIb := NewIfdBuilderFromExistingChain(childIfd) + bt = ib.NewBuilderTagFromBuilder(childIb) + } else { + // Non-IFD tag. + + rawBytes, err := ite.GetRawBytes() + log.PanicIf(err) + + value := NewIfdBuilderTagValueFromBytes(rawBytes) + + bt = NewBuilderTag( + ifd.ifdIdentity.UnindexedString(), + ite.TagId(), + ite.TagType(), + value, + ib.byteOrder) + } + + err := ib.add(bt) + log.PanicIf(err) + } + + return nil +} + +// AddStandard quickly and easily composes and adds the tag using the +// information already known about a tag. Only works with standard tags. +func (ib *IfdBuilder) AddStandard(tagId uint16, value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + it, err := ib.tagIndex.Get(ib.IfdIdentity(), tagId) + log.PanicIf(err) + + bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) + + err = ib.add(bt) + log.PanicIf(err) + + return nil +} + +// AddStandardWithName quickly and easily composes and adds the tag using the +// information already known about a tag (using the name). Only works with +// standard tags. +func (ib *IfdBuilder) AddStandardWithName(tagName string, value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) + log.PanicIf(err) + + bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) + + err = ib.add(bt) + log.PanicIf(err) + + return nil +} + +// SetStandard quickly and easily composes and adds or replaces the tag using +// the information already known about a tag. Only works with standard tags. +func (ib *IfdBuilder) SetStandard(tagId uint16, value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test for this function. + + it, err := ib.tagIndex.Get(ib.IfdIdentity(), tagId) + log.PanicIf(err) + + bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) + + i, err := ib.Find(tagId) + if err != nil { + if log.Is(err, ErrTagEntryNotFound) == false { + log.Panic(err) + } + + ib.tags = append(ib.tags, bt) + } else { + ib.tags[i] = bt + } + + return nil +} + +// SetStandardWithName quickly and easily composes and adds or replaces the +// tag using the information already known about a tag (using the name). Only +// works with standard tags. +func (ib *IfdBuilder) SetStandardWithName(tagName string, value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test for this function. + + it, err := ib.tagIndex.GetWithName(ib.IfdIdentity(), tagName) + log.PanicIf(err) + + bt := NewStandardBuilderTag(ib.IfdIdentity().UnindexedString(), it, ib.byteOrder, value) + + i, err := ib.Find(bt.tagId) + if err != nil { + if log.Is(err, ErrTagEntryNotFound) == false { + log.Panic(err) + } + + ib.tags = append(ib.tags, bt) + } else { + ib.tags[i] = bt + } + + return nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/ifd_builder_encode.go b/vendor/github.com/dsoprea/go-exif/v3/ifd_builder_encode.go new file mode 100644 index 000000000..a0f4ff79c --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/ifd_builder_encode.go @@ -0,0 +1,532 @@ +package exif + +import ( + "bytes" + "fmt" + "strings" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +const ( + // Tag-ID + Tag-Type + Unit-Count + Value/Offset. + IfdTagEntrySize = uint32(2 + 2 + 4 + 4) +) + +type ByteWriter struct { + b *bytes.Buffer + byteOrder binary.ByteOrder +} + +func NewByteWriter(b *bytes.Buffer, byteOrder binary.ByteOrder) (bw *ByteWriter) { + return &ByteWriter{ + b: b, + byteOrder: byteOrder, + } +} + +func (bw ByteWriter) writeAsBytes(value interface{}) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = binary.Write(bw.b, bw.byteOrder, value) + log.PanicIf(err) + + return nil +} + +func (bw ByteWriter) WriteUint32(value uint32) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = bw.writeAsBytes(value) + log.PanicIf(err) + + return nil +} + +func (bw ByteWriter) WriteUint16(value uint16) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = bw.writeAsBytes(value) + log.PanicIf(err) + + return nil +} + +func (bw ByteWriter) WriteFourBytes(value []byte) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + len_ := len(value) + if len_ != 4 { + log.Panicf("value is not four-bytes: (%d)", len_) + } + + _, err = bw.b.Write(value) + log.PanicIf(err) + + return nil +} + +// ifdOffsetIterator keeps track of where the next IFD should be written by +// keeping track of where the offsets start, the data that has been added, and +// bumping the offset *when* the data is added. +type ifdDataAllocator struct { + offset uint32 + b bytes.Buffer +} + +func newIfdDataAllocator(ifdDataAddressableOffset uint32) *ifdDataAllocator { + return &ifdDataAllocator{ + offset: ifdDataAddressableOffset, + } +} + +func (ida *ifdDataAllocator) Allocate(value []byte) (offset uint32, err error) { + _, err = ida.b.Write(value) + log.PanicIf(err) + + offset = ida.offset + ida.offset += uint32(len(value)) + + return offset, nil +} + +func (ida *ifdDataAllocator) NextOffset() uint32 { + return ida.offset +} + +func (ida *ifdDataAllocator) Bytes() []byte { + return ida.b.Bytes() +} + +// IfdByteEncoder converts an IB to raw bytes (for writing) while also figuring +// out all of the allocations and indirection that is required for extended +// data. +type IfdByteEncoder struct { + // journal holds a list of actions taken while encoding. + journal [][3]string +} + +func NewIfdByteEncoder() (ibe *IfdByteEncoder) { + return &IfdByteEncoder{ + journal: make([][3]string, 0), + } +} + +func (ibe *IfdByteEncoder) Journal() [][3]string { + return ibe.journal +} + +func (ibe *IfdByteEncoder) TableSize(entryCount int) uint32 { + // Tag-Count + (Entry-Size * Entry-Count) + Next-IFD-Offset. + return uint32(2) + (IfdTagEntrySize * uint32(entryCount)) + uint32(4) +} + +func (ibe *IfdByteEncoder) pushToJournal(where, direction, format string, args ...interface{}) { + event := [3]string{ + direction, + where, + fmt.Sprintf(format, args...), + } + + ibe.journal = append(ibe.journal, event) +} + +// PrintJournal prints a hierarchical representation of the steps taken during +// encoding. +func (ibe *IfdByteEncoder) PrintJournal() { + maxWhereLength := 0 + for _, event := range ibe.journal { + where := event[1] + + len_ := len(where) + if len_ > maxWhereLength { + maxWhereLength = len_ + } + } + + level := 0 + for i, event := range ibe.journal { + direction := event[0] + where := event[1] + message := event[2] + + if direction != ">" && direction != "<" && direction != "-" { + log.Panicf("journal operation not valid: [%s]", direction) + } + + if direction == "<" { + if level <= 0 { + log.Panicf("journal operations unbalanced (too many closes)") + } + + level-- + } + + indent := strings.Repeat(" ", level) + + fmt.Printf("%3d %s%s %s: %s\n", i, indent, direction, where, message) + + if direction == ">" { + level++ + } + } + + if level != 0 { + log.Panicf("journal operations unbalanced (too many opens)") + } +} + +// encodeTagToBytes encodes the given tag to a byte stream. If +// `nextIfdOffsetToWrite` is more than (0), recurse into child IFDs +// (`nextIfdOffsetToWrite` is required in order for them to know where the its +// IFD data will be written, in order for them to know the offset of where +// their allocated-data block will start, which follows right behind). +func (ibe *IfdByteEncoder) encodeTagToBytes(ib *IfdBuilder, bt *BuilderTag, bw *ByteWriter, ida *ifdDataAllocator, nextIfdOffsetToWrite uint32) (childIfdBlock []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Write tag-ID. + err = bw.WriteUint16(bt.tagId) + log.PanicIf(err) + + // Works for both values and child IFDs (which have an official size of + // LONG). + err = bw.WriteUint16(uint16(bt.typeId)) + log.PanicIf(err) + + // Write unit-count. + + if bt.value.IsBytes() == true { + effectiveType := bt.typeId + if bt.typeId == exifcommon.TypeUndefined { + effectiveType = exifcommon.TypeByte + } + + // It's a non-unknown value.Calculate the count of values of + // the type that we're writing and the raw bytes for the whole list. + + typeSize := uint32(effectiveType.Size()) + + valueBytes := bt.value.Bytes() + + len_ := len(valueBytes) + unitCount := uint32(len_) / typeSize + + if _, found := tagsWithoutAlignment[bt.tagId]; found == false { + remainder := uint32(len_) % typeSize + + if remainder > 0 { + log.Panicf("tag (0x%04x) value of (%d) bytes not evenly divisible by type-size (%d)", bt.tagId, len_, typeSize) + } + } + + err = bw.WriteUint32(unitCount) + log.PanicIf(err) + + // Write four-byte value/offset. + + if len_ > 4 { + offset, err := ida.Allocate(valueBytes) + log.PanicIf(err) + + err = bw.WriteUint32(offset) + log.PanicIf(err) + } else { + fourBytes := make([]byte, 4) + copy(fourBytes, valueBytes) + + err = bw.WriteFourBytes(fourBytes) + log.PanicIf(err) + } + } else { + if bt.value.IsIb() == false { + log.Panicf("tag value is not a byte-slice but also not a child IB: %v", bt) + } + + // Write unit-count (one LONG representing one offset). + err = bw.WriteUint32(1) + log.PanicIf(err) + + if nextIfdOffsetToWrite > 0 { + var err error + + ibe.pushToJournal("encodeTagToBytes", ">", "[%s]->[%s]", ib.IfdIdentity().UnindexedString(), bt.value.Ib().IfdIdentity().UnindexedString()) + + // Create the block of IFD data and everything it requires. + childIfdBlock, err = ibe.encodeAndAttachIfd(bt.value.Ib(), nextIfdOffsetToWrite) + log.PanicIf(err) + + ibe.pushToJournal("encodeTagToBytes", "<", "[%s]->[%s]", bt.value.Ib().IfdIdentity().UnindexedString(), ib.IfdIdentity().UnindexedString()) + + // Use the next-IFD offset for it. The IFD will actually get + // attached after we return. + err = bw.WriteUint32(nextIfdOffsetToWrite) + log.PanicIf(err) + + } else { + // No child-IFDs are to be allocated. Finish the entry with a NULL + // pointer. + + ibe.pushToJournal("encodeTagToBytes", "-", "*Not* descending to child: [%s]", bt.value.Ib().IfdIdentity().UnindexedString()) + + err = bw.WriteUint32(0) + log.PanicIf(err) + } + } + + return childIfdBlock, nil +} + +// encodeIfdToBytes encodes the given IB to a byte-slice. We are given the +// offset at which this IFD will be written. This method is used called both to +// pre-determine how big the table is going to be (so that we can calculate the +// address to allocate data at) as well as to write the final table. +// +// It is necessary to fully realize the table in order to predetermine its size +// because it is not enough to know the size of the table: If there are child +// IFDs, we will not be able to allocate them without first knowing how much +// data we need to allocate for the current IFD. +func (ibe *IfdByteEncoder) encodeIfdToBytes(ib *IfdBuilder, ifdAddressableOffset uint32, nextIfdOffsetToWrite uint32, setNextIb bool) (data []byte, tableSize uint32, dataSize uint32, childIfdSizes []uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ibe.pushToJournal("encodeIfdToBytes", ">", "%s", ib) + + tableSize = ibe.TableSize(len(ib.tags)) + + b := new(bytes.Buffer) + bw := NewByteWriter(b, ib.byteOrder) + + // Write tag count. + err = bw.WriteUint16(uint16(len(ib.tags))) + log.PanicIf(err) + + ida := newIfdDataAllocator(ifdAddressableOffset) + + childIfdBlocks := make([][]byte, 0) + + // Write raw bytes for each tag entry. Allocate larger data to be referred + // to in the follow-up data-block as required. Any "unknown"-byte tags that + // we can't parse will not be present here (using AddTagsFromExisting(), at + // least). + for _, bt := range ib.tags { + childIfdBlock, err := ibe.encodeTagToBytes(ib, bt, bw, ida, nextIfdOffsetToWrite) + log.PanicIf(err) + + if childIfdBlock != nil { + // We aren't allowed to have non-nil child IFDs if we're just + // sizing things up. + if nextIfdOffsetToWrite == 0 { + log.Panicf("no IFD offset provided for child-IFDs; no new child-IFDs permitted") + } + + nextIfdOffsetToWrite += uint32(len(childIfdBlock)) + childIfdBlocks = append(childIfdBlocks, childIfdBlock) + } + } + + dataBytes := ida.Bytes() + dataSize = uint32(len(dataBytes)) + + childIfdSizes = make([]uint32, len(childIfdBlocks)) + childIfdsTotalSize := uint32(0) + for i, childIfdBlock := range childIfdBlocks { + len_ := uint32(len(childIfdBlock)) + childIfdSizes[i] = len_ + childIfdsTotalSize += len_ + } + + // N the link from this IFD to the next IFD that will be written in the + // next cycle. + if setNextIb == true { + // Write address of next IFD in chain. This will be the original + // allocation offset plus the size of everything we have allocated for + // this IFD and its child-IFDs. + // + // It is critical that this number is stepped properly. We experienced + // an issue whereby it first looked like we were duplicating the IFD and + // then that we were duplicating the tags in the wrong IFD, and then + // finally we determined that the next-IFD offset for the first IFD was + // accidentally pointing back to the EXIF IFD, so we were visiting it + // twice when visiting through the tags after decoding. It was an + // expensive bug to find. + + ibe.pushToJournal("encodeIfdToBytes", "-", "Setting 'next' IFD to (0x%08x).", nextIfdOffsetToWrite) + + err := bw.WriteUint32(nextIfdOffsetToWrite) + log.PanicIf(err) + } else { + err := bw.WriteUint32(0) + log.PanicIf(err) + } + + _, err = b.Write(dataBytes) + log.PanicIf(err) + + // Append any child IFD blocks after our table and data blocks. These IFDs + // were equipped with the appropriate offset information so it's expected + // that all offsets referred to by these will be correct. + // + // Note that child-IFDs are append after the current IFD and before the + // next IFD, as opposed to the root IFDs, which are chained together but + // will be interrupted by these child-IFDs (which is expected, per the + // standard). + + for _, childIfdBlock := range childIfdBlocks { + _, err = b.Write(childIfdBlock) + log.PanicIf(err) + } + + ibe.pushToJournal("encodeIfdToBytes", "<", "%s", ib) + + return b.Bytes(), tableSize, dataSize, childIfdSizes, nil +} + +// encodeAndAttachIfd is a reentrant function that processes the IFD chain. +func (ibe *IfdByteEncoder) encodeAndAttachIfd(ib *IfdBuilder, ifdAddressableOffset uint32) (data []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ibe.pushToJournal("encodeAndAttachIfd", ">", "%s", ib) + + b := new(bytes.Buffer) + + i := 0 + + for thisIb := ib; thisIb != nil; thisIb = thisIb.nextIb { + + // Do a dry-run in order to pre-determine its size requirement. + + ibe.pushToJournal("encodeAndAttachIfd", ">", "Beginning encoding process: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) + + ibe.pushToJournal("encodeAndAttachIfd", ">", "Calculating size: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) + + _, tableSize, allocatedDataSize, _, err := ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, 0, false) + log.PanicIf(err) + + ibe.pushToJournal("encodeAndAttachIfd", "<", "Finished calculating size: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) + + ifdAddressableOffset += tableSize + nextIfdOffsetToWrite := ifdAddressableOffset + allocatedDataSize + + ibe.pushToJournal("encodeAndAttachIfd", ">", "Next IFD will be written at offset (0x%08x)", nextIfdOffsetToWrite) + + // Write our IFD as well as any child-IFDs (now that we know the offset + // where new IFDs and their data will be allocated). + + setNextIb := thisIb.nextIb != nil + + ibe.pushToJournal("encodeAndAttachIfd", ">", "Encoding starting: (%d) [%s] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, thisIb.IfdIdentity().UnindexedString(), nextIfdOffsetToWrite) + + tableAndAllocated, effectiveTableSize, effectiveAllocatedDataSize, childIfdSizes, err := + ibe.encodeIfdToBytes(thisIb, ifdAddressableOffset, nextIfdOffsetToWrite, setNextIb) + + log.PanicIf(err) + + if effectiveTableSize != tableSize { + log.Panicf("written table size does not match the pre-calculated table size: (%d) != (%d) %s", effectiveTableSize, tableSize, ib) + } else if effectiveAllocatedDataSize != allocatedDataSize { + log.Panicf("written allocated-data size does not match the pre-calculated allocated-data size: (%d) != (%d) %s", effectiveAllocatedDataSize, allocatedDataSize, ib) + } + + ibe.pushToJournal("encodeAndAttachIfd", "<", "Encoding done: (%d) [%s]", i, thisIb.IfdIdentity().UnindexedString()) + + totalChildIfdSize := uint32(0) + for _, childIfdSize := range childIfdSizes { + totalChildIfdSize += childIfdSize + } + + if len(tableAndAllocated) != int(tableSize+allocatedDataSize+totalChildIfdSize) { + log.Panicf("IFD table and data is not a consistent size: (%d) != (%d)", len(tableAndAllocated), tableSize+allocatedDataSize+totalChildIfdSize) + } + + // TODO(dustin): We might want to verify the original tableAndAllocated length, too. + + _, err = b.Write(tableAndAllocated) + log.PanicIf(err) + + // Advance past what we've allocated, thus far. + + ifdAddressableOffset += allocatedDataSize + totalChildIfdSize + + ibe.pushToJournal("encodeAndAttachIfd", "<", "Finishing encoding process: (%d) [%s] [FINAL:] NEXT-IFD-OFFSET-TO-WRITE=(0x%08x)", i, ib.IfdIdentity().UnindexedString(), nextIfdOffsetToWrite) + + i++ + } + + ibe.pushToJournal("encodeAndAttachIfd", "<", "%s", ib) + + return b.Bytes(), nil +} + +// EncodeToExifPayload is the base encoding step that transcribes the entire IB +// structure to its on-disk layout. +func (ibe *IfdByteEncoder) EncodeToExifPayload(ib *IfdBuilder) (data []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + data, err = ibe.encodeAndAttachIfd(ib, ExifDefaultFirstIfdOffset) + log.PanicIf(err) + + return data, nil +} + +// EncodeToExif calls EncodeToExifPayload and then packages the result into a +// complete EXIF block. +func (ibe *IfdByteEncoder) EncodeToExif(ib *IfdBuilder) (data []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + encodedIfds, err := ibe.EncodeToExifPayload(ib) + log.PanicIf(err) + + // Wrap the IFD in a formal EXIF block. + + b := new(bytes.Buffer) + + headerBytes, err := BuildExifHeader(ib.byteOrder, ExifDefaultFirstIfdOffset) + log.PanicIf(err) + + _, err = b.Write(headerBytes) + log.PanicIf(err) + + _, err = b.Write(encodedIfds) + log.PanicIf(err) + + return b.Bytes(), nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/ifd_enumerate.go b/vendor/github.com/dsoprea/go-exif/v3/ifd_enumerate.go new file mode 100644 index 000000000..3167596ef --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/ifd_enumerate.go @@ -0,0 +1,1672 @@ +package exif + +import ( + "bytes" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-exif/v3/undefined" +) + +var ( + ifdEnumerateLogger = log.NewLogger("exif.ifd_enumerate") +) + +var ( + // ErrNoThumbnail means that no thumbnail was found. + ErrNoThumbnail = errors.New("no thumbnail") + + // ErrNoGpsTags means that no GPS info was found. + ErrNoGpsTags = errors.New("no gps tags") + + // ErrTagTypeNotValid means that the tag-type is not valid. + ErrTagTypeNotValid = errors.New("tag type invalid") + + // ErrOffsetInvalid means that the file offset is not valid. + ErrOffsetInvalid = errors.New("file offset invalid") +) + +var ( + // ValidGpsVersions is the list of recognized EXIF GPS versions/signatures. + ValidGpsVersions = [][4]byte{ + // 2.0.0.0 appears to have a very similar format to 2.2.0.0, so enabling + // it under that assumption. + // + // IFD-PATH=[IFD] ID=(0x8825) NAME=[GPSTag] COUNT=(1) TYPE=[LONG] VALUE=[114] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0000) NAME=[GPSVersionID] COUNT=(4) TYPE=[BYTE] VALUE=[02 00 00 00] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0001) NAME=[GPSLatitudeRef] COUNT=(2) TYPE=[ASCII] VALUE=[S] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0002) NAME=[GPSLatitude] COUNT=(3) TYPE=[RATIONAL] VALUE=[38/1...] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0003) NAME=[GPSLongitudeRef] COUNT=(2) TYPE=[ASCII] VALUE=[E] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0004) NAME=[GPSLongitude] COUNT=(3) TYPE=[RATIONAL] VALUE=[144/1...] + // IFD-PATH=[IFD/GPSInfo] ID=(0x0012) NAME=[GPSMapDatum] COUNT=(7) TYPE=[ASCII] VALUE=[WGS-84] + // + {2, 0, 0, 0}, + + {2, 2, 0, 0}, + + // Suddenly appeared at the default in 2.31: https://home.jeita.or.jp/tsc/std-pdf/CP-3451D.pdf + // + // Note that the presence of 2.3.0.0 doesn't seem to guarantee + // coordinates. In some cases, we seen just the following: + // + // GPS Tag Version |2.3.0.0 + // GPS Receiver Status |V + // Geodetic Survey Data|WGS-84 + // GPS Differential Cor|0 + // + {2, 3, 0, 0}, + } +) + +// byteParser knows how to decode an IFD and all of the tags it +// describes. +// +// The IFDs and the actual values can float throughout the EXIF block, but the +// IFD itself is just a minor header followed by a set of repeating, +// statically-sized records. So, the tags (though notnecessarily their values) +// are fairly simple to enumerate. +type byteParser struct { + byteOrder binary.ByteOrder + rs io.ReadSeeker + ifdOffset uint32 + currentOffset uint32 +} + +// newByteParser returns a new byteParser struct. +// +// initialOffset is for arithmetic-based tracking of where we should be at in +// the stream. +func newByteParser(rs io.ReadSeeker, byteOrder binary.ByteOrder, initialOffset uint32) (bp *byteParser, err error) { + // TODO(dustin): Add test + + bp = &byteParser{ + rs: rs, + byteOrder: byteOrder, + currentOffset: initialOffset, + } + + return bp, nil +} + +// getUint16 reads a uint16 and advances both our current and our current +// accumulator (which allows us to know how far to seek to the beginning of the +// next IFD when it's time to jump). +func (bp *byteParser) getUint16() (value uint16, raw []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + needBytes := 2 + + raw = make([]byte, needBytes) + + _, err = io.ReadFull(bp.rs, raw) + log.PanicIf(err) + + value = bp.byteOrder.Uint16(raw) + + bp.currentOffset += uint32(needBytes) + + return value, raw, nil +} + +// getUint32 reads a uint32 and advances both our current and our current +// accumulator (which allows us to know how far to seek to the beginning of the +// next IFD when it's time to jump). +func (bp *byteParser) getUint32() (value uint32, raw []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + needBytes := 4 + + raw = make([]byte, needBytes) + + _, err = io.ReadFull(bp.rs, raw) + log.PanicIf(err) + + value = bp.byteOrder.Uint32(raw) + + bp.currentOffset += uint32(needBytes) + + return value, raw, nil +} + +// CurrentOffset returns the starting offset but the number of bytes that we +// have parsed. This is arithmetic-based tracking, not a seek(0) operation. +func (bp *byteParser) CurrentOffset() uint32 { + return bp.currentOffset +} + +// IfdEnumerate is the main enumeration type. It knows how to parse the IFD +// containers in the EXIF blob. +type IfdEnumerate struct { + ebs ExifBlobSeeker + byteOrder binary.ByteOrder + tagIndex *TagIndex + ifdMapping *exifcommon.IfdMapping + furthestOffset uint32 + + visitedIfdOffsets map[uint32]struct{} +} + +// NewIfdEnumerate returns a new instance of IfdEnumerate. +func NewIfdEnumerate(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ebs ExifBlobSeeker, byteOrder binary.ByteOrder) *IfdEnumerate { + return &IfdEnumerate{ + ebs: ebs, + byteOrder: byteOrder, + ifdMapping: ifdMapping, + tagIndex: tagIndex, + + visitedIfdOffsets: make(map[uint32]struct{}), + } +} + +func (ie *IfdEnumerate) getByteParser(ifdOffset uint32) (bp *byteParser, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + initialOffset := ExifAddressableAreaStart + ifdOffset + + rs, err := ie.ebs.GetReadSeeker(int64(initialOffset)) + log.PanicIf(err) + + bp, err = + newByteParser( + rs, + ie.byteOrder, + initialOffset) + + if err != nil { + if err == ErrOffsetInvalid { + return nil, err + } + + log.Panic(err) + } + + return bp, nil +} + +func (ie *IfdEnumerate) parseTag(ii *exifcommon.IfdIdentity, tagPosition int, bp *byteParser) (ite *IfdTagEntry, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + tagId, _, err := bp.getUint16() + log.PanicIf(err) + + tagTypeRaw, _, err := bp.getUint16() + log.PanicIf(err) + + tagType := exifcommon.TagTypePrimitive(tagTypeRaw) + + unitCount, _, err := bp.getUint32() + log.PanicIf(err) + + valueOffset, rawValueOffset, err := bp.getUint32() + log.PanicIf(err) + + // Check whether the embedded type indicator is valid. + + if tagType.IsValid() == false { + // Technically, we have the type on-file in the tags-index, but + // if the type stored alongside the data disagrees with it, + // which it apparently does, all bets are off. + ifdEnumerateLogger.Warningf(nil, + "Tag (0x%04x) in IFD [%s] at position (%d) has invalid type (0x%04x) and will be skipped.", + tagId, ii, tagPosition, int(tagType)) + + ite = &IfdTagEntry{ + tagId: tagId, + tagType: tagType, + } + + return ite, ErrTagTypeNotValid + } + + // Check whether the embedded type is listed among the supported types for + // the registered tag. If not, skip processing the tag. + + it, err := ie.tagIndex.Get(ii, tagId) + if err != nil { + if log.Is(err, ErrTagNotFound) == true { + ifdEnumerateLogger.Warningf(nil, "Tag (0x%04x) is not known and will be skipped.", tagId) + + ite = &IfdTagEntry{ + tagId: tagId, + } + + return ite, ErrTagNotFound + } + + log.Panic(err) + } + + // If we're trying to be as forgiving as possible then use whatever type was + // reported in the format. Otherwise, only accept a type that's expected for + // this tag. + if ie.tagIndex.UniversalSearch() == false && it.DoesSupportType(tagType) == false { + // The type in the stream disagrees with the type that this tag is + // expected to have. This can present issues with how we handle the + // special-case tags (e.g. thumbnails, GPS, etc..) when those tags + // suddenly have data that we no longer manipulate correctly/ + // accurately. + ifdEnumerateLogger.Warningf(nil, + "Tag (0x%04x) in IFD [%s] at position (%d) has unsupported type (0x%02x) and will be skipped.", + tagId, ii, tagPosition, int(tagType)) + + return nil, ErrTagTypeNotValid + } + + // Construct tag struct. + + rs, err := ie.ebs.GetReadSeeker(0) + log.PanicIf(err) + + ite = newIfdTagEntry( + ii, + tagId, + tagPosition, + tagType, + unitCount, + valueOffset, + rawValueOffset, + rs, + ie.byteOrder) + + ifdPath := ii.UnindexedString() + + // If it's an IFD but not a standard one, it'll just be seen as a LONG + // (the standard IFD tag type), later, unless we skip it because it's + // [likely] not even in the standard list of known tags. + mi, err := ie.ifdMapping.GetChild(ifdPath, tagId) + if err == nil { + currentIfdTag := ii.IfdTag() + + childIt := exifcommon.NewIfdTag(¤tIfdTag, tagId, mi.Name) + iiChild := ii.NewChild(childIt, 0) + ite.SetChildIfd(iiChild) + + // We also need to set `tag.ChildFqIfdPath` but can't do it here + // because we don't have the IFD index. + } else if log.Is(err, exifcommon.ErrChildIfdNotMapped) == false { + log.Panic(err) + } + + return ite, nil +} + +// TagVisitorFn is called for each tag when enumerating through the EXIF. +type TagVisitorFn func(ite *IfdTagEntry) (err error) + +// tagPostParse do some tag-level processing here following the parse of each. +func (ie *IfdEnumerate) tagPostParse(ite *IfdTagEntry, med *MiscellaneousExifData) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + ii := ite.IfdIdentity() + + tagId := ite.TagId() + tagType := ite.TagType() + + it, err := ie.tagIndex.Get(ii, tagId) + if err == nil { + ite.setTagName(it.Name) + } else { + if err != ErrTagNotFound { + log.Panic(err) + } + + // This is an unknown tag. + + originalBt := exifcommon.BasicTag{ + FqIfdPath: ii.String(), + IfdPath: ii.UnindexedString(), + TagId: tagId, + } + + if med != nil { + med.unknownTags[originalBt] = exifcommon.BasicTag{} + } + + utilityLogger.Debugf(nil, + "Tag (0x%04x) is not valid for IFD [%s]. Attempting secondary "+ + "lookup.", tagId, ii.String()) + + // This will overwrite the existing `it` and `err`. Since `FindFirst()` + // might generate different Errors than `Get()`, the log message above + // is import to try and mitigate confusion in that case. + it, err = ie.tagIndex.FindFirst(tagId, tagType, nil) + if err != nil { + if err != ErrTagNotFound { + log.Panic(err) + } + + // This is supposed to be a convenience function and if we were + // to keep the name empty or set it to some placeholder, it + // might be mismanaged by the package that is calling us. If + // they want to specifically manage these types of tags, they + // can use more advanced functionality to specifically -handle + // unknown tags. + utilityLogger.Warningf(nil, + "Tag with ID (0x%04x) in IFD [%s] is not recognized and "+ + "will be ignored.", tagId, ii.String()) + + return ErrTagNotFound + } + + ite.setTagName(it.Name) + + utilityLogger.Warningf(nil, + "Tag with ID (0x%04x) is not valid for IFD [%s], but it *is* "+ + "valid as tag [%s] under IFD [%s] and has the same type "+ + "[%s], so we will use that. This EXIF blob was probably "+ + "written by a buggy implementation.", + tagId, ii.UnindexedString(), it.Name, it.IfdPath, + tagType) + + if med != nil { + med.unknownTags[originalBt] = exifcommon.BasicTag{ + IfdPath: it.IfdPath, + TagId: tagId, + } + } + } + + // This is a known tag (from the standard, unless the user did + // something different). + + // Skip any tags that have a type that doesn't match the type in the + // index (which is loaded with the standard and accept tag + // information unless configured otherwise). + // + // We've run into multiple instances of the same tag, where a) no + // tag should ever be repeated, and b) all but one had an incorrect + // type and caused parsing/conversion woes. So, this is a quick fix + // for those scenarios. + if ie.tagIndex.UniversalSearch() == false && it.DoesSupportType(tagType) == false { + ifdEnumerateLogger.Warningf(nil, + "Skipping tag [%s] (0x%04x) [%s] with an unexpected type: %v ∉ %v", + ii.UnindexedString(), tagId, it.Name, + tagType, it.SupportedTypes) + + return ErrTagNotFound + } + + return nil +} + +// parseIfd decodes the IFD block that we're currently sitting on the first +// byte of. +func (ie *IfdEnumerate) parseIfd(ii *exifcommon.IfdIdentity, bp *byteParser, visitor TagVisitorFn, doDescend bool, med *MiscellaneousExifData) (nextIfdOffset uint32, entries []*IfdTagEntry, thumbnailData []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + tagCount, _, err := bp.getUint16() + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "IFD [%s] tag-count: (%d)", ii.String(), tagCount) + + entries = make([]*IfdTagEntry, 0) + + var enumeratorThumbnailOffset *IfdTagEntry + var enumeratorThumbnailSize *IfdTagEntry + + for i := 0; i < int(tagCount); i++ { + ite, err := ie.parseTag(ii, i, bp) + if err != nil { + if log.Is(err, ErrTagNotFound) == true || log.Is(err, ErrTagTypeNotValid) == true { + // These tags should've been fully logged in parseTag(). The + // ITE returned is nil so we can't print anything about them, now. + continue + } + + log.Panic(err) + } + + err = ie.tagPostParse(ite, med) + if err == nil { + if err == ErrTagNotFound { + continue + } + + log.PanicIf(err) + } + + tagId := ite.TagId() + + if visitor != nil { + err := visitor(ite) + log.PanicIf(err) + } + + if ite.IsThumbnailOffset() == true { + ifdEnumerateLogger.Debugf(nil, "Skipping the thumbnail offset tag (0x%04x). Use accessors to get it or set it.", tagId) + + enumeratorThumbnailOffset = ite + entries = append(entries, ite) + + continue + } else if ite.IsThumbnailSize() == true { + ifdEnumerateLogger.Debugf(nil, "Skipping the thumbnail size tag (0x%04x). Use accessors to get it or set it.", tagId) + + enumeratorThumbnailSize = ite + entries = append(entries, ite) + + continue + } + + if ite.TagType() != exifcommon.TypeUndefined { + // If this tag's value is an offset, bump our max-offset value to + // what that offset is plus however large that value is. + + vc := ite.getValueContext() + + farOffset, err := vc.GetFarOffset() + if err == nil { + candidateOffset := farOffset + uint32(vc.SizeInBytes()) + if candidateOffset > ie.furthestOffset { + ie.furthestOffset = candidateOffset + } + } else if err != exifcommon.ErrNotFarValue { + log.PanicIf(err) + } + } + + // If it's an IFD but not a standard one, it'll just be seen as a LONG + // (the standard IFD tag type), later, unless we skip it because it's + // [likely] not even in the standard list of known tags. + if ite.ChildIfdPath() != "" { + if doDescend == true { + ifdEnumerateLogger.Debugf(nil, "Descending from IFD [%s] to IFD [%s].", ii, ite.ChildIfdPath()) + + currentIfdTag := ii.IfdTag() + + childIfdTag := + exifcommon.NewIfdTag( + ¤tIfdTag, + ite.TagId(), + ite.ChildIfdName()) + + iiChild := ii.NewChild(childIfdTag, 0) + + err := ie.scan(iiChild, ite.getValueOffset(), visitor, med) + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Ascending from IFD [%s] to IFD [%s].", ite.ChildIfdPath(), ii) + } + } + + entries = append(entries, ite) + } + + if enumeratorThumbnailOffset != nil && enumeratorThumbnailSize != nil { + thumbnailData, err = ie.parseThumbnail(enumeratorThumbnailOffset, enumeratorThumbnailSize) + if err != nil { + ifdEnumerateLogger.Errorf( + nil, err, + "We tried to bump our furthest-offset counter but there was an issue first seeking past the thumbnail.") + } else { + // In this case, the value is always an offset. + offset := enumeratorThumbnailOffset.getValueOffset() + + // This this case, the value is always a length. + length := enumeratorThumbnailSize.getValueOffset() + + ifdEnumerateLogger.Debugf(nil, "Found thumbnail in IFD [%s]. Its offset is (%d) and is (%d) bytes.", ii, offset, length) + + furthestOffset := offset + length + + if furthestOffset > ie.furthestOffset { + ie.furthestOffset = furthestOffset + } + } + } + + nextIfdOffset, _, err = bp.getUint32() + log.PanicIf(err) + + _, alreadyVisited := ie.visitedIfdOffsets[nextIfdOffset] + + if alreadyVisited == true { + ifdEnumerateLogger.Warningf(nil, "IFD at offset (0x%08x) has been linked-to more than once. There might be a cycle in the IFD chain. Not reparsing.", nextIfdOffset) + nextIfdOffset = 0 + } + + if nextIfdOffset != 0 { + ie.visitedIfdOffsets[nextIfdOffset] = struct{}{} + ifdEnumerateLogger.Debugf(nil, "[%s] Next IFD at offset: (0x%08x)", ii.String(), nextIfdOffset) + } else { + ifdEnumerateLogger.Debugf(nil, "[%s] IFD chain has terminated.", ii.String()) + } + + return nextIfdOffset, entries, thumbnailData, nil +} + +func (ie *IfdEnumerate) parseThumbnail(offsetIte, lengthIte *IfdTagEntry) (thumbnailData []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + vRaw, err := lengthIte.Value() + log.PanicIf(err) + + vList := vRaw.([]uint32) + if len(vList) != 1 { + log.Panicf("not exactly one long: (%d)", len(vList)) + } + + length := vList[0] + + // The tag is official a LONG type, but it's actually an offset to a blob of bytes. + offsetIte.updateTagType(exifcommon.TypeByte) + offsetIte.updateUnitCount(length) + + thumbnailData, err = offsetIte.GetRawBytes() + log.PanicIf(err) + + return thumbnailData, nil +} + +// scan parses and enumerates the different IFD blocks and invokes a visitor +// callback for each tag. No information is kept or returned. +func (ie *IfdEnumerate) scan(iiGeneral *exifcommon.IfdIdentity, ifdOffset uint32, visitor TagVisitorFn, med *MiscellaneousExifData) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + for ifdIndex := 0; ; ifdIndex++ { + iiSibling := iiGeneral.NewSibling(ifdIndex) + + ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] at offset (0x%04x) (scan).", iiSibling.String(), ifdOffset) + + bp, err := ie.getByteParser(ifdOffset) + if err != nil { + if err == ErrOffsetInvalid { + ifdEnumerateLogger.Errorf(nil, nil, "IFD [%s] at offset (0x%04x) is unreachable. Terminating scan.", iiSibling.String(), ifdOffset) + break + } + + log.Panic(err) + } + + nextIfdOffset, _, _, err := ie.parseIfd(iiSibling, bp, visitor, true, med) + log.PanicIf(err) + + currentOffset := bp.CurrentOffset() + if currentOffset > ie.furthestOffset { + ie.furthestOffset = currentOffset + } + + if nextIfdOffset == 0 { + break + } + + ifdOffset = nextIfdOffset + } + + return nil +} + +// MiscellaneousExifData is reports additional data collected during the parse. +type MiscellaneousExifData struct { + // UnknownTags contains all tags that were invalid for their containing + // IFDs. The values represent alternative IFDs that were correctly matched + // to those tags and used instead. + unknownTags map[exifcommon.BasicTag]exifcommon.BasicTag +} + +// UnknownTags returns the unknown tags encountered during the scan. +func (med *MiscellaneousExifData) UnknownTags() map[exifcommon.BasicTag]exifcommon.BasicTag { + return med.unknownTags +} + +// ScanOptions tweaks parser behavior/choices. +type ScanOptions struct { + // NOTE(dustin): Reserved for future usage. +} + +// Scan enumerates the different EXIF blocks (called IFDs). `rootIfdName` will +// be "IFD" in the TIFF standard. +func (ie *IfdEnumerate) Scan(iiRoot *exifcommon.IfdIdentity, ifdOffset uint32, visitor TagVisitorFn, so *ScanOptions) (med *MiscellaneousExifData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + med = &MiscellaneousExifData{ + unknownTags: make(map[exifcommon.BasicTag]exifcommon.BasicTag), + } + + err = ie.scan(iiRoot, ifdOffset, visitor, med) + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Scan: It looks like the furthest offset that contained EXIF data in the EXIF blob was (%d) (Scan).", ie.FurthestOffset()) + + return med, nil +} + +// Ifd represents a single, parsed IFD. +type Ifd struct { + ifdIdentity *exifcommon.IfdIdentity + + ifdMapping *exifcommon.IfdMapping + tagIndex *TagIndex + + offset uint32 + byteOrder binary.ByteOrder + id int + + parentIfd *Ifd + + // ParentTagIndex is our tag position in the parent IFD, if we had a parent + // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling + // instead of as a child). + parentTagIndex int + + entries []*IfdTagEntry + entriesByTagId map[uint16][]*IfdTagEntry + + children []*Ifd + childIfdIndex map[string]*Ifd + + thumbnailData []byte + + nextIfdOffset uint32 + nextIfd *Ifd +} + +// IfdIdentity returns IFD identity that this struct represents. +func (ifd *Ifd) IfdIdentity() *exifcommon.IfdIdentity { + return ifd.ifdIdentity +} + +// Entries returns a flat list of all tags for this IFD. +func (ifd *Ifd) Entries() []*IfdTagEntry { + + // TODO(dustin): Add test + + return ifd.entries +} + +// EntriesByTagId returns a map of all tags for this IFD. +func (ifd *Ifd) EntriesByTagId() map[uint16][]*IfdTagEntry { + + // TODO(dustin): Add test + + return ifd.entriesByTagId +} + +// Children returns a flat list of all child IFDs of this IFD. +func (ifd *Ifd) Children() []*Ifd { + + // TODO(dustin): Add test + + return ifd.children +} + +// ChildWithIfdPath returns a map of all child IFDs of this IFD. +func (ifd *Ifd) ChildIfdIndex() map[string]*Ifd { + + // TODO(dustin): Add test + + return ifd.childIfdIndex +} + +// ParentTagIndex returns the position of this IFD's tag in its parent IFD (*if* +// there is a parent). +func (ifd *Ifd) ParentTagIndex() int { + + // TODO(dustin): Add test + + return ifd.parentTagIndex +} + +// Offset returns the offset of the IFD in the stream. +func (ifd *Ifd) Offset() uint32 { + + // TODO(dustin): Add test + + return ifd.offset +} + +// Offset returns the offset of the IFD in the stream. +func (ifd *Ifd) ByteOrder() binary.ByteOrder { + + // TODO(dustin): Add test + + return ifd.byteOrder +} + +// NextIfd returns the Ifd struct for the next IFD in the chain. +func (ifd *Ifd) NextIfd() *Ifd { + + // TODO(dustin): Add test + + return ifd.nextIfd +} + +// ChildWithIfdPath returns an `Ifd` struct for the given child of the current +// IFD. +func (ifd *Ifd) ChildWithIfdPath(iiChild *exifcommon.IfdIdentity) (childIfd *Ifd, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): This is a bridge while we're introducing the IFD type-system. We should be able to use the (IfdIdentity).Equals() method for this. + ifdPath := iiChild.UnindexedString() + + for _, childIfd := range ifd.children { + if childIfd.ifdIdentity.UnindexedString() == ifdPath { + return childIfd, nil + } + } + + log.Panic(ErrTagNotFound) + return nil, nil +} + +// FindTagWithId returns a list of tags (usually just zero or one) that match +// the given tag ID. This is efficient. +func (ifd *Ifd) FindTagWithId(tagId uint16) (results []*IfdTagEntry, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + results, found := ifd.entriesByTagId[tagId] + if found != true { + log.Panic(ErrTagNotFound) + } + + return results, nil +} + +// FindTagWithName returns a list of tags (usually just zero or one) that match +// the given tag name. This is not efficient (though the labor is trivial). +func (ifd *Ifd) FindTagWithName(tagName string) (results []*IfdTagEntry, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + it, err := ifd.tagIndex.GetWithName(ifd.ifdIdentity, tagName) + if log.Is(err, ErrTagNotFound) == true { + log.Panic(ErrTagNotKnown) + } else if err != nil { + log.Panic(err) + } + + results = make([]*IfdTagEntry, 0) + for _, ite := range ifd.entries { + if ite.TagId() == it.Id { + results = append(results, ite) + } + } + + if len(results) == 0 { + log.Panic(ErrTagNotFound) + } + + return results, nil +} + +// String returns a description string. +func (ifd *Ifd) String() string { + parentOffset := uint32(0) + if ifd.parentIfd != nil { + parentOffset = ifd.parentIfd.offset + } + + return fmt.Sprintf("Ifd", ifd.id, ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index(), len(ifd.entries), ifd.offset, len(ifd.children), parentOffset, ifd.nextIfdOffset) +} + +// Thumbnail returns the raw thumbnail bytes. This is typically directly +// readable by any standard image viewer. +func (ifd *Ifd) Thumbnail() (data []byte, err error) { + + if ifd.thumbnailData == nil { + return nil, ErrNoThumbnail + } + + return ifd.thumbnailData, nil +} + +// dumpTags recursively builds a list of tags from an IFD. +func (ifd *Ifd) dumpTags(tags []*IfdTagEntry) []*IfdTagEntry { + if tags == nil { + tags = make([]*IfdTagEntry, 0) + } + + // Now, print the tags while also descending to child-IFDS as we encounter them. + + ifdsFoundCount := 0 + + for _, ite := range ifd.entries { + tags = append(tags, ite) + + childIfdPath := ite.ChildIfdPath() + if childIfdPath != "" { + ifdsFoundCount++ + + childIfd, found := ifd.childIfdIndex[childIfdPath] + if found != true { + log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) + } + + tags = childIfd.dumpTags(tags) + } + } + + if len(ifd.children) != ifdsFoundCount { + log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.children), ifdsFoundCount) + } + + if ifd.nextIfd != nil { + tags = ifd.nextIfd.dumpTags(tags) + } + + return tags +} + +// DumpTags prints the IFD hierarchy. +func (ifd *Ifd) DumpTags() []*IfdTagEntry { + return ifd.dumpTags(nil) +} + +func (ifd *Ifd) printTagTree(populateValues bool, index, level int, nextLink bool) { + indent := strings.Repeat(" ", level*2) + + prefix := " " + if nextLink { + prefix = ">" + } + + fmt.Printf("%s%sIFD: %s\n", indent, prefix, ifd) + + // Now, print the tags while also descending to child-IFDS as we encounter them. + + ifdsFoundCount := 0 + + for _, ite := range ifd.entries { + if ite.ChildIfdPath() != "" { + fmt.Printf("%s - TAG: %s\n", indent, ite) + } else { + // This will just add noise to the output (byte-tags are fully + // dumped). + if ite.IsThumbnailOffset() == true || ite.IsThumbnailSize() == true { + continue + } + + it, err := ifd.tagIndex.Get(ifd.ifdIdentity, ite.TagId()) + + tagName := "" + if err == nil { + tagName = it.Name + } + + var valuePhrase string + if populateValues == true { + var err error + + valuePhrase, err = ite.Format() + if err != nil { + if log.Is(err, exifcommon.ErrUnhandledUndefinedTypedTag) == true { + ifdEnumerateLogger.Warningf(nil, "Skipping non-standard undefined tag: [%s] (%04x)", ifd.ifdIdentity.UnindexedString(), ite.TagId()) + continue + } else if err == exifundefined.ErrUnparseableValue { + ifdEnumerateLogger.Warningf(nil, "Skipping unparseable undefined tag: [%s] (%04x) [%s]", ifd.ifdIdentity.UnindexedString(), ite.TagId(), it.Name) + continue + } + + log.Panic(err) + } + } else { + valuePhrase = "!UNRESOLVED" + } + + fmt.Printf("%s - TAG: %s NAME=[%s] VALUE=[%v]\n", indent, ite, tagName, valuePhrase) + } + + childIfdPath := ite.ChildIfdPath() + if childIfdPath != "" { + ifdsFoundCount++ + + childIfd, found := ifd.childIfdIndex[childIfdPath] + if found != true { + log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) + } + + childIfd.printTagTree(populateValues, 0, level+1, false) + } + } + + if len(ifd.children) != ifdsFoundCount { + log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.children), ifdsFoundCount) + } + + if ifd.nextIfd != nil { + ifd.nextIfd.printTagTree(populateValues, index+1, level, true) + } +} + +// PrintTagTree prints the IFD hierarchy. +func (ifd *Ifd) PrintTagTree(populateValues bool) { + ifd.printTagTree(populateValues, 0, 0, false) +} + +func (ifd *Ifd) printIfdTree(level int, nextLink bool) { + indent := strings.Repeat(" ", level*2) + + prefix := " " + if nextLink { + prefix = ">" + } + + fmt.Printf("%s%s%s\n", indent, prefix, ifd) + + // Now, print the tags while also descending to child-IFDS as we encounter them. + + ifdsFoundCount := 0 + + for _, ite := range ifd.entries { + childIfdPath := ite.ChildIfdPath() + if childIfdPath != "" { + ifdsFoundCount++ + + childIfd, found := ifd.childIfdIndex[childIfdPath] + if found != true { + log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) + } + + childIfd.printIfdTree(level+1, false) + } + } + + if len(ifd.children) != ifdsFoundCount { + log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.children), ifdsFoundCount) + } + + if ifd.nextIfd != nil { + ifd.nextIfd.printIfdTree(level, true) + } +} + +// PrintIfdTree prints the IFD hierarchy. +func (ifd *Ifd) PrintIfdTree() { + ifd.printIfdTree(0, false) +} + +func (ifd *Ifd) dumpTree(tagsDump []string, level int) []string { + if tagsDump == nil { + tagsDump = make([]string, 0) + } + + indent := strings.Repeat(" ", level*2) + + var ifdPhrase string + if ifd.parentIfd != nil { + ifdPhrase = fmt.Sprintf("[%s]->[%s]:(%d)", ifd.parentIfd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index()) + } else { + ifdPhrase = fmt.Sprintf("[ROOT]->[%s]:(%d)", ifd.ifdIdentity.UnindexedString(), ifd.ifdIdentity.Index()) + } + + startBlurb := fmt.Sprintf("%s> IFD %s TOP", indent, ifdPhrase) + tagsDump = append(tagsDump, startBlurb) + + ifdsFoundCount := 0 + for _, ite := range ifd.entries { + tagsDump = append(tagsDump, fmt.Sprintf("%s - (0x%04x)", indent, ite.TagId())) + + childIfdPath := ite.ChildIfdPath() + if childIfdPath != "" { + ifdsFoundCount++ + + childIfd, found := ifd.childIfdIndex[childIfdPath] + if found != true { + log.Panicf("alien child IFD referenced by a tag: [%s]", childIfdPath) + } + + tagsDump = childIfd.dumpTree(tagsDump, level+1) + } + } + + if len(ifd.children) != ifdsFoundCount { + log.Panicf("have one or more dangling child IFDs: (%d) != (%d)", len(ifd.children), ifdsFoundCount) + } + + finishBlurb := fmt.Sprintf("%s< IFD %s BOTTOM", indent, ifdPhrase) + tagsDump = append(tagsDump, finishBlurb) + + if ifd.nextIfd != nil { + siblingBlurb := fmt.Sprintf("%s* LINKING TO SIBLING IFD [%s]:(%d)", indent, ifd.nextIfd.ifdIdentity.UnindexedString(), ifd.nextIfd.ifdIdentity.Index()) + tagsDump = append(tagsDump, siblingBlurb) + + tagsDump = ifd.nextIfd.dumpTree(tagsDump, level) + } + + return tagsDump +} + +// DumpTree returns a list of strings describing the IFD hierarchy. +func (ifd *Ifd) DumpTree() []string { + return ifd.dumpTree(nil, 0) +} + +// GpsInfo parses and consolidates the GPS info. This can only be called on the +// GPS IFD. +func (ifd *Ifd) GpsInfo() (gi *GpsInfo, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + gi = new(GpsInfo) + + if ifd.ifdIdentity.Equals(exifcommon.IfdGpsInfoStandardIfdIdentity) == false { + log.Panicf("GPS can only be read on GPS IFD: [%s]", ifd.ifdIdentity.UnindexedString()) + } + + if tags, found := ifd.entriesByTagId[TagGpsVersionId]; found == false { + // We've seen this. We'll just have to default to assuming we're in a + // 2.2.0.0 format. + ifdEnumerateLogger.Warningf(nil, "No GPS version tag (0x%04x) found.", TagGpsVersionId) + } else { + versionBytes, err := tags[0].GetRawBytes() + log.PanicIf(err) + + hit := false + for _, acceptedGpsVersion := range ValidGpsVersions { + if bytes.Compare(versionBytes, acceptedGpsVersion[:]) == 0 { + hit = true + break + } + } + + if hit != true { + ifdEnumerateLogger.Warningf(nil, "GPS version not supported: %v", versionBytes) + log.Panic(ErrNoGpsTags) + } + } + + tags, found := ifd.entriesByTagId[TagLatitudeId] + if found == false { + ifdEnumerateLogger.Warningf(nil, "latitude not found") + log.Panic(ErrNoGpsTags) + } + + latitudeValue, err := tags[0].Value() + log.PanicIf(err) + + // Look for whether North or South. + tags, found = ifd.entriesByTagId[TagLatitudeRefId] + if found == false { + ifdEnumerateLogger.Warningf(nil, "latitude-ref not found") + log.Panic(ErrNoGpsTags) + } + + latitudeRefValue, err := tags[0].Value() + log.PanicIf(err) + + tags, found = ifd.entriesByTagId[TagLongitudeId] + if found == false { + ifdEnumerateLogger.Warningf(nil, "longitude not found") + log.Panic(ErrNoGpsTags) + } + + longitudeValue, err := tags[0].Value() + log.PanicIf(err) + + // Look for whether West or East. + tags, found = ifd.entriesByTagId[TagLongitudeRefId] + if found == false { + ifdEnumerateLogger.Warningf(nil, "longitude-ref not found") + log.Panic(ErrNoGpsTags) + } + + longitudeRefValue, err := tags[0].Value() + log.PanicIf(err) + + // Parse location. + + latitudeRaw := latitudeValue.([]exifcommon.Rational) + + gi.Latitude, err = NewGpsDegreesFromRationals(latitudeRefValue.(string), latitudeRaw) + log.PanicIf(err) + + longitudeRaw := longitudeValue.([]exifcommon.Rational) + + gi.Longitude, err = NewGpsDegreesFromRationals(longitudeRefValue.(string), longitudeRaw) + log.PanicIf(err) + + // Parse altitude. + + altitudeTags, foundAltitude := ifd.entriesByTagId[TagAltitudeId] + altitudeRefTags, foundAltitudeRef := ifd.entriesByTagId[TagAltitudeRefId] + + if foundAltitude == true && foundAltitudeRef == true { + altitudePhrase, err := altitudeTags[0].Format() + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Altitude is [%s].", altitudePhrase) + + altitudeValue, err := altitudeTags[0].Value() + log.PanicIf(err) + + altitudeRefPhrase, err := altitudeRefTags[0].Format() + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Altitude-reference is [%s].", altitudeRefPhrase) + + altitudeRefValue, err := altitudeRefTags[0].Value() + log.PanicIf(err) + + altitudeRaw := altitudeValue.([]exifcommon.Rational) + if altitudeRaw[0].Denominator > 0 { + altitude := int(altitudeRaw[0].Numerator / altitudeRaw[0].Denominator) + + if altitudeRefValue.([]byte)[0] == 1 { + altitude *= -1 + } + + gi.Altitude = altitude + } + } + + // Parse timestamp from separate date and time tags. + + timestampTags, foundTimestamp := ifd.entriesByTagId[TagTimestampId] + datestampTags, foundDatestamp := ifd.entriesByTagId[TagDatestampId] + + if foundTimestamp == true && foundDatestamp == true { + datestampValue, err := datestampTags[0].Value() + log.PanicIf(err) + + datePhrase := datestampValue.(string) + ifdEnumerateLogger.Debugf(nil, "Date tag value is [%s].", datePhrase) + + // Normalize the separators. + datePhrase = strings.ReplaceAll(datePhrase, "-", ":") + + dateParts := strings.Split(datePhrase, ":") + + year, err1 := strconv.ParseUint(dateParts[0], 10, 16) + month, err2 := strconv.ParseUint(dateParts[1], 10, 8) + day, err3 := strconv.ParseUint(dateParts[2], 10, 8) + + if err1 == nil && err2 == nil && err3 == nil { + timestampValue, err := timestampTags[0].Value() + log.PanicIf(err) + + timePhrase, err := timestampTags[0].Format() + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Time tag value is [%s].", timePhrase) + + timestampRaw := timestampValue.([]exifcommon.Rational) + + hour := int(timestampRaw[0].Numerator / timestampRaw[0].Denominator) + minute := int(timestampRaw[1].Numerator / timestampRaw[1].Denominator) + second := int(timestampRaw[2].Numerator / timestampRaw[2].Denominator) + + gi.Timestamp = time.Date(int(year), time.Month(month), int(day), hour, minute, second, 0, time.UTC) + } + } + + return gi, nil +} + +// ParsedTagVisitor is a callback used if wanting to visit through all tags and +// child IFDs from the current IFD and going down. +type ParsedTagVisitor func(*Ifd, *IfdTagEntry) error + +// EnumerateTagsRecursively calls the given visitor function for every tag and +// IFD in the current IFD, recursively. +func (ifd *Ifd) EnumerateTagsRecursively(visitor ParsedTagVisitor) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for ptr := ifd; ptr != nil; ptr = ptr.nextIfd { + for _, ite := range ifd.entries { + childIfdPath := ite.ChildIfdPath() + if childIfdPath != "" { + childIfd := ifd.childIfdIndex[childIfdPath] + + err := childIfd.EnumerateTagsRecursively(visitor) + log.PanicIf(err) + } else { + err := visitor(ifd, ite) + log.PanicIf(err) + } + } + } + + return nil +} + +// QueuedIfd is one IFD that has been identified but yet to be processed. +type QueuedIfd struct { + IfdIdentity *exifcommon.IfdIdentity + + Offset uint32 + Parent *Ifd + + // ParentTagIndex is our tag position in the parent IFD, if we had a parent + // (if `ParentIfd` is not nil and we weren't an IFD referenced as a sibling + // instead of as a child). + ParentTagIndex int +} + +// IfdIndex collects a bunch of IFD and tag information stored in several +// different ways in order to provide convenient lookups. +type IfdIndex struct { + RootIfd *Ifd + Ifds []*Ifd + Tree map[int]*Ifd + Lookup map[string]*Ifd +} + +// Collect enumerates the different EXIF blocks (called IFDs) and builds out an +// index struct for referencing all of the parsed data. +func (ie *IfdEnumerate) Collect(rootIfdOffset uint32) (index IfdIndex, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add MiscellaneousExifData to IfdIndex + + tree := make(map[int]*Ifd) + ifds := make([]*Ifd, 0) + lookup := make(map[string]*Ifd) + + queue := []QueuedIfd{ + { + IfdIdentity: exifcommon.IfdStandardIfdIdentity, + Offset: rootIfdOffset, + }, + } + + edges := make(map[uint32]*Ifd) + + for { + if len(queue) == 0 { + break + } + + qi := queue[0] + ii := qi.IfdIdentity + + offset := qi.Offset + parentIfd := qi.Parent + + queue = queue[1:] + + ifdEnumerateLogger.Debugf(nil, "Parsing IFD [%s] (%d) at offset (0x%04x) (Collect).", ii.String(), ii.Index(), offset) + + bp, err := ie.getByteParser(offset) + if err != nil { + if err == ErrOffsetInvalid { + return index, err + } + + log.Panic(err) + } + + // TODO(dustin): We don't need to pass the index in as a separate argument. Get from the II. + + nextIfdOffset, entries, thumbnailData, err := ie.parseIfd(ii, bp, nil, false, nil) + log.PanicIf(err) + + currentOffset := bp.CurrentOffset() + if currentOffset > ie.furthestOffset { + ie.furthestOffset = currentOffset + } + + id := len(ifds) + + entriesByTagId := make(map[uint16][]*IfdTagEntry) + for _, ite := range entries { + tagId := ite.TagId() + + tags, found := entriesByTagId[tagId] + if found == false { + tags = make([]*IfdTagEntry, 0) + } + + entriesByTagId[tagId] = append(tags, ite) + } + + ifd := &Ifd{ + ifdIdentity: ii, + + byteOrder: ie.byteOrder, + + id: id, + + parentIfd: parentIfd, + parentTagIndex: qi.ParentTagIndex, + + offset: offset, + entries: entries, + entriesByTagId: entriesByTagId, + + // This is populated as each child is processed. + children: make([]*Ifd, 0), + + nextIfdOffset: nextIfdOffset, + thumbnailData: thumbnailData, + + ifdMapping: ie.ifdMapping, + tagIndex: ie.tagIndex, + } + + // Add ourselves to a big list of IFDs. + ifds = append(ifds, ifd) + + // Install ourselves into a by-id lookup table (keys are unique). + tree[id] = ifd + + // Install into by-name buckets. + lookup[ii.String()] = ifd + + // Add a link from the previous IFD in the chain to us. + if previousIfd, found := edges[offset]; found == true { + previousIfd.nextIfd = ifd + } + + // Attach as a child to our parent (where we appeared as a tag in + // that IFD). + if parentIfd != nil { + parentIfd.children = append(parentIfd.children, ifd) + } + + // Determine if any of our entries is a child IFD and queue it. + for i, ite := range entries { + if ite.ChildIfdPath() == "" { + continue + } + + tagId := ite.TagId() + childIfdName := ite.ChildIfdName() + + currentIfdTag := ii.IfdTag() + + childIfdTag := + exifcommon.NewIfdTag( + ¤tIfdTag, + tagId, + childIfdName) + + iiChild := ii.NewChild(childIfdTag, 0) + + qi := QueuedIfd{ + IfdIdentity: iiChild, + + Offset: ite.getValueOffset(), + Parent: ifd, + ParentTagIndex: i, + } + + queue = append(queue, qi) + } + + // If there's another IFD in the chain. + if nextIfdOffset != 0 { + iiSibling := ii.NewSibling(ii.Index() + 1) + + // Allow the next link to know what the previous link was. + edges[nextIfdOffset] = ifd + + qi := QueuedIfd{ + IfdIdentity: iiSibling, + Offset: nextIfdOffset, + } + + queue = append(queue, qi) + } + } + + index.RootIfd = tree[0] + index.Ifds = ifds + index.Tree = tree + index.Lookup = lookup + + err = ie.setChildrenIndex(index.RootIfd) + log.PanicIf(err) + + ifdEnumerateLogger.Debugf(nil, "Collect: It looks like the furthest offset that contained EXIF data in the EXIF blob was (%d).", ie.FurthestOffset()) + + return index, nil +} + +func (ie *IfdEnumerate) setChildrenIndex(ifd *Ifd) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + childIfdIndex := make(map[string]*Ifd) + for _, childIfd := range ifd.children { + childIfdIndex[childIfd.ifdIdentity.UnindexedString()] = childIfd + } + + ifd.childIfdIndex = childIfdIndex + + for _, childIfd := range ifd.children { + err := ie.setChildrenIndex(childIfd) + log.PanicIf(err) + } + + return nil +} + +// FurthestOffset returns the furthest offset visited in the EXIF blob. This +// *does not* account for the locations of any undefined tags since we always +// evaluate the furthest offset, whether or not the user wants to know it. +// +// We are not willing to incur the cost of actually parsing those tags just to +// know their length when there are still undefined tags that are out there +// that we still won't have any idea how to parse, thus making this an +// approximation regardless of how clever we get. +func (ie *IfdEnumerate) FurthestOffset() uint32 { + + // TODO(dustin): Add test + + return ie.furthestOffset +} + +// parseOneIfd is a hack to use an IE to parse a raw IFD block. Can be used for +// testing. The fqIfdPath ("fully-qualified IFD path") will be less qualified +// in that the numeric index will always be zero (the zeroth child) rather than +// the proper number (if its actually a sibling to the first child, for +// instance). +func parseOneIfd(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder, ifdBlock []byte, visitor TagVisitorFn) (nextIfdOffset uint32, entries []*IfdTagEntry, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + ebs := NewExifReadSeekerWithBytes(ifdBlock) + + rs, err := ebs.GetReadSeeker(0) + log.PanicIf(err) + + bp, err := newByteParser(rs, byteOrder, 0) + if err != nil { + if err == ErrOffsetInvalid { + return 0, nil, err + } + + log.Panic(err) + } + + dummyEbs := NewExifReadSeekerWithBytes([]byte{}) + ie := NewIfdEnumerate(ifdMapping, tagIndex, dummyEbs, byteOrder) + + nextIfdOffset, entries, _, err = ie.parseIfd(ii, bp, visitor, true, nil) + log.PanicIf(err) + + return nextIfdOffset, entries, nil +} + +// parseOneTag is a hack to use an IE to parse a raw tag block. +func parseOneTag(ifdMapping *exifcommon.IfdMapping, tagIndex *TagIndex, ii *exifcommon.IfdIdentity, byteOrder binary.ByteOrder, tagBlock []byte) (ite *IfdTagEntry, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + ebs := NewExifReadSeekerWithBytes(tagBlock) + + rs, err := ebs.GetReadSeeker(0) + log.PanicIf(err) + + bp, err := newByteParser(rs, byteOrder, 0) + if err != nil { + if err == ErrOffsetInvalid { + return nil, err + } + + log.Panic(err) + } + + dummyEbs := NewExifReadSeekerWithBytes([]byte{}) + ie := NewIfdEnumerate(ifdMapping, tagIndex, dummyEbs, byteOrder) + + ite, err = ie.parseTag(ii, 0, bp) + log.PanicIf(err) + + err = ie.tagPostParse(ite, nil) + if err != nil { + if err == ErrTagNotFound { + return nil, err + } + + log.Panic(err) + } + + return ite, nil +} + +// FindIfdFromRootIfd returns the given `Ifd` given the root-IFD and path of the +// desired IFD. +func FindIfdFromRootIfd(rootIfd *Ifd, ifdPath string) (ifd *Ifd, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): !! Add test. + + lineage, err := rootIfd.ifdMapping.ResolvePath(ifdPath) + log.PanicIf(err) + + // Confirm the first IFD is our root IFD type, and then prune it because + // from then on we'll be searching down through our children. + + if len(lineage) == 0 { + log.Panicf("IFD path must be non-empty.") + } else if lineage[0].Name != exifcommon.IfdStandardIfdIdentity.Name() { + log.Panicf("First IFD path item must be [%s].", exifcommon.IfdStandardIfdIdentity.Name()) + } + + desiredRootIndex := lineage[0].Index + lineage = lineage[1:] + + // TODO(dustin): !! This is a poorly conceived fix that just doubles the work we already have to do below, which then interacts badly with the indices not being properly represented in the IFD-phrase. + // TODO(dustin): !! <-- However, we're not sure whether we shouldn't store a secondary IFD-path with the indices. Some IFDs may not necessarily restrict which IFD indices they can be a child of (only the IFD itself matters). Validation should be delegated to the caller. + thisIfd := rootIfd + for currentRootIndex := 0; currentRootIndex < desiredRootIndex; currentRootIndex++ { + if thisIfd.nextIfd == nil { + log.Panicf("Root-IFD index (%d) does not exist in the data.", currentRootIndex) + } + + thisIfd = thisIfd.nextIfd + } + + for _, itii := range lineage { + var hit *Ifd + for _, childIfd := range thisIfd.children { + if childIfd.ifdIdentity.TagId() == itii.TagId { + hit = childIfd + break + } + } + + // If we didn't find the child, add it. + if hit == nil { + log.Panicf("IFD [%s] in [%s] not found: %s", itii.Name, ifdPath, thisIfd.children) + } + + thisIfd = hit + + // If we didn't find the sibling, add it. + for i := 0; i < itii.Index; i++ { + if thisIfd.nextIfd == nil { + log.Panicf("IFD [%s] does not have (%d) occurrences/siblings", thisIfd.ifdIdentity.UnindexedString(), itii.Index) + } + + thisIfd = thisIfd.nextIfd + } + } + + return thisIfd, nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/ifd_tag_entry.go b/vendor/github.com/dsoprea/go-exif/v3/ifd_tag_entry.go new file mode 100644 index 000000000..ed6ba2291 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/ifd_tag_entry.go @@ -0,0 +1,298 @@ +package exif + +import ( + "fmt" + "io" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-exif/v3/undefined" +) + +var ( + iteLogger = log.NewLogger("exif.ifd_tag_entry") +) + +// IfdTagEntry refers to a tag in the loaded EXIF block. +type IfdTagEntry struct { + tagId uint16 + tagIndex int + tagType exifcommon.TagTypePrimitive + unitCount uint32 + valueOffset uint32 + rawValueOffset []byte + + // childIfdName is the right most atom in the IFD-path. We need this to + // construct the fully-qualified IFD-path. + childIfdName string + + // childIfdPath is the IFD-path of the child if this tag represents a child + // IFD. + childIfdPath string + + // childFqIfdPath is the IFD-path of the child if this tag represents a + // child IFD. Includes indices. + childFqIfdPath string + + // TODO(dustin): !! IB's host the child-IBs directly in the tag, but that's not the case here. Refactor to accommodate it for a consistent experience. + + ifdIdentity *exifcommon.IfdIdentity + + isUnhandledUnknown bool + + rs io.ReadSeeker + byteOrder binary.ByteOrder + + tagName string +} + +func newIfdTagEntry(ii *exifcommon.IfdIdentity, tagId uint16, tagIndex int, tagType exifcommon.TagTypePrimitive, unitCount uint32, valueOffset uint32, rawValueOffset []byte, rs io.ReadSeeker, byteOrder binary.ByteOrder) *IfdTagEntry { + return &IfdTagEntry{ + ifdIdentity: ii, + tagId: tagId, + tagIndex: tagIndex, + tagType: tagType, + unitCount: unitCount, + valueOffset: valueOffset, + rawValueOffset: rawValueOffset, + rs: rs, + byteOrder: byteOrder, + } +} + +// String returns a stringified representation of the struct. +func (ite *IfdTagEntry) String() string { + return fmt.Sprintf("IfdTagEntry", ite.ifdIdentity.String(), ite.tagId, ite.tagType.String(), ite.unitCount) +} + +// TagName returns the name of the tag. This is determined else and set after +// the parse (since it's not actually stored in the stream). If it's empty, it +// is because it is an unknown tag (nonstandard or otherwise unavailable in the +// tag-index). +func (ite *IfdTagEntry) TagName() string { + return ite.tagName +} + +// setTagName sets the tag-name. This provides the name for convenience and +// efficiency by determining it when most efficient while we're parsing rather +// than delegating it to the caller (or, worse, the user). +func (ite *IfdTagEntry) setTagName(tagName string) { + ite.tagName = tagName +} + +// IfdPath returns the fully-qualified path of the IFD that owns this tag. +func (ite *IfdTagEntry) IfdPath() string { + return ite.ifdIdentity.String() +} + +// TagId returns the ID of the tag that we represent. The combination of +// (IfdPath(), TagId()) is unique. +func (ite *IfdTagEntry) TagId() uint16 { + return ite.tagId +} + +// IsThumbnailOffset returns true if the tag has the IFD and tag-ID of a +// thumbnail offset. +func (ite *IfdTagEntry) IsThumbnailOffset() bool { + return ite.tagId == ThumbnailOffsetTagId && ite.ifdIdentity.String() == ThumbnailFqIfdPath +} + +// IsThumbnailSize returns true if the tag has the IFD and tag-ID of a thumbnail +// size. +func (ite *IfdTagEntry) IsThumbnailSize() bool { + return ite.tagId == ThumbnailSizeTagId && ite.ifdIdentity.String() == ThumbnailFqIfdPath +} + +// TagType is the type of value for this tag. +func (ite *IfdTagEntry) TagType() exifcommon.TagTypePrimitive { + return ite.tagType +} + +// updateTagType sets an alternatively interpreted tag-type. +func (ite *IfdTagEntry) updateTagType(tagType exifcommon.TagTypePrimitive) { + ite.tagType = tagType +} + +// UnitCount returns the unit-count of the tag's value. +func (ite *IfdTagEntry) UnitCount() uint32 { + return ite.unitCount +} + +// updateUnitCount sets an alternatively interpreted unit-count. +func (ite *IfdTagEntry) updateUnitCount(unitCount uint32) { + ite.unitCount = unitCount +} + +// getValueOffset is the four-byte offset converted to an integer to point to +// the location of its value in the EXIF block. The "get" parameter is obviously +// used in order to differentiate the naming of the method from the field. +func (ite *IfdTagEntry) getValueOffset() uint32 { + return ite.valueOffset +} + +// GetRawBytes renders a specific list of bytes from the value in this tag. +func (ite *IfdTagEntry) GetRawBytes() (rawBytes []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext := ite.getValueContext() + + if ite.tagType == exifcommon.TypeUndefined { + value, err := exifundefined.Decode(valueContext) + if err != nil { + if err == exifcommon.ErrUnhandledUndefinedTypedTag { + ite.setIsUnhandledUnknown(true) + return nil, exifundefined.ErrUnparseableValue + } else if err == exifundefined.ErrUnparseableValue { + return nil, err + } else { + log.Panic(err) + } + } + + // Encode it back, in order to get the raw bytes. This is the best, + // general way to do it with an undefined tag. + + rawBytes, _, err := exifundefined.Encode(value, ite.byteOrder) + log.PanicIf(err) + + return rawBytes, nil + } + + rawBytes, err = valueContext.ReadRawEncoded() + log.PanicIf(err) + + return rawBytes, nil +} + +// Value returns the specific, parsed, typed value from the tag. +func (ite *IfdTagEntry) Value() (value interface{}, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext := ite.getValueContext() + + if ite.tagType == exifcommon.TypeUndefined { + var err error + + value, err = exifundefined.Decode(valueContext) + if err != nil { + if err == exifcommon.ErrUnhandledUndefinedTypedTag || err == exifundefined.ErrUnparseableValue { + return nil, err + } + + log.Panic(err) + } + } else { + var err error + + value, err = valueContext.Values() + log.PanicIf(err) + } + + return value, nil +} + +// Format returns the tag's value as a string. +func (ite *IfdTagEntry) Format() (phrase string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + value, err := ite.Value() + if err != nil { + if err == exifcommon.ErrUnhandledUndefinedTypedTag { + return exifundefined.UnparseableUnknownTagValuePlaceholder, nil + } else if err == exifundefined.ErrUnparseableValue { + return exifundefined.UnparseableHandledTagValuePlaceholder, nil + } + + log.Panic(err) + } + + phrase, err = exifcommon.FormatFromType(value, false) + log.PanicIf(err) + + return phrase, nil +} + +// FormatFirst returns the same as Format() but only the first item. +func (ite *IfdTagEntry) FormatFirst() (phrase string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): We should add a convenience type "timestamp", to simplify translating to and from the physical ASCII and provide validation. + + value, err := ite.Value() + if err != nil { + if err == exifcommon.ErrUnhandledUndefinedTypedTag { + return exifundefined.UnparseableUnknownTagValuePlaceholder, nil + } + + log.Panic(err) + } + + phrase, err = exifcommon.FormatFromType(value, true) + log.PanicIf(err) + + return phrase, nil +} + +func (ite *IfdTagEntry) setIsUnhandledUnknown(isUnhandledUnknown bool) { + ite.isUnhandledUnknown = isUnhandledUnknown +} + +// SetChildIfd sets child-IFD information (if we represent a child IFD). +func (ite *IfdTagEntry) SetChildIfd(ii *exifcommon.IfdIdentity) { + ite.childFqIfdPath = ii.String() + ite.childIfdPath = ii.UnindexedString() + ite.childIfdName = ii.Name() +} + +// ChildIfdName returns the name of the child IFD +func (ite *IfdTagEntry) ChildIfdName() string { + return ite.childIfdName +} + +// ChildIfdPath returns the path of the child IFD. +func (ite *IfdTagEntry) ChildIfdPath() string { + return ite.childIfdPath +} + +// ChildFqIfdPath returns the complete path of the child IFD along with the +// numeric suffixes differentiating sibling occurrences of the same type. "0" +// indices are omitted. +func (ite *IfdTagEntry) ChildFqIfdPath() string { + return ite.childFqIfdPath +} + +// IfdIdentity returns the IfdIdentity associated with this tag. +func (ite *IfdTagEntry) IfdIdentity() *exifcommon.IfdIdentity { + return ite.ifdIdentity +} + +func (ite *IfdTagEntry) getValueContext() *exifcommon.ValueContext { + return exifcommon.NewValueContext( + ite.ifdIdentity.String(), + ite.tagId, + ite.unitCount, + ite.valueOffset, + ite.rawValueOffset, + ite.rs, + ite.tagType, + ite.byteOrder) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/package.go b/vendor/github.com/dsoprea/go-exif/v3/package.go new file mode 100644 index 000000000..428f74e3a --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/package.go @@ -0,0 +1,8 @@ +// Package exif parses raw EXIF information given a block of raw EXIF data. It +// can also construct new EXIF information, and provides tools for doing so. +// This package is not involved with the parsing of particular file-formats. +// +// The EXIF data must first be extracted and then provided to us. Conversely, +// when constructing new EXIF data, the caller is responsible for packaging +// this in whichever format they require. +package exif diff --git a/vendor/github.com/dsoprea/go-exif/v3/tags.go b/vendor/github.com/dsoprea/go-exif/v3/tags.go new file mode 100644 index 000000000..aca902c5d --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/tags.go @@ -0,0 +1,475 @@ +package exif + +import ( + "fmt" + "sync" + + "github.com/dsoprea/go-logging" + "gopkg.in/yaml.v2" + + "github.com/dsoprea/go-exif/v3/common" +) + +const ( + // IFD1 + + // ThumbnailFqIfdPath is the fully-qualified IFD path that the thumbnail + // must be found in. + ThumbnailFqIfdPath = "IFD1" + + // ThumbnailOffsetTagId returns the tag-ID of the thumbnail offset. + ThumbnailOffsetTagId = 0x0201 + + // ThumbnailSizeTagId returns the tag-ID of the thumbnail size. + ThumbnailSizeTagId = 0x0202 +) + +const ( + // GPS + + // TagGpsVersionId is the ID of the GPS version tag. + TagGpsVersionId = 0x0000 + + // TagLatitudeId is the ID of the GPS latitude tag. + TagLatitudeId = 0x0002 + + // TagLatitudeRefId is the ID of the GPS latitude orientation tag. + TagLatitudeRefId = 0x0001 + + // TagLongitudeId is the ID of the GPS longitude tag. + TagLongitudeId = 0x0004 + + // TagLongitudeRefId is the ID of the GPS longitude-orientation tag. + TagLongitudeRefId = 0x0003 + + // TagTimestampId is the ID of the GPS time tag. + TagTimestampId = 0x0007 + + // TagDatestampId is the ID of the GPS date tag. + TagDatestampId = 0x001d + + // TagAltitudeId is the ID of the GPS altitude tag. + TagAltitudeId = 0x0006 + + // TagAltitudeRefId is the ID of the GPS altitude-orientation tag. + TagAltitudeRefId = 0x0005 +) + +var ( + // tagsWithoutAlignment is a tag-lookup for tags whose value size won't + // necessarily be a multiple of its tag-type. + tagsWithoutAlignment = map[uint16]struct{}{ + // The thumbnail offset is stored as a long, but its data is a binary + // blob (not a slice of longs). + ThumbnailOffsetTagId: {}, + } +) + +var ( + tagsLogger = log.NewLogger("exif.tags") +) + +// File structures. + +type encodedTag struct { + // id is signed, here, because YAML doesn't have enough information to + // support unsigned. + Id int `yaml:"id"` + Name string `yaml:"name"` + TypeName string `yaml:"type_name"` + TypeNames []string `yaml:"type_names"` +} + +// Indexing structures. + +// IndexedTag describes one index lookup result. +type IndexedTag struct { + // Id is the tag-ID. + Id uint16 + + // Name is the tag name. + Name string + + // IfdPath is the proper IFD path of this tag. This is not fully-qualified. + IfdPath string + + // SupportedTypes is an unsorted list of allowed tag-types. + SupportedTypes []exifcommon.TagTypePrimitive +} + +// String returns a descriptive string. +func (it *IndexedTag) String() string { + return fmt.Sprintf("TAG", it.Id, it.Name, it.IfdPath) +} + +// IsName returns true if this tag matches the given tag name. +func (it *IndexedTag) IsName(ifdPath, name string) bool { + return it.Name == name && it.IfdPath == ifdPath +} + +// Is returns true if this tag matched the given tag ID. +func (it *IndexedTag) Is(ifdPath string, id uint16) bool { + return it.Id == id && it.IfdPath == ifdPath +} + +// GetEncodingType returns the largest type that this tag's value can occupy. +func (it *IndexedTag) GetEncodingType(value interface{}) exifcommon.TagTypePrimitive { + // For convenience, we handle encoding a `time.Time` directly. + if exifcommon.IsTime(value) == true { + // Timestamps are encoded as ASCII. + value = "" + } + + if len(it.SupportedTypes) == 0 { + log.Panicf("IndexedTag [%s] (%d) has no supported types.", it.IfdPath, it.Id) + } else if len(it.SupportedTypes) == 1 { + return it.SupportedTypes[0] + } + + supportsLong := false + supportsShort := false + supportsRational := false + supportsSignedRational := false + for _, supportedType := range it.SupportedTypes { + if supportedType == exifcommon.TypeLong { + supportsLong = true + } else if supportedType == exifcommon.TypeShort { + supportsShort = true + } else if supportedType == exifcommon.TypeRational { + supportsRational = true + } else if supportedType == exifcommon.TypeSignedRational { + supportsSignedRational = true + } + } + + // We specifically check for the cases that we know to expect. + + if supportsLong == true && supportsShort == true { + return exifcommon.TypeLong + } else if supportsRational == true && supportsSignedRational == true { + if value == nil { + log.Panicf("GetEncodingType: require value to be given") + } + + if _, ok := value.(exifcommon.SignedRational); ok == true { + return exifcommon.TypeSignedRational + } + + return exifcommon.TypeRational + } + + log.Panicf("WidestSupportedType() case is not handled for tag [%s] (0x%04x): %v", it.IfdPath, it.Id, it.SupportedTypes) + return 0 +} + +// DoesSupportType returns true if this tag can be found/decoded with this type. +func (it *IndexedTag) DoesSupportType(tagType exifcommon.TagTypePrimitive) bool { + // This is always a very small collection. So, we keep it unsorted. + for _, thisTagType := range it.SupportedTypes { + if thisTagType == tagType { + return true + } + } + + return false +} + +// TagIndex is a tag-lookup facility. +type TagIndex struct { + tagsByIfd map[string]map[uint16]*IndexedTag + tagsByIfdR map[string]map[string]*IndexedTag + + mutex sync.Mutex + + doUniversalSearch bool +} + +// NewTagIndex returns a new TagIndex struct. +func NewTagIndex() *TagIndex { + ti := new(TagIndex) + + ti.tagsByIfd = make(map[string]map[uint16]*IndexedTag) + ti.tagsByIfdR = make(map[string]map[string]*IndexedTag) + + return ti +} + +// SetUniversalSearch enables a fallback to matching tags under *any* IFD. +func (ti *TagIndex) SetUniversalSearch(flag bool) { + ti.doUniversalSearch = flag +} + +// UniversalSearch enables a fallback to matching tags under *any* IFD. +func (ti *TagIndex) UniversalSearch() bool { + return ti.doUniversalSearch +} + +// Add registers a new tag to be recognized during the parse. +func (ti *TagIndex) Add(it *IndexedTag) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ti.mutex.Lock() + defer ti.mutex.Unlock() + + // Store by ID. + + family, found := ti.tagsByIfd[it.IfdPath] + if found == false { + family = make(map[uint16]*IndexedTag) + ti.tagsByIfd[it.IfdPath] = family + } + + if _, found := family[it.Id]; found == true { + log.Panicf("tag-ID defined more than once for IFD [%s]: (%02x)", it.IfdPath, it.Id) + } + + family[it.Id] = it + + // Store by name. + + familyR, found := ti.tagsByIfdR[it.IfdPath] + if found == false { + familyR = make(map[string]*IndexedTag) + ti.tagsByIfdR[it.IfdPath] = familyR + } + + if _, found := familyR[it.Name]; found == true { + log.Panicf("tag-name defined more than once for IFD [%s]: (%s)", it.IfdPath, it.Name) + } + + familyR[it.Name] = it + + return nil +} + +func (ti *TagIndex) getOne(ifdPath string, id uint16) (it *IndexedTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if len(ti.tagsByIfd) == 0 { + err := LoadStandardTags(ti) + log.PanicIf(err) + } + + ti.mutex.Lock() + defer ti.mutex.Unlock() + + family, found := ti.tagsByIfd[ifdPath] + if found == false { + return nil, ErrTagNotFound + } + + it, found = family[id] + if found == false { + return nil, ErrTagNotFound + } + + return it, nil +} + +// Get returns information about the non-IFD tag given a tag ID. `ifdPath` must +// not be fully-qualified. +func (ti *TagIndex) Get(ii *exifcommon.IfdIdentity, id uint16) (it *IndexedTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ifdPath := ii.UnindexedString() + + it, err = ti.getOne(ifdPath, id) + if err == nil { + return it, nil + } else if err != ErrTagNotFound { + log.Panic(err) + } + + if ti.doUniversalSearch == false { + return nil, ErrTagNotFound + } + + // We've been told to fallback to look for the tag in other IFDs. + + skipIfdPath := ii.UnindexedString() + + for currentIfdPath, _ := range ti.tagsByIfd { + if currentIfdPath == skipIfdPath { + // Skip the primary IFD, which has already been checked. + continue + } + + it, err = ti.getOne(currentIfdPath, id) + if err == nil { + tagsLogger.Warningf(nil, + "Found tag (0x%02x) in the wrong IFD: [%s] != [%s]", + id, currentIfdPath, ifdPath) + + return it, nil + } else if err != ErrTagNotFound { + log.Panic(err) + } + } + + return nil, ErrTagNotFound +} + +var ( + // tagGuessDefaultIfdIdentities describes which IFDs we'll look for a given + // tag-ID in, if it's not found where it's supposed to be. We suppose that + // Exif-IFD tags might be found in IFD0 or IFD1, or IFD0/IFD1 tags might be + // found in the Exif IFD. This is the only thing we've seen so far. So, this + // is the limit of our guessing. + tagGuessDefaultIfdIdentities = []*exifcommon.IfdIdentity{ + exifcommon.IfdExifStandardIfdIdentity, + exifcommon.IfdStandardIfdIdentity, + } +) + +// FindFirst looks for the given tag-ID in each of the given IFDs in the given +// order. If `fqIfdPaths` is `nil` then use a default search order. This defies +// the standard, which requires each tag to exist in certain IFDs. This is a +// contingency to make recommendations for malformed data. +// +// Things *can* end badly here, in that the same tag-ID in different IFDs might +// describe different data and different ata-types, and our decode might then +// produce binary and non-printable data. +func (ti *TagIndex) FindFirst(id uint16, typeId exifcommon.TagTypePrimitive, ifdIdentities []*exifcommon.IfdIdentity) (it *IndexedTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if ifdIdentities == nil { + ifdIdentities = tagGuessDefaultIfdIdentities + } + + for _, ii := range ifdIdentities { + it, err := ti.Get(ii, id) + if err != nil { + if err == ErrTagNotFound { + continue + } + + log.Panic(err) + } + + // Even though the tag might be mislocated, the type should still be the + // same. Check this so we don't accidentally end-up on a complete + // irrelevant tag with a totally different data type. This attempts to + // mitigate producing garbage. + for _, supportedType := range it.SupportedTypes { + if supportedType == typeId { + return it, nil + } + } + } + + return nil, ErrTagNotFound +} + +// GetWithName returns information about the non-IFD tag given a tag name. +func (ti *TagIndex) GetWithName(ii *exifcommon.IfdIdentity, name string) (it *IndexedTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if len(ti.tagsByIfdR) == 0 { + err := LoadStandardTags(ti) + log.PanicIf(err) + } + + ifdPath := ii.UnindexedString() + + it, found := ti.tagsByIfdR[ifdPath][name] + if found != true { + log.Panic(ErrTagNotFound) + } + + return it, nil +} + +// LoadStandardTags registers the tags that all devices/applications should +// support. +func LoadStandardTags(ti *TagIndex) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Read static data. + + encodedIfds := make(map[string][]encodedTag) + + err = yaml.Unmarshal([]byte(tagsYaml), encodedIfds) + log.PanicIf(err) + + // Load structure. + + count := 0 + for ifdPath, tags := range encodedIfds { + for _, tagInfo := range tags { + tagId := uint16(tagInfo.Id) + tagName := tagInfo.Name + tagTypeName := tagInfo.TypeName + tagTypeNames := tagInfo.TypeNames + + if tagTypeNames == nil { + if tagTypeName == "" { + log.Panicf("no tag-types were given when registering standard tag [%s] (0x%04x) [%s]", ifdPath, tagId, tagName) + } + + tagTypeNames = []string{ + tagTypeName, + } + } else if tagTypeName != "" { + log.Panicf("both 'type_names' and 'type_name' were given when registering standard tag [%s] (0x%04x) [%s]", ifdPath, tagId, tagName) + } + + tagTypes := make([]exifcommon.TagTypePrimitive, 0) + for _, tagTypeName := range tagTypeNames { + + // TODO(dustin): Discard unsupported types. This helps us with non-standard types that have actually been found in real data, that we ignore for right now. e.g. SSHORT, FLOAT, DOUBLE + tagTypeId, found := exifcommon.GetTypeByName(tagTypeName) + if found == false { + tagsLogger.Warningf(nil, "Type [%s] for tag [%s] being loaded is not valid and is being ignored.", tagTypeName, tagName) + continue + } + + tagTypes = append(tagTypes, tagTypeId) + } + + if len(tagTypes) == 0 { + tagsLogger.Warningf(nil, "Tag [%s] (0x%04x) [%s] being loaded does not have any supported types and will not be registered.", ifdPath, tagId, tagName) + continue + } + + it := &IndexedTag{ + IfdPath: ifdPath, + Id: tagId, + Name: tagName, + SupportedTypes: tagTypes, + } + + err = ti.Add(it) + log.PanicIf(err) + + count++ + } + } + + tagsLogger.Debugf(nil, "(%d) tags loaded.", count) + + return nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/tags_data.go b/vendor/github.com/dsoprea/go-exif/v3/tags_data.go new file mode 100644 index 000000000..dcf0cc4f4 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/tags_data.go @@ -0,0 +1,968 @@ +package exif + +var ( + // From assets/tags.yaml . Needs to be here so it's embedded in the binary. + tagsYaml = ` +# Notes: +# +# This file was produced from http://www.exiv2.org/tags.html, using the included +# tool, though that document appears to have some duplicates when all IDs are +# supposed to be unique (EXIF information only has IDs, not IFDs; IFDs are +# determined by our pre-existing knowledge of those tags). +# +# The webpage that we've produced this file from appears to indicate that +# ImageWidth is represented by both 0x0100 and 0x0001 depending on whether the +# encoding is RGB or YCbCr. +IFD/Exif: +- id: 0x829a + name: ExposureTime + type_name: RATIONAL +- id: 0x829d + name: FNumber + type_name: RATIONAL +- id: 0x8822 + name: ExposureProgram + type_name: SHORT +- id: 0x8824 + name: SpectralSensitivity + type_name: ASCII +- id: 0x8827 + name: ISOSpeedRatings + type_name: SHORT +- id: 0x8828 + name: OECF + type_name: UNDEFINED +- id: 0x8830 + name: SensitivityType + type_name: SHORT +- id: 0x8831 + name: StandardOutputSensitivity + type_name: LONG +- id: 0x8832 + name: RecommendedExposureIndex + type_name: LONG +- id: 0x8833 + name: ISOSpeed + type_name: LONG +- id: 0x8834 + name: ISOSpeedLatitudeyyy + type_name: LONG +- id: 0x8835 + name: ISOSpeedLatitudezzz + type_name: LONG +- id: 0x9000 + name: ExifVersion + type_name: UNDEFINED +- id: 0x9003 + name: DateTimeOriginal + type_name: ASCII +- id: 0x9004 + name: DateTimeDigitized + type_name: ASCII +- id: 0x9010 + name: OffsetTime + type_name: ASCII +- id: 0x9011 + name: OffsetTimeOriginal + type_name: ASCII +- id: 0x9012 + name: OffsetTimeDigitized + type_name: ASCII +- id: 0x9101 + name: ComponentsConfiguration + type_name: UNDEFINED +- id: 0x9102 + name: CompressedBitsPerPixel + type_name: RATIONAL +- id: 0x9201 + name: ShutterSpeedValue + type_name: SRATIONAL +- id: 0x9202 + name: ApertureValue + type_name: RATIONAL +- id: 0x9203 + name: BrightnessValue + type_name: SRATIONAL +- id: 0x9204 + name: ExposureBiasValue + type_name: SRATIONAL +- id: 0x9205 + name: MaxApertureValue + type_name: RATIONAL +- id: 0x9206 + name: SubjectDistance + type_name: RATIONAL +- id: 0x9207 + name: MeteringMode + type_name: SHORT +- id: 0x9208 + name: LightSource + type_name: SHORT +- id: 0x9209 + name: Flash + type_name: SHORT +- id: 0x920a + name: FocalLength + type_name: RATIONAL +- id: 0x9214 + name: SubjectArea + type_name: SHORT +- id: 0x927c + name: MakerNote + type_name: UNDEFINED +- id: 0x9286 + name: UserComment + type_name: UNDEFINED +- id: 0x9290 + name: SubSecTime + type_name: ASCII +- id: 0x9291 + name: SubSecTimeOriginal + type_name: ASCII +- id: 0x9292 + name: SubSecTimeDigitized + type_name: ASCII +- id: 0xa000 + name: FlashpixVersion + type_name: UNDEFINED +- id: 0xa001 + name: ColorSpace + type_name: SHORT +- id: 0xa002 + name: PixelXDimension + type_names: [LONG, SHORT] +- id: 0xa003 + name: PixelYDimension + type_names: [LONG, SHORT] +- id: 0xa004 + name: RelatedSoundFile + type_name: ASCII +- id: 0xa005 + name: InteroperabilityTag + type_name: LONG +- id: 0xa20b + name: FlashEnergy + type_name: RATIONAL +- id: 0xa20c + name: SpatialFrequencyResponse + type_name: UNDEFINED +- id: 0xa20e + name: FocalPlaneXResolution + type_name: RATIONAL +- id: 0xa20f + name: FocalPlaneYResolution + type_name: RATIONAL +- id: 0xa210 + name: FocalPlaneResolutionUnit + type_name: SHORT +- id: 0xa214 + name: SubjectLocation + type_name: SHORT +- id: 0xa215 + name: ExposureIndex + type_name: RATIONAL +- id: 0xa217 + name: SensingMethod + type_name: SHORT +- id: 0xa300 + name: FileSource + type_name: UNDEFINED +- id: 0xa301 + name: SceneType + type_name: UNDEFINED +- id: 0xa302 + name: CFAPattern + type_name: UNDEFINED +- id: 0xa401 + name: CustomRendered + type_name: SHORT +- id: 0xa402 + name: ExposureMode + type_name: SHORT +- id: 0xa403 + name: WhiteBalance + type_name: SHORT +- id: 0xa404 + name: DigitalZoomRatio + type_name: RATIONAL +- id: 0xa405 + name: FocalLengthIn35mmFilm + type_name: SHORT +- id: 0xa406 + name: SceneCaptureType + type_name: SHORT +- id: 0xa407 + name: GainControl + type_name: SHORT +- id: 0xa408 + name: Contrast + type_name: SHORT +- id: 0xa409 + name: Saturation + type_name: SHORT +- id: 0xa40a + name: Sharpness + type_name: SHORT +- id: 0xa40b + name: DeviceSettingDescription + type_name: UNDEFINED +- id: 0xa40c + name: SubjectDistanceRange + type_name: SHORT +- id: 0xa420 + name: ImageUniqueID + type_name: ASCII +- id: 0xa430 + name: CameraOwnerName + type_name: ASCII +- id: 0xa431 + name: BodySerialNumber + type_name: ASCII +- id: 0xa432 + name: LensSpecification + type_name: RATIONAL +- id: 0xa433 + name: LensMake + type_name: ASCII +- id: 0xa434 + name: LensModel + type_name: ASCII +- id: 0xa435 + name: LensSerialNumber + type_name: ASCII +IFD/GPSInfo: +- id: 0x0000 + name: GPSVersionID + type_name: BYTE +- id: 0x0001 + name: GPSLatitudeRef + type_name: ASCII +- id: 0x0002 + name: GPSLatitude + type_name: RATIONAL +- id: 0x0003 + name: GPSLongitudeRef + type_name: ASCII +- id: 0x0004 + name: GPSLongitude + type_name: RATIONAL +- id: 0x0005 + name: GPSAltitudeRef + type_name: BYTE +- id: 0x0006 + name: GPSAltitude + type_name: RATIONAL +- id: 0x0007 + name: GPSTimeStamp + type_name: RATIONAL +- id: 0x0008 + name: GPSSatellites + type_name: ASCII +- id: 0x0009 + name: GPSStatus + type_name: ASCII +- id: 0x000a + name: GPSMeasureMode + type_name: ASCII +- id: 0x000b + name: GPSDOP + type_name: RATIONAL +- id: 0x000c + name: GPSSpeedRef + type_name: ASCII +- id: 0x000d + name: GPSSpeed + type_name: RATIONAL +- id: 0x000e + name: GPSTrackRef + type_name: ASCII +- id: 0x000f + name: GPSTrack + type_name: RATIONAL +- id: 0x0010 + name: GPSImgDirectionRef + type_name: ASCII +- id: 0x0011 + name: GPSImgDirection + type_name: RATIONAL +- id: 0x0012 + name: GPSMapDatum + type_name: ASCII +- id: 0x0013 + name: GPSDestLatitudeRef + type_name: ASCII +- id: 0x0014 + name: GPSDestLatitude + type_name: RATIONAL +- id: 0x0015 + name: GPSDestLongitudeRef + type_name: ASCII +- id: 0x0016 + name: GPSDestLongitude + type_name: RATIONAL +- id: 0x0017 + name: GPSDestBearingRef + type_name: ASCII +- id: 0x0018 + name: GPSDestBearing + type_name: RATIONAL +- id: 0x0019 + name: GPSDestDistanceRef + type_name: ASCII +- id: 0x001a + name: GPSDestDistance + type_name: RATIONAL +- id: 0x001b + name: GPSProcessingMethod + type_name: UNDEFINED +- id: 0x001c + name: GPSAreaInformation + type_name: UNDEFINED +- id: 0x001d + name: GPSDateStamp + type_name: ASCII +- id: 0x001e + name: GPSDifferential + type_name: SHORT +IFD: +- id: 0x000b + name: ProcessingSoftware + type_name: ASCII +- id: 0x00fe + name: NewSubfileType + type_name: LONG +- id: 0x00ff + name: SubfileType + type_name: SHORT +- id: 0x0100 + name: ImageWidth + type_names: [LONG, SHORT] +- id: 0x0101 + name: ImageLength + type_names: [LONG, SHORT] +- id: 0x0102 + name: BitsPerSample + type_name: SHORT +- id: 0x0103 + name: Compression + type_name: SHORT +- id: 0x0106 + name: PhotometricInterpretation + type_name: SHORT +- id: 0x0107 + name: Thresholding + type_name: SHORT +- id: 0x0108 + name: CellWidth + type_name: SHORT +- id: 0x0109 + name: CellLength + type_name: SHORT +- id: 0x010a + name: FillOrder + type_name: SHORT +- id: 0x010d + name: DocumentName + type_name: ASCII +- id: 0x010e + name: ImageDescription + type_name: ASCII +- id: 0x010f + name: Make + type_name: ASCII +- id: 0x0110 + name: Model + type_name: ASCII +- id: 0x0111 + name: StripOffsets + type_names: [LONG, SHORT] +- id: 0x0112 + name: Orientation + type_name: SHORT +- id: 0x0115 + name: SamplesPerPixel + type_name: SHORT +- id: 0x0116 + name: RowsPerStrip + type_names: [LONG, SHORT] +- id: 0x0117 + name: StripByteCounts + type_names: [LONG, SHORT] +- id: 0x011a + name: XResolution + type_name: RATIONAL +- id: 0x011b + name: YResolution + type_name: RATIONAL +- id: 0x011c + name: PlanarConfiguration + type_name: SHORT +- id: 0x0122 + name: GrayResponseUnit + type_name: SHORT +- id: 0x0123 + name: GrayResponseCurve + type_name: SHORT +- id: 0x0124 + name: T4Options + type_name: LONG +- id: 0x0125 + name: T6Options + type_name: LONG +- id: 0x0128 + name: ResolutionUnit + type_name: SHORT +- id: 0x0129 + name: PageNumber + type_name: SHORT +- id: 0x012d + name: TransferFunction + type_name: SHORT +- id: 0x0131 + name: Software + type_name: ASCII +- id: 0x0132 + name: DateTime + type_name: ASCII +- id: 0x013b + name: Artist + type_name: ASCII +- id: 0x013c + name: HostComputer + type_name: ASCII +- id: 0x013d + name: Predictor + type_name: SHORT +- id: 0x013e + name: WhitePoint + type_name: RATIONAL +- id: 0x013f + name: PrimaryChromaticities + type_name: RATIONAL +- id: 0x0140 + name: ColorMap + type_name: SHORT +- id: 0x0141 + name: HalftoneHints + type_name: SHORT +- id: 0x0142 + name: TileWidth + type_name: SHORT +- id: 0x0143 + name: TileLength + type_name: SHORT +- id: 0x0144 + name: TileOffsets + type_name: SHORT +- id: 0x0145 + name: TileByteCounts + type_name: SHORT +- id: 0x014a + name: SubIFDs + type_name: LONG +- id: 0x014c + name: InkSet + type_name: SHORT +- id: 0x014d + name: InkNames + type_name: ASCII +- id: 0x014e + name: NumberOfInks + type_name: SHORT +- id: 0x0150 + name: DotRange + type_name: BYTE +- id: 0x0151 + name: TargetPrinter + type_name: ASCII +- id: 0x0152 + name: ExtraSamples + type_name: SHORT +- id: 0x0153 + name: SampleFormat + type_name: SHORT +- id: 0x0154 + name: SMinSampleValue + type_name: SHORT +- id: 0x0155 + name: SMaxSampleValue + type_name: SHORT +- id: 0x0156 + name: TransferRange + type_name: SHORT +- id: 0x0157 + name: ClipPath + type_name: BYTE +- id: 0x015a + name: Indexed + type_name: SHORT +- id: 0x015b + name: JPEGTables + type_name: UNDEFINED +- id: 0x015f + name: OPIProxy + type_name: SHORT +- id: 0x0200 + name: JPEGProc + type_name: LONG +- id: 0x0201 + name: JPEGInterchangeFormat + type_name: LONG +- id: 0x0202 + name: JPEGInterchangeFormatLength + type_name: LONG +- id: 0x0203 + name: JPEGRestartInterval + type_name: SHORT +- id: 0x0205 + name: JPEGLosslessPredictors + type_name: SHORT +- id: 0x0206 + name: JPEGPointTransforms + type_name: SHORT +- id: 0x0207 + name: JPEGQTables + type_name: LONG +- id: 0x0208 + name: JPEGDCTables + type_name: LONG +- id: 0x0209 + name: JPEGACTables + type_name: LONG +- id: 0x0211 + name: YCbCrCoefficients + type_name: RATIONAL +- id: 0x0212 + name: YCbCrSubSampling + type_name: SHORT +- id: 0x0213 + name: YCbCrPositioning + type_name: SHORT +- id: 0x0214 + name: ReferenceBlackWhite + type_name: RATIONAL +- id: 0x02bc + name: XMLPacket + type_name: BYTE +- id: 0x4746 + name: Rating + type_name: SHORT +- id: 0x4749 + name: RatingPercent + type_name: SHORT +- id: 0x800d + name: ImageID + type_name: ASCII +- id: 0x828d + name: CFARepeatPatternDim + type_name: SHORT +- id: 0x828e + name: CFAPattern + type_name: BYTE +- id: 0x828f + name: BatteryLevel + type_name: RATIONAL +- id: 0x8298 + name: Copyright + type_name: ASCII +- id: 0x829a + name: ExposureTime +# NOTE(dustin): SRATIONAL isn't mentioned in the standard, but we have seen it in real data. + type_names: [RATIONAL, SRATIONAL] +- id: 0x829d + name: FNumber +# NOTE(dustin): SRATIONAL isn't mentioned in the standard, but we have seen it in real data. + type_names: [RATIONAL, SRATIONAL] +- id: 0x83bb + name: IPTCNAA + type_name: LONG +- id: 0x8649 + name: ImageResources + type_name: BYTE +- id: 0x8769 + name: ExifTag + type_name: LONG +- id: 0x8773 + name: InterColorProfile + type_name: UNDEFINED +- id: 0x8822 + name: ExposureProgram + type_name: SHORT +- id: 0x8824 + name: SpectralSensitivity + type_name: ASCII +- id: 0x8825 + name: GPSTag + type_name: LONG +- id: 0x8827 + name: ISOSpeedRatings + type_name: SHORT +- id: 0x8828 + name: OECF + type_name: UNDEFINED +- id: 0x8829 + name: Interlace + type_name: SHORT +- id: 0x882b + name: SelfTimerMode + type_name: SHORT +- id: 0x9003 + name: DateTimeOriginal + type_name: ASCII +- id: 0x9102 + name: CompressedBitsPerPixel + type_name: RATIONAL +- id: 0x9201 + name: ShutterSpeedValue + type_name: SRATIONAL +- id: 0x9202 + name: ApertureValue + type_name: RATIONAL +- id: 0x9203 + name: BrightnessValue + type_name: SRATIONAL +- id: 0x9204 + name: ExposureBiasValue + type_name: SRATIONAL +- id: 0x9205 + name: MaxApertureValue + type_name: RATIONAL +- id: 0x9206 + name: SubjectDistance + type_name: SRATIONAL +- id: 0x9207 + name: MeteringMode + type_name: SHORT +- id: 0x9208 + name: LightSource + type_name: SHORT +- id: 0x9209 + name: Flash + type_name: SHORT +- id: 0x920a + name: FocalLength + type_name: RATIONAL +- id: 0x920b + name: FlashEnergy + type_name: RATIONAL +- id: 0x920c + name: SpatialFrequencyResponse + type_name: UNDEFINED +- id: 0x920d + name: Noise + type_name: UNDEFINED +- id: 0x920e + name: FocalPlaneXResolution + type_name: RATIONAL +- id: 0x920f + name: FocalPlaneYResolution + type_name: RATIONAL +- id: 0x9210 + name: FocalPlaneResolutionUnit + type_name: SHORT +- id: 0x9211 + name: ImageNumber + type_name: LONG +- id: 0x9212 + name: SecurityClassification + type_name: ASCII +- id: 0x9213 + name: ImageHistory + type_name: ASCII +- id: 0x9214 + name: SubjectLocation + type_name: SHORT +- id: 0x9215 + name: ExposureIndex + type_name: RATIONAL +- id: 0x9216 + name: TIFFEPStandardID + type_name: BYTE +- id: 0x9217 + name: SensingMethod + type_name: SHORT +- id: 0x9c9b + name: XPTitle + type_name: BYTE +- id: 0x9c9c + name: XPComment + type_name: BYTE +- id: 0x9c9d + name: XPAuthor + type_name: BYTE +- id: 0x9c9e + name: XPKeywords + type_name: BYTE +- id: 0x9c9f + name: XPSubject + type_name: BYTE +- id: 0xc4a5 + name: PrintImageMatching + type_name: UNDEFINED +- id: 0xc612 + name: DNGVersion + type_name: BYTE +- id: 0xc613 + name: DNGBackwardVersion + type_name: BYTE +- id: 0xc614 + name: UniqueCameraModel + type_name: ASCII +- id: 0xc615 + name: LocalizedCameraModel + type_name: BYTE +- id: 0xc616 + name: CFAPlaneColor + type_name: BYTE +- id: 0xc617 + name: CFALayout + type_name: SHORT +- id: 0xc618 + name: LinearizationTable + type_name: SHORT +- id: 0xc619 + name: BlackLevelRepeatDim + type_name: SHORT +- id: 0xc61a + name: BlackLevel + type_name: RATIONAL +- id: 0xc61b + name: BlackLevelDeltaH + type_name: SRATIONAL +- id: 0xc61c + name: BlackLevelDeltaV + type_name: SRATIONAL +- id: 0xc61d + name: WhiteLevel + type_name: SHORT +- id: 0xc61e + name: DefaultScale + type_name: RATIONAL +- id: 0xc61f + name: DefaultCropOrigin + type_name: SHORT +- id: 0xc620 + name: DefaultCropSize + type_name: SHORT +- id: 0xc621 + name: ColorMatrix1 + type_name: SRATIONAL +- id: 0xc622 + name: ColorMatrix2 + type_name: SRATIONAL +- id: 0xc623 + name: CameraCalibration1 + type_name: SRATIONAL +- id: 0xc624 + name: CameraCalibration2 + type_name: SRATIONAL +- id: 0xc625 + name: ReductionMatrix1 + type_name: SRATIONAL +- id: 0xc626 + name: ReductionMatrix2 + type_name: SRATIONAL +- id: 0xc627 + name: AnalogBalance + type_name: RATIONAL +- id: 0xc628 + name: AsShotNeutral + type_name: SHORT +- id: 0xc629 + name: AsShotWhiteXY + type_name: RATIONAL +- id: 0xc62a + name: BaselineExposure + type_name: SRATIONAL +- id: 0xc62b + name: BaselineNoise + type_name: RATIONAL +- id: 0xc62c + name: BaselineSharpness + type_name: RATIONAL +- id: 0xc62d + name: BayerGreenSplit + type_name: LONG +- id: 0xc62e + name: LinearResponseLimit + type_name: RATIONAL +- id: 0xc62f + name: CameraSerialNumber + type_name: ASCII +- id: 0xc630 + name: LensInfo + type_name: RATIONAL +- id: 0xc631 + name: ChromaBlurRadius + type_name: RATIONAL +- id: 0xc632 + name: AntiAliasStrength + type_name: RATIONAL +- id: 0xc633 + name: ShadowScale + type_name: SRATIONAL +- id: 0xc634 + name: DNGPrivateData + type_name: BYTE +- id: 0xc635 + name: MakerNoteSafety + type_name: SHORT +- id: 0xc65a + name: CalibrationIlluminant1 + type_name: SHORT +- id: 0xc65b + name: CalibrationIlluminant2 + type_name: SHORT +- id: 0xc65c + name: BestQualityScale + type_name: RATIONAL +- id: 0xc65d + name: RawDataUniqueID + type_name: BYTE +- id: 0xc68b + name: OriginalRawFileName + type_name: BYTE +- id: 0xc68c + name: OriginalRawFileData + type_name: UNDEFINED +- id: 0xc68d + name: ActiveArea + type_name: SHORT +- id: 0xc68e + name: MaskedAreas + type_name: SHORT +- id: 0xc68f + name: AsShotICCProfile + type_name: UNDEFINED +- id: 0xc690 + name: AsShotPreProfileMatrix + type_name: SRATIONAL +- id: 0xc691 + name: CurrentICCProfile + type_name: UNDEFINED +- id: 0xc692 + name: CurrentPreProfileMatrix + type_name: SRATIONAL +- id: 0xc6bf + name: ColorimetricReference + type_name: SHORT +- id: 0xc6f3 + name: CameraCalibrationSignature + type_name: BYTE +- id: 0xc6f4 + name: ProfileCalibrationSignature + type_name: BYTE +- id: 0xc6f6 + name: AsShotProfileName + type_name: BYTE +- id: 0xc6f7 + name: NoiseReductionApplied + type_name: RATIONAL +- id: 0xc6f8 + name: ProfileName + type_name: BYTE +- id: 0xc6f9 + name: ProfileHueSatMapDims + type_name: LONG +- id: 0xc6fd + name: ProfileEmbedPolicy + type_name: LONG +- id: 0xc6fe + name: ProfileCopyright + type_name: BYTE +- id: 0xc714 + name: ForwardMatrix1 + type_name: SRATIONAL +- id: 0xc715 + name: ForwardMatrix2 + type_name: SRATIONAL +- id: 0xc716 + name: PreviewApplicationName + type_name: BYTE +- id: 0xc717 + name: PreviewApplicationVersion + type_name: BYTE +- id: 0xc718 + name: PreviewSettingsName + type_name: BYTE +- id: 0xc719 + name: PreviewSettingsDigest + type_name: BYTE +- id: 0xc71a + name: PreviewColorSpace + type_name: LONG +- id: 0xc71b + name: PreviewDateTime + type_name: ASCII +- id: 0xc71c + name: RawImageDigest + type_name: UNDEFINED +- id: 0xc71d + name: OriginalRawFileDigest + type_name: UNDEFINED +- id: 0xc71e + name: SubTileBlockSize + type_name: LONG +- id: 0xc71f + name: RowInterleaveFactor + type_name: LONG +- id: 0xc725 + name: ProfileLookTableDims + type_name: LONG +- id: 0xc740 + name: OpcodeList1 + type_name: UNDEFINED +- id: 0xc741 + name: OpcodeList2 + type_name: UNDEFINED +- id: 0xc74e + name: OpcodeList3 + type_name: UNDEFINED +# This tag may be used to specify the size of raster pixel spacing in the +# model space units, when the raster space can be embedded in the model space +# coordinate system without rotation, and consists of the following 3 values: +# ModelPixelScaleTag = (ScaleX, ScaleY, ScaleZ) +# where ScaleX and ScaleY give the horizontal and vertical spacing of raster +# pixels. The ScaleZ is primarily used to map the pixel value of a digital +# elevation model into the correct Z-scale, and so for most other purposes +# this value should be zero (since most model spaces are 2-D, with Z=0). +# Source: http://geotiff.maptools.org/spec/geotiff2.6.html#2.6.1 +- id: 0x830e + name: ModelPixelScaleTag + type_name: DOUBLE +# This tag stores raster->model tiepoint pairs in the order +# ModelTiepointTag = (...,I,J,K, X,Y,Z...), +# where (I,J,K) is the point at location (I,J) in raster space with +# pixel-value K, and (X,Y,Z) is a vector in model space. In most cases the +# model space is only two-dimensional, in which case both K and Z should be +# set to zero; this third dimension is provided in anticipation of future +# support for 3D digital elevation models and vertical coordinate systems. +# Source: http://geotiff.maptools.org/spec/geotiff2.6.html#2.6.1 +- id: 0x8482 + name: ModelTiepointTag + type_name: DOUBLE +# This tag may be used to specify the transformation matrix between the +# raster space (and its dependent pixel-value space) and the (possibly 3D) +# model space. +# Source: http://geotiff.maptools.org/spec/geotiff2.6.html#2.6.1 +- id: 0x85d8 + name: ModelTransformationTag + type_name: DOUBLE +IFD/Exif/Iop: +- id: 0x0001 + name: InteroperabilityIndex + type_name: ASCII +- id: 0x0002 + name: InteroperabilityVersion + type_name: UNDEFINED +- id: 0x1000 + name: RelatedImageFileFormat + type_name: ASCII +- id: 0x1001 + name: RelatedImageWidth + type_name: LONG +- id: 0x1002 + name: RelatedImageLength + type_name: LONG +` +) diff --git a/vendor/github.com/dsoprea/go-exif/v3/testing_common.go b/vendor/github.com/dsoprea/go-exif/v3/testing_common.go new file mode 100644 index 000000000..061276430 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/testing_common.go @@ -0,0 +1,188 @@ +package exif + +import ( + "path" + "reflect" + "testing" + + "io/ioutil" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +var ( + testExifData []byte +) + +func getExifSimpleTestIb() *IfdBuilder { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + im := exifcommon.NewIfdMapping() + + err := exifcommon.LoadStandardIfds(im) + log.PanicIf(err) + + ti := NewTagIndex() + ib := NewIfdBuilder(im, ti, exifcommon.IfdStandardIfdIdentity, exifcommon.TestDefaultByteOrder) + + err = ib.AddStandard(0x000b, "asciivalue") + log.PanicIf(err) + + err = ib.AddStandard(0x00ff, []uint16{0x1122}) + log.PanicIf(err) + + err = ib.AddStandard(0x0100, []uint32{0x33445566}) + log.PanicIf(err) + + err = ib.AddStandard(0x013e, []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}) + log.PanicIf(err) + + return ib +} + +func getExifSimpleTestIbBytes() []byte { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + im := exifcommon.NewIfdMapping() + + err := exifcommon.LoadStandardIfds(im) + log.PanicIf(err) + + ti := NewTagIndex() + ib := NewIfdBuilder(im, ti, exifcommon.IfdStandardIfdIdentity, exifcommon.TestDefaultByteOrder) + + err = ib.AddStandard(0x000b, "asciivalue") + log.PanicIf(err) + + err = ib.AddStandard(0x00ff, []uint16{0x1122}) + log.PanicIf(err) + + err = ib.AddStandard(0x0100, []uint32{0x33445566}) + log.PanicIf(err) + + err = ib.AddStandard(0x013e, []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}) + log.PanicIf(err) + + ibe := NewIfdByteEncoder() + + exifData, err := ibe.EncodeToExif(ib) + log.PanicIf(err) + + return exifData +} + +func validateExifSimpleTestIb(exifData []byte, t *testing.T) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + im := exifcommon.NewIfdMapping() + + err := exifcommon.LoadStandardIfds(im) + log.PanicIf(err) + + ti := NewTagIndex() + + eh, index, err := Collect(im, ti, exifData) + log.PanicIf(err) + + if eh.ByteOrder != exifcommon.TestDefaultByteOrder { + t.Fatalf("EXIF byte-order is not correct: %v", eh.ByteOrder) + } else if eh.FirstIfdOffset != ExifDefaultFirstIfdOffset { + t.Fatalf("EXIF first IFD-offset not correct: (0x%02x)", eh.FirstIfdOffset) + } + + if len(index.Ifds) != 1 { + t.Fatalf("There wasn't exactly one IFD decoded: (%d)", len(index.Ifds)) + } + + ifd := index.RootIfd + + if ifd.ByteOrder() != exifcommon.TestDefaultByteOrder { + t.Fatalf("IFD byte-order not correct.") + } else if ifd.ifdIdentity.UnindexedString() != exifcommon.IfdStandardIfdIdentity.UnindexedString() { + t.Fatalf("IFD name not correct.") + } else if ifd.ifdIdentity.Index() != 0 { + t.Fatalf("IFD index not zero: (%d)", ifd.ifdIdentity.Index()) + } else if ifd.Offset() != uint32(0x0008) { + t.Fatalf("IFD offset not correct.") + } else if len(ifd.Entries()) != 4 { + t.Fatalf("IFD number of entries not correct: (%d)", len(ifd.Entries())) + } else if ifd.nextIfdOffset != uint32(0) { + t.Fatalf("Next-IFD offset is non-zero.") + } else if ifd.nextIfd != nil { + t.Fatalf("Next-IFD pointer is non-nil.") + } + + // Verify the values by using the actual, original types (this is awesome). + + expected := []struct { + tagId uint16 + value interface{} + }{ + {tagId: 0x000b, value: "asciivalue"}, + {tagId: 0x00ff, value: []uint16{0x1122}}, + {tagId: 0x0100, value: []uint32{0x33445566}}, + {tagId: 0x013e, value: []exifcommon.Rational{{Numerator: 0x11112222, Denominator: 0x33334444}}}, + } + + for i, ite := range ifd.Entries() { + if ite.TagId() != expected[i].tagId { + t.Fatalf("Tag-ID for entry (%d) not correct: (0x%02x) != (0x%02x)", i, ite.TagId(), expected[i].tagId) + } + + value, err := ite.Value() + log.PanicIf(err) + + if reflect.DeepEqual(value, expected[i].value) != true { + t.Fatalf("Value for entry (%d) not correct: [%v] != [%v]", i, value, expected[i].value) + } + } +} + +func getTestImageFilepath() string { + assetsPath := exifcommon.GetTestAssetsPath() + testImageFilepath := path.Join(assetsPath, "NDM_8901.jpg") + return testImageFilepath +} + +func getTestExifData() []byte { + if testExifData == nil { + assetsPath := exifcommon.GetTestAssetsPath() + filepath := path.Join(assetsPath, "NDM_8901.jpg.exif") + + var err error + + testExifData, err = ioutil.ReadFile(filepath) + log.PanicIf(err) + } + + return testExifData +} + +func getTestGpsImageFilepath() string { + assetsPath := exifcommon.GetTestAssetsPath() + testGpsImageFilepath := path.Join(assetsPath, "gps.jpg") + return testGpsImageFilepath +} + +func getTestGeotiffFilepath() string { + assetsPath := exifcommon.GetTestAssetsPath() + testGeotiffFilepath := path.Join(assetsPath, "geotiff_example.tif") + return testGeotiffFilepath +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/README.md b/vendor/github.com/dsoprea/go-exif/v3/undefined/README.md new file mode 100644 index 000000000..d2caa6e51 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/README.md @@ -0,0 +1,4 @@ + +## 0xa40b + +The specification is not specific/clear enough to be handled. Without a working example ,we're deferring until some point in the future when either we or someone else has a better understanding. diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/accessor.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/accessor.go new file mode 100644 index 000000000..11a21e1f0 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/accessor.go @@ -0,0 +1,62 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +// Encode encodes the given encodeable undefined value to bytes. +func Encode(value EncodeableValue, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + encoderName := value.EncoderName() + + encoder, found := encoders[encoderName] + if found == false { + log.Panicf("no encoder registered for type [%s]", encoderName) + } + + encoded, unitCount, err = encoder.Encode(value, byteOrder) + log.PanicIf(err) + + return encoded, unitCount, nil +} + +// Decode constructs a value from raw encoded bytes +func Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + uth := UndefinedTagHandle{ + IfdPath: valueContext.IfdPath(), + TagId: valueContext.TagId(), + } + + decoder, found := decoders[uth] + if found == false { + // We have no choice but to return the error. We have no way of knowing how + // much data there is without already knowing what data-type this tag is. + return nil, exifcommon.ErrUnhandledUndefinedTypedTag + } + + value, err = decoder.Decode(valueContext) + if err != nil { + if err == ErrUnparseableValue { + return nil, err + } + + log.Panic(err) + } + + return value, nil +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_8828_oecf.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_8828_oecf.go new file mode 100644 index 000000000..26f3675ab --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_8828_oecf.go @@ -0,0 +1,148 @@ +package exifundefined + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag8828Oecf struct { + Columns uint16 + Rows uint16 + ColumnNames []string + Values []exifcommon.SignedRational +} + +func (oecf Tag8828Oecf) String() string { + return fmt.Sprintf("Tag8828Oecf", oecf.Columns, oecf.Rows) +} + +func (oecf Tag8828Oecf) EncoderName() string { + return "Codec8828Oecf" +} + +type Codec8828Oecf struct { +} + +func (Codec8828Oecf) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + oecf, ok := value.(Tag8828Oecf) + if ok == false { + log.Panicf("can only encode a Tag8828Oecf") + } + + b := new(bytes.Buffer) + + err = binary.Write(b, byteOrder, oecf.Columns) + log.PanicIf(err) + + err = binary.Write(b, byteOrder, oecf.Rows) + log.PanicIf(err) + + for _, name := range oecf.ColumnNames { + _, err := b.Write([]byte(name)) + log.PanicIf(err) + + _, err = b.Write([]byte{0}) + log.PanicIf(err) + } + + ve := exifcommon.NewValueEncoder(byteOrder) + + ed, err := ve.Encode(oecf.Values) + log.PanicIf(err) + + _, err = b.Write(ed.Encoded) + log.PanicIf(err) + + return b.Bytes(), uint32(b.Len()), nil +} + +func (Codec8828Oecf) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test using known good data. + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + oecf := Tag8828Oecf{} + + oecf.Columns = valueContext.ByteOrder().Uint16(valueBytes[0:2]) + oecf.Rows = valueContext.ByteOrder().Uint16(valueBytes[2:4]) + + columnNames := make([]string, oecf.Columns) + + // startAt is where the current column name starts. + startAt := 4 + + // offset is our current position. + offset := startAt + + currentColumnNumber := uint16(0) + + for currentColumnNumber < oecf.Columns { + if valueBytes[offset] == 0 { + columnName := string(valueBytes[startAt:offset]) + if len(columnName) == 0 { + log.Panicf("SFR column (%d) has zero length", currentColumnNumber) + } + + columnNames[currentColumnNumber] = columnName + currentColumnNumber++ + + offset++ + startAt = offset + continue + } + + offset++ + } + + oecf.ColumnNames = columnNames + + rawRationalBytes := valueBytes[offset:] + + rationalSize := exifcommon.TypeSignedRational.Size() + if len(rawRationalBytes)%rationalSize > 0 { + log.Panicf("OECF signed-rationals not aligned: (%d) %% (%d) > 0", len(rawRationalBytes), rationalSize) + } + + rationalCount := len(rawRationalBytes) / rationalSize + + parser := new(exifcommon.Parser) + + byteOrder := valueContext.ByteOrder() + + items, err := parser.ParseSignedRationals(rawRationalBytes, uint32(rationalCount), byteOrder) + log.PanicIf(err) + + oecf.Values = items + + return oecf, nil +} + +func init() { + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0x8828, + Codec8828Oecf{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9000_exif_version.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9000_exif_version.go new file mode 100644 index 000000000..8f18c8114 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9000_exif_version.go @@ -0,0 +1,69 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag9000ExifVersion struct { + ExifVersion string +} + +func (Tag9000ExifVersion) EncoderName() string { + return "Codec9000ExifVersion" +} + +func (ev Tag9000ExifVersion) String() string { + return ev.ExifVersion +} + +type Codec9000ExifVersion struct { +} + +func (Codec9000ExifVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s, ok := value.(Tag9000ExifVersion) + if ok == false { + log.Panicf("can only encode a Tag9000ExifVersion") + } + + return []byte(s.ExifVersion), uint32(len(s.ExifVersion)), nil +} + +func (Codec9000ExifVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) + + valueString, err := valueContext.ReadAsciiNoNul() + log.PanicIf(err) + + ev := Tag9000ExifVersion{ + ExifVersion: valueString, + } + + return ev, nil +} + +func init() { + registerEncoder( + Tag9000ExifVersion{}, + Codec9000ExifVersion{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0x9000, + Codec9000ExifVersion{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9101_components_configuration.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9101_components_configuration.go new file mode 100644 index 000000000..e357fe0a6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9101_components_configuration.go @@ -0,0 +1,124 @@ +package exifundefined + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +const ( + TagUndefinedType_9101_ComponentsConfiguration_Channel_Y = 0x1 + TagUndefinedType_9101_ComponentsConfiguration_Channel_Cb = 0x2 + TagUndefinedType_9101_ComponentsConfiguration_Channel_Cr = 0x3 + TagUndefinedType_9101_ComponentsConfiguration_Channel_R = 0x4 + TagUndefinedType_9101_ComponentsConfiguration_Channel_G = 0x5 + TagUndefinedType_9101_ComponentsConfiguration_Channel_B = 0x6 +) + +const ( + TagUndefinedType_9101_ComponentsConfiguration_OTHER = iota + TagUndefinedType_9101_ComponentsConfiguration_RGB = iota + TagUndefinedType_9101_ComponentsConfiguration_YCBCR = iota +) + +var ( + TagUndefinedType_9101_ComponentsConfiguration_Names = map[int]string{ + TagUndefinedType_9101_ComponentsConfiguration_OTHER: "OTHER", + TagUndefinedType_9101_ComponentsConfiguration_RGB: "RGB", + TagUndefinedType_9101_ComponentsConfiguration_YCBCR: "YCBCR", + } + + TagUndefinedType_9101_ComponentsConfiguration_Configurations = map[int][]byte{ + TagUndefinedType_9101_ComponentsConfiguration_RGB: { + TagUndefinedType_9101_ComponentsConfiguration_Channel_R, + TagUndefinedType_9101_ComponentsConfiguration_Channel_G, + TagUndefinedType_9101_ComponentsConfiguration_Channel_B, + 0, + }, + + TagUndefinedType_9101_ComponentsConfiguration_YCBCR: { + TagUndefinedType_9101_ComponentsConfiguration_Channel_Y, + TagUndefinedType_9101_ComponentsConfiguration_Channel_Cb, + TagUndefinedType_9101_ComponentsConfiguration_Channel_Cr, + 0, + }, + } +) + +type TagExif9101ComponentsConfiguration struct { + ConfigurationId int + ConfigurationBytes []byte +} + +func (TagExif9101ComponentsConfiguration) EncoderName() string { + return "CodecExif9101ComponentsConfiguration" +} + +func (cc TagExif9101ComponentsConfiguration) String() string { + return fmt.Sprintf("Exif9101ComponentsConfiguration", TagUndefinedType_9101_ComponentsConfiguration_Names[cc.ConfigurationId], cc.ConfigurationBytes) +} + +type CodecExif9101ComponentsConfiguration struct { +} + +func (CodecExif9101ComponentsConfiguration) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + cc, ok := value.(TagExif9101ComponentsConfiguration) + if ok == false { + log.Panicf("can only encode a TagExif9101ComponentsConfiguration") + } + + return cc.ConfigurationBytes, uint32(len(cc.ConfigurationBytes)), nil +} + +func (CodecExif9101ComponentsConfiguration) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + for configurationId, configurationBytes := range TagUndefinedType_9101_ComponentsConfiguration_Configurations { + if bytes.Equal(configurationBytes, valueBytes) == true { + cc := TagExif9101ComponentsConfiguration{ + ConfigurationId: configurationId, + ConfigurationBytes: valueBytes, + } + + return cc, nil + } + } + + cc := TagExif9101ComponentsConfiguration{ + ConfigurationId: TagUndefinedType_9101_ComponentsConfiguration_OTHER, + ConfigurationBytes: valueBytes, + } + + return cc, nil +} + +func init() { + registerEncoder( + TagExif9101ComponentsConfiguration{}, + CodecExif9101ComponentsConfiguration{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0x9101, + CodecExif9101ComponentsConfiguration{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_927C_maker_note.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_927C_maker_note.go new file mode 100644 index 000000000..f9cd2788e --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_927C_maker_note.go @@ -0,0 +1,114 @@ +package exifundefined + +import ( + "fmt" + "strings" + + "crypto/sha1" + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag927CMakerNote struct { + MakerNoteType []byte + MakerNoteBytes []byte +} + +func (Tag927CMakerNote) EncoderName() string { + return "Codec927CMakerNote" +} + +func (mn Tag927CMakerNote) String() string { + parts := make([]string, len(mn.MakerNoteType)) + + for i, c := range mn.MakerNoteType { + parts[i] = fmt.Sprintf("%02x", c) + } + + h := sha1.New() + + _, err := h.Write(mn.MakerNoteBytes) + log.PanicIf(err) + + digest := h.Sum(nil) + + return fmt.Sprintf("MakerNote", strings.Join(parts, " "), len(mn.MakerNoteBytes), digest) +} + +type Codec927CMakerNote struct { +} + +func (Codec927CMakerNote) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + mn, ok := value.(Tag927CMakerNote) + if ok == false { + log.Panicf("can only encode a Tag927CMakerNote") + } + + // TODO(dustin): Confirm this size against the specification. + + return mn.MakerNoteBytes, uint32(len(mn.MakerNoteBytes)), nil +} + +func (Codec927CMakerNote) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // MakerNote + // TODO(dustin): !! This is the Wild Wild West. This very well might be a child IFD, but any and all OEM's define their own formats. If we're going to be writing changes and this is complete EXIF (which may not have the first eight bytes), it might be fine. However, if these are just IFDs they'll be relative to the main EXIF, this will invalidate the MakerNote data for IFDs and any other implementations that use offsets unless we can interpret them all. It be best to return to this later and just exclude this from being written for now, though means a loss of a wealth of image metadata. + // -> We can also just blindly try to interpret as an IFD and just validate that it's looks good (maybe it will even have a 'next ifd' pointer that we can validate is 0x0). + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + // TODO(dustin): Doesn't work, but here as an example. + // ie := NewIfdEnumerate(valueBytes, byteOrder) + + // // TODO(dustin): !! Validate types (might have proprietary types, but it might be worth splitting the list between valid and not valid; maybe fail if a certain proportion are invalid, or maybe aren't less then a certain small integer)? + // ii, err := ie.Collect(0x0) + + // for _, entry := range ii.RootIfd.Entries { + // fmt.Printf("ENTRY: 0x%02x %d\n", entry.TagId, entry.TagType) + // } + + var makerNoteType []byte + if len(valueBytes) >= 20 { + makerNoteType = valueBytes[:20] + } else { + makerNoteType = valueBytes + } + + mn := Tag927CMakerNote{ + MakerNoteType: makerNoteType, + + // MakerNoteBytes has the whole length of bytes. There's always + // the chance that the first 20 bytes includes actual data. + MakerNoteBytes: valueBytes, + } + + return mn, nil +} + +func init() { + registerEncoder( + Tag927CMakerNote{}, + Codec927CMakerNote{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0x927c, + Codec927CMakerNote{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9286_user_comment.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9286_user_comment.go new file mode 100644 index 000000000..320edc145 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_9286_user_comment.go @@ -0,0 +1,142 @@ +package exifundefined + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +var ( + exif9286Logger = log.NewLogger("exifundefined.exif_9286_user_comment") +) + +const ( + TagUndefinedType_9286_UserComment_Encoding_ASCII = iota + TagUndefinedType_9286_UserComment_Encoding_JIS = iota + TagUndefinedType_9286_UserComment_Encoding_UNICODE = iota + TagUndefinedType_9286_UserComment_Encoding_UNDEFINED = iota +) + +var ( + TagUndefinedType_9286_UserComment_Encoding_Names = map[int]string{ + TagUndefinedType_9286_UserComment_Encoding_ASCII: "ASCII", + TagUndefinedType_9286_UserComment_Encoding_JIS: "JIS", + TagUndefinedType_9286_UserComment_Encoding_UNICODE: "UNICODE", + TagUndefinedType_9286_UserComment_Encoding_UNDEFINED: "UNDEFINED", + } + + TagUndefinedType_9286_UserComment_Encodings = map[int][]byte{ + TagUndefinedType_9286_UserComment_Encoding_ASCII: {'A', 'S', 'C', 'I', 'I', 0, 0, 0}, + TagUndefinedType_9286_UserComment_Encoding_JIS: {'J', 'I', 'S', 0, 0, 0, 0, 0}, + TagUndefinedType_9286_UserComment_Encoding_UNICODE: {'U', 'n', 'i', 'c', 'o', 'd', 'e', 0}, + TagUndefinedType_9286_UserComment_Encoding_UNDEFINED: {0, 0, 0, 0, 0, 0, 0, 0}, + } +) + +type Tag9286UserComment struct { + EncodingType int + EncodingBytes []byte +} + +func (Tag9286UserComment) EncoderName() string { + return "Codec9286UserComment" +} + +func (uc Tag9286UserComment) String() string { + var valuePhrase string + + if uc.EncodingType == TagUndefinedType_9286_UserComment_Encoding_ASCII { + return fmt.Sprintf("[ASCII] %s", string(uc.EncodingBytes)) + } else { + if len(uc.EncodingBytes) <= 8 { + valuePhrase = fmt.Sprintf("%v", uc.EncodingBytes) + } else { + valuePhrase = fmt.Sprintf("%v...", uc.EncodingBytes[:8]) + } + } + + return fmt.Sprintf("UserComment", len(uc.EncodingBytes), TagUndefinedType_9286_UserComment_Encoding_Names[uc.EncodingType], valuePhrase, len(uc.EncodingBytes)) +} + +type Codec9286UserComment struct { +} + +func (Codec9286UserComment) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + uc, ok := value.(Tag9286UserComment) + if ok == false { + log.Panicf("can only encode a Tag9286UserComment") + } + + encodingTypeBytes, found := TagUndefinedType_9286_UserComment_Encodings[uc.EncodingType] + if found == false { + log.Panicf("encoding-type not valid for unknown-type tag 9286 (UserComment): (%d)", uc.EncodingType) + } + + encoded = make([]byte, len(uc.EncodingBytes)+8) + + copy(encoded[:8], encodingTypeBytes) + copy(encoded[8:], uc.EncodingBytes) + + // TODO(dustin): Confirm this size against the specification. + + return encoded, uint32(len(encoded)), nil +} + +func (Codec9286UserComment) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + if len(valueBytes) < 8 { + return nil, ErrUnparseableValue + } + + unknownUc := Tag9286UserComment{ + EncodingType: TagUndefinedType_9286_UserComment_Encoding_UNDEFINED, + EncodingBytes: []byte{}, + } + + encoding := valueBytes[:8] + for encodingIndex, encodingBytes := range TagUndefinedType_9286_UserComment_Encodings { + if bytes.Compare(encoding, encodingBytes) == 0 { + uc := Tag9286UserComment{ + EncodingType: encodingIndex, + EncodingBytes: valueBytes[8:], + } + + return uc, nil + } + } + + exif9286Logger.Warningf(nil, "User-comment encoding not valid. Returning 'unknown' type (the default).") + return unknownUc, nil +} + +func init() { + registerEncoder( + Tag9286UserComment{}, + Codec9286UserComment{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0x9286, + Codec9286UserComment{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A000_flashpix_version.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A000_flashpix_version.go new file mode 100644 index 000000000..4a0fefad7 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A000_flashpix_version.go @@ -0,0 +1,69 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type TagA000FlashpixVersion struct { + FlashpixVersion string +} + +func (TagA000FlashpixVersion) EncoderName() string { + return "CodecA000FlashpixVersion" +} + +func (fv TagA000FlashpixVersion) String() string { + return fv.FlashpixVersion +} + +type CodecA000FlashpixVersion struct { +} + +func (CodecA000FlashpixVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s, ok := value.(TagA000FlashpixVersion) + if ok == false { + log.Panicf("can only encode a TagA000FlashpixVersion") + } + + return []byte(s.FlashpixVersion), uint32(len(s.FlashpixVersion)), nil +} + +func (CodecA000FlashpixVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) + + valueString, err := valueContext.ReadAsciiNoNul() + log.PanicIf(err) + + fv := TagA000FlashpixVersion{ + FlashpixVersion: valueString, + } + + return fv, nil +} + +func init() { + registerEncoder( + TagA000FlashpixVersion{}, + CodecA000FlashpixVersion{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0xa000, + CodecA000FlashpixVersion{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A20C_spatial_frequency_response.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A20C_spatial_frequency_response.go new file mode 100644 index 000000000..0311175d6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A20C_spatial_frequency_response.go @@ -0,0 +1,160 @@ +package exifundefined + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type TagA20CSpatialFrequencyResponse struct { + Columns uint16 + Rows uint16 + ColumnNames []string + Values []exifcommon.Rational +} + +func (TagA20CSpatialFrequencyResponse) EncoderName() string { + return "CodecA20CSpatialFrequencyResponse" +} + +func (sfr TagA20CSpatialFrequencyResponse) String() string { + return fmt.Sprintf("CodecA20CSpatialFrequencyResponse", sfr.Columns, sfr.Rows) +} + +type CodecA20CSpatialFrequencyResponse struct { +} + +func (CodecA20CSpatialFrequencyResponse) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test. + + sfr, ok := value.(TagA20CSpatialFrequencyResponse) + if ok == false { + log.Panicf("can only encode a TagA20CSpatialFrequencyResponse") + } + + b := new(bytes.Buffer) + + err = binary.Write(b, byteOrder, sfr.Columns) + log.PanicIf(err) + + err = binary.Write(b, byteOrder, sfr.Rows) + log.PanicIf(err) + + // Write columns. + + for _, name := range sfr.ColumnNames { + _, err := b.WriteString(name) + log.PanicIf(err) + + err = b.WriteByte(0) + log.PanicIf(err) + } + + // Write values. + + ve := exifcommon.NewValueEncoder(byteOrder) + + ed, err := ve.Encode(sfr.Values) + log.PanicIf(err) + + _, err = b.Write(ed.Encoded) + log.PanicIf(err) + + encoded = b.Bytes() + + // TODO(dustin): Confirm this size against the specification. + + return encoded, uint32(len(encoded)), nil +} + +func (CodecA20CSpatialFrequencyResponse) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test using known good data. + + byteOrder := valueContext.ByteOrder() + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + sfr := TagA20CSpatialFrequencyResponse{} + + sfr.Columns = byteOrder.Uint16(valueBytes[0:2]) + sfr.Rows = byteOrder.Uint16(valueBytes[2:4]) + + columnNames := make([]string, sfr.Columns) + + // startAt is where the current column name starts. + startAt := 4 + + // offset is our current position. + offset := 4 + + currentColumnNumber := uint16(0) + + for currentColumnNumber < sfr.Columns { + if valueBytes[offset] == 0 { + columnName := string(valueBytes[startAt:offset]) + if len(columnName) == 0 { + log.Panicf("SFR column (%d) has zero length", currentColumnNumber) + } + + columnNames[currentColumnNumber] = columnName + currentColumnNumber++ + + offset++ + startAt = offset + continue + } + + offset++ + } + + sfr.ColumnNames = columnNames + + rawRationalBytes := valueBytes[offset:] + + rationalSize := exifcommon.TypeRational.Size() + if len(rawRationalBytes)%rationalSize > 0 { + log.Panicf("SFR rationals not aligned: (%d) %% (%d) > 0", len(rawRationalBytes), rationalSize) + } + + rationalCount := len(rawRationalBytes) / rationalSize + + parser := new(exifcommon.Parser) + + items, err := parser.ParseRationals(rawRationalBytes, uint32(rationalCount), byteOrder) + log.PanicIf(err) + + sfr.Values = items + + return sfr, nil +} + +func init() { + registerEncoder( + TagA20CSpatialFrequencyResponse{}, + CodecA20CSpatialFrequencyResponse{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0xa20c, + CodecA20CSpatialFrequencyResponse{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A300_file_source.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A300_file_source.go new file mode 100644 index 000000000..f4f3a49f9 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A300_file_source.go @@ -0,0 +1,79 @@ +package exifundefined + +import ( + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type TagExifA300FileSource uint32 + +func (TagExifA300FileSource) EncoderName() string { + return "CodecExifA300FileSource" +} + +func (af TagExifA300FileSource) String() string { + return fmt.Sprintf("0x%08x", uint32(af)) +} + +const ( + TagUndefinedType_A300_SceneType_Others TagExifA300FileSource = 0 + TagUndefinedType_A300_SceneType_ScannerOfTransparentType TagExifA300FileSource = 1 + TagUndefinedType_A300_SceneType_ScannerOfReflexType TagExifA300FileSource = 2 + TagUndefinedType_A300_SceneType_Dsc TagExifA300FileSource = 3 +) + +type CodecExifA300FileSource struct { +} + +func (CodecExifA300FileSource) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + st, ok := value.(TagExifA300FileSource) + if ok == false { + log.Panicf("can only encode a TagExifA300FileSource") + } + + ve := exifcommon.NewValueEncoder(byteOrder) + + ed, err := ve.Encode([]uint32{uint32(st)}) + log.PanicIf(err) + + // TODO(dustin): Confirm this size against the specification. It's non-specific about what type it is, but it looks to be no more than a single integer scalar. So, we're assuming it's a LONG. + + return ed.Encoded, 1, nil +} + +func (CodecExifA300FileSource) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeLong) + + valueLongs, err := valueContext.ReadLongs() + log.PanicIf(err) + + return TagExifA300FileSource(valueLongs[0]), nil +} + +func init() { + registerEncoder( + TagExifA300FileSource(0), + CodecExifA300FileSource{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0xa300, + CodecExifA300FileSource{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A301_scene_type.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A301_scene_type.go new file mode 100644 index 000000000..a29fd7673 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A301_scene_type.go @@ -0,0 +1,76 @@ +package exifundefined + +import ( + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type TagExifA301SceneType uint32 + +func (TagExifA301SceneType) EncoderName() string { + return "CodecExifA301SceneType" +} + +func (st TagExifA301SceneType) String() string { + return fmt.Sprintf("0x%08x", uint32(st)) +} + +const ( + TagUndefinedType_A301_SceneType_DirectlyPhotographedImage TagExifA301SceneType = 1 +) + +type CodecExifA301SceneType struct { +} + +func (CodecExifA301SceneType) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + st, ok := value.(TagExifA301SceneType) + if ok == false { + log.Panicf("can only encode a TagExif9101ComponentsConfiguration") + } + + ve := exifcommon.NewValueEncoder(byteOrder) + + ed, err := ve.Encode([]uint32{uint32(st)}) + log.PanicIf(err) + + // TODO(dustin): Confirm this size against the specification. It's non-specific about what type it is, but it looks to be no more than a single integer scalar. So, we're assuming it's a LONG. + + return ed.Encoded, uint32(int(ed.UnitCount)), nil +} + +func (CodecExifA301SceneType) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeLong) + + valueLongs, err := valueContext.ReadLongs() + log.PanicIf(err) + + return TagExifA301SceneType(valueLongs[0]), nil +} + +func init() { + registerEncoder( + TagExifA301SceneType(0), + CodecExifA301SceneType{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0xa301, + CodecExifA301SceneType{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A302_cfa_pattern.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A302_cfa_pattern.go new file mode 100644 index 000000000..88976296d --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_A302_cfa_pattern.go @@ -0,0 +1,97 @@ +package exifundefined + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type TagA302CfaPattern struct { + HorizontalRepeat uint16 + VerticalRepeat uint16 + CfaValue []byte +} + +func (TagA302CfaPattern) EncoderName() string { + return "CodecA302CfaPattern" +} + +func (cp TagA302CfaPattern) String() string { + return fmt.Sprintf("TagA302CfaPattern", cp.HorizontalRepeat, cp.VerticalRepeat, len(cp.CfaValue)) +} + +type CodecA302CfaPattern struct { +} + +func (CodecA302CfaPattern) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test. + + cp, ok := value.(TagA302CfaPattern) + if ok == false { + log.Panicf("can only encode a TagA302CfaPattern") + } + + b := new(bytes.Buffer) + + err = binary.Write(b, byteOrder, cp.HorizontalRepeat) + log.PanicIf(err) + + err = binary.Write(b, byteOrder, cp.VerticalRepeat) + log.PanicIf(err) + + _, err = b.Write(cp.CfaValue) + log.PanicIf(err) + + encoded = b.Bytes() + + // TODO(dustin): Confirm this size against the specification. + + return encoded, uint32(len(encoded)), nil +} + +func (CodecA302CfaPattern) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test using known good data. + + valueContext.SetUndefinedValueType(exifcommon.TypeByte) + + valueBytes, err := valueContext.ReadBytes() + log.PanicIf(err) + + cp := TagA302CfaPattern{} + + cp.HorizontalRepeat = valueContext.ByteOrder().Uint16(valueBytes[0:2]) + cp.VerticalRepeat = valueContext.ByteOrder().Uint16(valueBytes[2:4]) + + expectedLength := int(cp.HorizontalRepeat * cp.VerticalRepeat) + cp.CfaValue = valueBytes[4 : 4+expectedLength] + + return cp, nil +} + +func init() { + registerEncoder( + TagA302CfaPattern{}, + CodecA302CfaPattern{}) + + registerDecoder( + exifcommon.IfdExifStandardIfdIdentity.UnindexedString(), + 0xa302, + CodecA302CfaPattern{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_iop_0002_interop_version.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_iop_0002_interop_version.go new file mode 100644 index 000000000..09ec98703 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/exif_iop_0002_interop_version.go @@ -0,0 +1,69 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag0002InteropVersion struct { + InteropVersion string +} + +func (Tag0002InteropVersion) EncoderName() string { + return "Codec0002InteropVersion" +} + +func (iv Tag0002InteropVersion) String() string { + return iv.InteropVersion +} + +type Codec0002InteropVersion struct { +} + +func (Codec0002InteropVersion) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s, ok := value.(Tag0002InteropVersion) + if ok == false { + log.Panicf("can only encode a Tag0002InteropVersion") + } + + return []byte(s.InteropVersion), uint32(len(s.InteropVersion)), nil +} + +func (Codec0002InteropVersion) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) + + valueString, err := valueContext.ReadAsciiNoNul() + log.PanicIf(err) + + iv := Tag0002InteropVersion{ + InteropVersion: valueString, + } + + return iv, nil +} + +func init() { + registerEncoder( + Tag0002InteropVersion{}, + Codec0002InteropVersion{}) + + registerDecoder( + exifcommon.IfdExifIopStandardIfdIdentity.UnindexedString(), + 0x0002, + Codec0002InteropVersion{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001B_gps_processing_method.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001B_gps_processing_method.go new file mode 100644 index 000000000..6f54d2fc6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001B_gps_processing_method.go @@ -0,0 +1,65 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag001BGPSProcessingMethod struct { + string +} + +func (Tag001BGPSProcessingMethod) EncoderName() string { + return "Codec001BGPSProcessingMethod" +} + +func (gpm Tag001BGPSProcessingMethod) String() string { + return gpm.string +} + +type Codec001BGPSProcessingMethod struct { +} + +func (Codec001BGPSProcessingMethod) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s, ok := value.(Tag001BGPSProcessingMethod) + if ok == false { + log.Panicf("can only encode a Tag001BGPSProcessingMethod") + } + + return []byte(s.string), uint32(len(s.string)), nil +} + +func (Codec001BGPSProcessingMethod) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) + + valueString, err := valueContext.ReadAsciiNoNul() + log.PanicIf(err) + + return Tag001BGPSProcessingMethod{valueString}, nil +} + +func init() { + registerEncoder( + Tag001BGPSProcessingMethod{}, + Codec001BGPSProcessingMethod{}) + + registerDecoder( + exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString(), + 0x001b, + Codec001BGPSProcessingMethod{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001C_gps_area_information.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001C_gps_area_information.go new file mode 100644 index 000000000..ffdeb905b --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/gps_001C_gps_area_information.go @@ -0,0 +1,65 @@ +package exifundefined + +import ( + "encoding/binary" + + "github.com/dsoprea/go-logging" + + "github.com/dsoprea/go-exif/v3/common" +) + +type Tag001CGPSAreaInformation struct { + string +} + +func (Tag001CGPSAreaInformation) EncoderName() string { + return "Codec001CGPSAreaInformation" +} + +func (gai Tag001CGPSAreaInformation) String() string { + return gai.string +} + +type Codec001CGPSAreaInformation struct { +} + +func (Codec001CGPSAreaInformation) Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s, ok := value.(Tag001CGPSAreaInformation) + if ok == false { + log.Panicf("can only encode a Tag001CGPSAreaInformation") + } + + return []byte(s.string), uint32(len(s.string)), nil +} + +func (Codec001CGPSAreaInformation) Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + valueContext.SetUndefinedValueType(exifcommon.TypeAsciiNoNul) + + valueString, err := valueContext.ReadAsciiNoNul() + log.PanicIf(err) + + return Tag001CGPSAreaInformation{valueString}, nil +} + +func init() { + registerEncoder( + Tag001CGPSAreaInformation{}, + Codec001CGPSAreaInformation{}) + + registerDecoder( + exifcommon.IfdGpsInfoStandardIfdIdentity.UnindexedString(), + 0x001c, + Codec001CGPSAreaInformation{}) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/registration.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/registration.go new file mode 100644 index 000000000..cccc20a82 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/registration.go @@ -0,0 +1,42 @@ +package exifundefined + +import ( + "github.com/dsoprea/go-logging" +) + +// UndefinedTagHandle defines one undefined-type tag with a corresponding +// decoder. +type UndefinedTagHandle struct { + IfdPath string + TagId uint16 +} + +func registerEncoder(entity EncodeableValue, encoder UndefinedValueEncoder) { + typeName := entity.EncoderName() + + _, found := encoders[typeName] + if found == true { + log.Panicf("encoder already registered: %v", typeName) + } + + encoders[typeName] = encoder +} + +func registerDecoder(ifdPath string, tagId uint16, decoder UndefinedValueDecoder) { + uth := UndefinedTagHandle{ + IfdPath: ifdPath, + TagId: tagId, + } + + _, found := decoders[uth] + if found == true { + log.Panicf("decoder already registered: %v", uth) + } + + decoders[uth] = decoder +} + +var ( + encoders = make(map[string]UndefinedValueEncoder) + decoders = make(map[UndefinedTagHandle]UndefinedValueDecoder) +) diff --git a/vendor/github.com/dsoprea/go-exif/v3/undefined/type.go b/vendor/github.com/dsoprea/go-exif/v3/undefined/type.go new file mode 100644 index 000000000..ff6ac2b4c --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/undefined/type.go @@ -0,0 +1,44 @@ +package exifundefined + +import ( + "errors" + + "encoding/binary" + + "github.com/dsoprea/go-exif/v3/common" +) + +const ( + // UnparseableUnknownTagValuePlaceholder is the string to use for an unknown + // undefined tag. + UnparseableUnknownTagValuePlaceholder = "!UNKNOWN" + + // UnparseableHandledTagValuePlaceholder is the string to use for a known + // value that is not parseable. + UnparseableHandledTagValuePlaceholder = "!MALFORMED" +) + +var ( + // ErrUnparseableValue is the error for a value that we should have been + // able to parse but were not able to. + ErrUnparseableValue = errors.New("unparseable undefined tag") +) + +// UndefinedValueEncoder knows how to encode an undefined-type tag's value to +// bytes. +type UndefinedValueEncoder interface { + Encode(value interface{}, byteOrder binary.ByteOrder) (encoded []byte, unitCount uint32, err error) +} + +// EncodeableValue wraps a value with the information that will be needed to re- +// encode it later. +type EncodeableValue interface { + EncoderName() string + String() string +} + +// UndefinedValueDecoder knows how to decode an undefined-type tag's value from +// bytes. +type UndefinedValueDecoder interface { + Decode(valueContext *exifcommon.ValueContext) (value EncodeableValue, err error) +} diff --git a/vendor/github.com/dsoprea/go-exif/v3/utility.go b/vendor/github.com/dsoprea/go-exif/v3/utility.go new file mode 100644 index 000000000..f0b5e6383 --- /dev/null +++ b/vendor/github.com/dsoprea/go-exif/v3/utility.go @@ -0,0 +1,237 @@ +package exif + +import ( + "fmt" + "io" + "math" + + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-utility/v2/filesystem" + + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-exif/v3/undefined" +) + +var ( + utilityLogger = log.NewLogger("exif.utility") +) + +// ExifTag is one simple representation of a tag in a flat list of all of them. +type ExifTag struct { + // IfdPath is the fully-qualified IFD path (even though it is not named as + // such). + IfdPath string `json:"ifd_path"` + + // TagId is the tag-ID. + TagId uint16 `json:"id"` + + // TagName is the tag-name. This is never empty. + TagName string `json:"name"` + + // UnitCount is the recorded number of units constution of the value. + UnitCount uint32 `json:"unit_count"` + + // TagTypeId is the type-ID. + TagTypeId exifcommon.TagTypePrimitive `json:"type_id"` + + // TagTypeName is the type name. + TagTypeName string `json:"type_name"` + + // Value is the decoded value. + Value interface{} `json:"value"` + + // ValueBytes is the raw, encoded value. + ValueBytes []byte `json:"value_bytes"` + + // Formatted is the human representation of the first value (tag values are + // always an array). + FormattedFirst string `json:"formatted_first"` + + // Formatted is the human representation of the complete value. + Formatted string `json:"formatted"` + + // ChildIfdPath is the name of the child IFD this tag represents (if it + // represents any). Otherwise, this is empty. + ChildIfdPath string `json:"child_ifd_path"` +} + +// String returns a string representation. +func (et ExifTag) String() string { + return fmt.Sprintf( + "ExifTag<"+ + "IFD-PATH=[%s] "+ + "TAG-ID=(0x%02x) "+ + "TAG-NAME=[%s] "+ + "TAG-TYPE=[%s] "+ + "VALUE=[%v] "+ + "VALUE-BYTES=(%d) "+ + "CHILD-IFD-PATH=[%s]", + et.IfdPath, et.TagId, et.TagName, et.TagTypeName, et.FormattedFirst, + len(et.ValueBytes), et.ChildIfdPath) +} + +// GetFlatExifData returns a simple, flat representation of all tags. +func GetFlatExifData(exifData []byte, so *ScanOptions) (exifTags []ExifTag, med *MiscellaneousExifData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + sb := rifs.NewSeekableBufferWithBytes(exifData) + + exifTags, med, err = getFlatExifDataUniversalSearchWithReadSeeker(sb, so, false) + log.PanicIf(err) + + return exifTags, med, nil +} + +// RELEASE(dustin): GetFlatExifDataUniversalSearch is a kludge to allow univeral tag searching in a backwards-compatible manner. For the next release, undo this and simply add the flag to GetFlatExifData. + +// GetFlatExifDataUniversalSearch returns a simple, flat representation of all +// tags. +func GetFlatExifDataUniversalSearch(exifData []byte, so *ScanOptions, doUniversalSearch bool) (exifTags []ExifTag, med *MiscellaneousExifData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + sb := rifs.NewSeekableBufferWithBytes(exifData) + + exifTags, med, err = getFlatExifDataUniversalSearchWithReadSeeker(sb, so, doUniversalSearch) + log.PanicIf(err) + + return exifTags, med, nil +} + +// RELEASE(dustin): GetFlatExifDataUniversalSearchWithReadSeeker is a kludge to allow using a ReadSeeker in a backwards-compatible manner. For the next release, drop this and refactor GetFlatExifDataUniversalSearch to take a ReadSeeker. + +// GetFlatExifDataUniversalSearchWithReadSeeker returns a simple, flat +// representation of all tags given a ReadSeeker. +func GetFlatExifDataUniversalSearchWithReadSeeker(rs io.ReadSeeker, so *ScanOptions, doUniversalSearch bool) (exifTags []ExifTag, med *MiscellaneousExifData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + exifTags, med, err = getFlatExifDataUniversalSearchWithReadSeeker(rs, so, doUniversalSearch) + log.PanicIf(err) + + return exifTags, med, nil +} + +// getFlatExifDataUniversalSearchWithReadSeeker returns a simple, flat +// representation of all tags given a ReadSeeker. +func getFlatExifDataUniversalSearchWithReadSeeker(rs io.ReadSeeker, so *ScanOptions, doUniversalSearch bool) (exifTags []ExifTag, med *MiscellaneousExifData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + headerData := make([]byte, ExifSignatureLength) + if _, err = io.ReadFull(rs, headerData); err != nil { + if err == io.EOF { + return nil, nil, err + } + + log.Panic(err) + } + + eh, err := ParseExifHeader(headerData) + log.PanicIf(err) + + im, err := exifcommon.NewIfdMappingWithStandard() + log.PanicIf(err) + + ti := NewTagIndex() + + if doUniversalSearch == true { + ti.SetUniversalSearch(true) + } + + ebs := NewExifReadSeeker(rs) + ie := NewIfdEnumerate(im, ti, ebs, eh.ByteOrder) + + exifTags = make([]ExifTag, 0) + + visitor := func(ite *IfdTagEntry) (err error) { + // This encodes down to base64. Since this an example tool and we do not + // expect to ever decode the output, we are not worried about + // specifically base64-encoding it in order to have a measure of + // control. + valueBytes, err := ite.GetRawBytes() + if err != nil { + if err == exifundefined.ErrUnparseableValue { + return nil + } + + log.Panic(err) + } + + value, err := ite.Value() + if err != nil { + if err == exifcommon.ErrUnhandledUndefinedTypedTag { + value = exifundefined.UnparseableUnknownTagValuePlaceholder + } else if log.Is(err, exifcommon.ErrParseFail) == true { + utilityLogger.Warningf(nil, + "Could not parse value for tag [%s] (%04x) [%s].", + ite.IfdPath(), ite.TagId(), ite.TagName()) + + return nil + } else { + log.Panic(err) + } + } + + et := ExifTag{ + IfdPath: ite.IfdPath(), + TagId: ite.TagId(), + TagName: ite.TagName(), + UnitCount: ite.UnitCount(), + TagTypeId: ite.TagType(), + TagTypeName: ite.TagType().String(), + Value: value, + ValueBytes: valueBytes, + ChildIfdPath: ite.ChildIfdPath(), + } + + et.Formatted, err = ite.Format() + log.PanicIf(err) + + et.FormattedFirst, err = ite.FormatFirst() + log.PanicIf(err) + + exifTags = append(exifTags, et) + + return nil + } + + med, err = ie.Scan(exifcommon.IfdStandardIfdIdentity, eh.FirstIfdOffset, visitor, nil) + log.PanicIf(err) + + return exifTags, med, nil +} + +// GpsDegreesEquals returns true if the two `GpsDegrees` are identical. +func GpsDegreesEquals(gi1, gi2 GpsDegrees) bool { + if gi2.Orientation != gi1.Orientation { + return false + } + + degreesRightBound := math.Nextafter(gi1.Degrees, gi1.Degrees+1) + minutesRightBound := math.Nextafter(gi1.Minutes, gi1.Minutes+1) + secondsRightBound := math.Nextafter(gi1.Seconds, gi1.Seconds+1) + + if gi2.Degrees < gi1.Degrees || gi2.Degrees >= degreesRightBound { + return false + } else if gi2.Minutes < gi1.Minutes || gi2.Minutes >= minutesRightBound { + return false + } else if gi2.Seconds < gi1.Seconds || gi2.Seconds >= secondsRightBound { + return false + } + + return true +} diff --git a/vendor/github.com/dsoprea/go-exif/value_context.go b/vendor/github.com/dsoprea/go-exif/value_context.go deleted file mode 100644 index 3fce352a3..000000000 --- a/vendor/github.com/dsoprea/go-exif/value_context.go +++ /dev/null @@ -1,367 +0,0 @@ -package exif - -import ( - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -var ( - parser *Parser -) - -// ValueContext describes all of the parameters required to find and extract -// the actual tag value. -type ValueContext struct { - unitCount uint32 - valueOffset uint32 - rawValueOffset []byte - addressableData []byte - - tagType TagTypePrimitive - byteOrder binary.ByteOrder - - // undefinedValueTagType is the effective type to use if this is an - // "undefined" value. - undefinedValueTagType TagTypePrimitive - - ifdPath string - tagId uint16 -} - -func newValueContext(ifdPath string, tagId uint16, unitCount, valueOffset uint32, rawValueOffset, addressableData []byte, tagType TagTypePrimitive, byteOrder binary.ByteOrder) *ValueContext { - return &ValueContext{ - unitCount: unitCount, - valueOffset: valueOffset, - rawValueOffset: rawValueOffset, - addressableData: addressableData, - - tagType: tagType, - byteOrder: byteOrder, - - ifdPath: ifdPath, - tagId: tagId, - } -} - -func newValueContextFromTag(ite *IfdTagEntry, addressableData []byte, byteOrder binary.ByteOrder) *ValueContext { - return newValueContext( - ite.IfdPath, - ite.TagId, - ite.UnitCount, - ite.ValueOffset, - ite.RawValueOffset, - addressableData, - ite.TagType, - byteOrder) -} - -func (vc *ValueContext) SetUnknownValueType(tagType TagTypePrimitive) { - vc.undefinedValueTagType = tagType -} - -func (vc *ValueContext) UnitCount() uint32 { - return vc.unitCount -} - -func (vc *ValueContext) ValueOffset() uint32 { - return vc.valueOffset -} - -func (vc *ValueContext) RawValueOffset() []byte { - return vc.rawValueOffset -} - -func (vc *ValueContext) AddressableData() []byte { - return vc.addressableData -} - -// isEmbedded returns whether the value is embedded or a reference. This can't -// be precalculated since the size is not defined for all types (namely the -// "undefined" types). -func (vc *ValueContext) isEmbedded() bool { - tagType := vc.effectiveValueType() - - return (tagType.Size() * int(vc.unitCount)) <= 4 -} - -func (vc *ValueContext) effectiveValueType() (tagType TagTypePrimitive) { - if vc.tagType == TypeUndefined { - tagType = vc.undefinedValueTagType - - if tagType == 0 { - log.Panicf("undefined-value type not set") - } - } else { - tagType = vc.tagType - } - - return tagType -} - -func (vc *ValueContext) readRawEncoded() (rawBytes []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - tagType := vc.effectiveValueType() - - unitSizeRaw := uint32(tagType.Size()) - - if vc.isEmbedded() == true { - byteLength := unitSizeRaw * vc.unitCount - return vc.rawValueOffset[:byteLength], nil - } else { - return vc.addressableData[vc.valueOffset : vc.valueOffset+vc.unitCount*unitSizeRaw], nil - } -} - -// Format returns a string representation for the value. -// -// Where the type is not ASCII, `justFirst` indicates whether to just stringify -// the first item in the slice (or return an empty string if the slice is -// empty). -// -// Since this method lacks the information to process undefined-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (vc *ValueContext) Format() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawBytes, err := vc.readRawEncoded() - log.PanicIf(err) - - phrase, err := Format(rawBytes, vc.tagType, false, vc.byteOrder) - log.PanicIf(err) - - return phrase, nil -} - -// FormatOne is similar to `Format` but only gets and stringifies the first -// item. -func (vc *ValueContext) FormatFirst() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawBytes, err := vc.readRawEncoded() - log.PanicIf(err) - - phrase, err := Format(rawBytes, vc.tagType, true, vc.byteOrder) - log.PanicIf(err) - - return phrase, nil -} - -func (vc *ValueContext) ReadBytes() (value []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseBytes(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadAscii() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseAscii(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadAsciiNoNul() (value string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseAsciiNoNul(rawValue, vc.unitCount) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadShorts() (value []uint16, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseShorts(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadLongs() (value []uint32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseLongs(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadRationals() (value []Rational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseRationals(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadSignedLongs() (value []int32, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseSignedLongs(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -func (vc *ValueContext) ReadSignedRationals() (value []SignedRational, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rawValue, err := vc.readRawEncoded() - log.PanicIf(err) - - value, err = parser.ParseSignedRationals(rawValue, vc.unitCount, vc.byteOrder) - log.PanicIf(err) - - return value, nil -} - -// Values knows how to resolve the given value. This value is always a list -// (undefined-values aside), so we're named accordingly. -// -// Since this method lacks the information to process unknown-type tags (e.g. -// byte-order, tag-ID, IFD type), it will return an error if attempted. See -// `Undefined()`. -func (vc *ValueContext) Values() (values interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if vc.tagType == TypeByte { - values, err = vc.ReadBytes() - log.PanicIf(err) - } else if vc.tagType == TypeAscii { - values, err = vc.ReadAscii() - log.PanicIf(err) - } else if vc.tagType == TypeAsciiNoNul { - values, err = vc.ReadAsciiNoNul() - log.PanicIf(err) - } else if vc.tagType == TypeShort { - values, err = vc.ReadShorts() - log.PanicIf(err) - } else if vc.tagType == TypeLong { - values, err = vc.ReadLongs() - log.PanicIf(err) - } else if vc.tagType == TypeRational { - values, err = vc.ReadRationals() - log.PanicIf(err) - } else if vc.tagType == TypeSignedLong { - values, err = vc.ReadSignedLongs() - log.PanicIf(err) - } else if vc.tagType == TypeSignedRational { - values, err = vc.ReadSignedRationals() - log.PanicIf(err) - } else if vc.tagType == TypeUndefined { - log.Panicf("will not parse undefined-type value") - - // Never called. - return nil, nil - } else { - log.Panicf("value of type [%s] is unparseable", vc.tagType) - - // Never called. - return nil, nil - } - - return values, nil -} - -// Undefined attempts to identify and decode supported undefined-type fields. -// This is the primary, preferred interface to reading undefined values. -func (vc *ValueContext) Undefined() (value interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - value, err = UndefinedValue(vc.ifdPath, vc.tagId, vc, vc.byteOrder) - if err != nil { - if err == ErrUnhandledUnknownTypedTag { - return nil, err - } - - log.Panic(err) - } - - return value, nil -} - -func init() { - parser = &Parser{} -} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/.MODULE_ROOT b/vendor/github.com/dsoprea/go-jpeg-image-structure/.MODULE_ROOT deleted file mode 100644 index e69de29bb..000000000 diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/.travis.yml b/vendor/github.com/dsoprea/go-jpeg-image-structure/.travis.yml deleted file mode 100644 index 4c79875ed..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -go: - - master - - stable - - "1.14" - - "1.13" - - "1.12" -env: - - GO111MODULE=on -install: - - go get -t ./... -script: -# v1 - - go test -v . -# v2 - - cd v2 - - go test -v ./... -coverprofile=coverage.txt -covermode=atomic - - cd .. -after_success: - - cd v2 - - curl -s https://codecov.io/bash | bash diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/LICENSE b/vendor/github.com/dsoprea/go-jpeg-image-structure/LICENSE deleted file mode 100644 index 163291ed6..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT LICENSE - -Copyright 2020 Dustin Oprea - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/README.md b/vendor/github.com/dsoprea/go-jpeg-image-structure/README.md deleted file mode 100644 index 67bc57617..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/README.md +++ /dev/null @@ -1,10 +0,0 @@ -[![Build Status](https://travis-ci.org/dsoprea/go-jpeg-image-structure.svg?branch=master)](https://travis-ci.org/dsoprea/go-jpeg-image-structure) -[![codecov](https://codecov.io/gh/dsoprea/go-jpeg-image-structure/branch/master/graph/badge.svg?token=Twxyx7kpAa)](https://codecov.io/gh/dsoprea/go-jpeg-image-structure) -[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-jpeg-image-structure/v2)](https://goreportcard.com/report/github.com/dsoprea/go-jpeg-image-structure/v2) -[![GoDoc](https://godoc.org/github.com/dsoprea/go-jpeg-image-structure/v2?status.svg)](https://godoc.org/github.com/dsoprea/go-jpeg-image-structure/v2) - -## Overview - -Parse raw JPEG data into individual segments of data. You can print or export this data, including hash digests for each. You can also parse/modify the EXIF data and write an updated image. - -EXIF, XMP, and IPTC data can also be extracted. The provided CLI tool can print this data as well. diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/markers.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/markers.go deleted file mode 100644 index a12171bd8..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/markers.go +++ /dev/null @@ -1,212 +0,0 @@ -package jpegstructure - -import ( - "github.com/dsoprea/go-logging" -) - -const ( - // MARKER_SOI marker - MARKER_SOI = 0xd8 - - // MARKER_EOI marker - MARKER_EOI = 0xd9 - - // MARKER_SOS marker - MARKER_SOS = 0xda - - // MARKER_SOD marker - MARKER_SOD = 0x93 - - // MARKER_DQT marker - MARKER_DQT = 0xdb - - // MARKER_APP0 marker - MARKER_APP0 = 0xe0 - - // MARKER_APP1 marker - MARKER_APP1 = 0xe1 - - // MARKER_APP2 marker - MARKER_APP2 = 0xe2 - - // MARKER_APP3 marker - MARKER_APP3 = 0xe3 - - // MARKER_APP4 marker - MARKER_APP4 = 0xe4 - - // MARKER_APP5 marker - MARKER_APP5 = 0xe5 - - // MARKER_APP6 marker - MARKER_APP6 = 0xe6 - - // MARKER_APP7 marker - MARKER_APP7 = 0xe7 - - // MARKER_APP8 marker - MARKER_APP8 = 0xe8 - - // MARKER_APP10 marker - MARKER_APP10 = 0xea - - // MARKER_APP12 marker - MARKER_APP12 = 0xec - - // MARKER_APP13 marker - MARKER_APP13 = 0xed - - // MARKER_APP14 marker - MARKER_APP14 = 0xee - - // MARKER_APP15 marker - MARKER_APP15 = 0xef - - // MARKER_COM marker - MARKER_COM = 0xfe - - // MARKER_CME marker - MARKER_CME = 0x64 - - // MARKER_SIZ marker - MARKER_SIZ = 0x51 - - // MARKER_DHT marker - MARKER_DHT = 0xc4 - - // MARKER_JPG marker - MARKER_JPG = 0xc8 - - // MARKER_DAC marker - MARKER_DAC = 0xcc - - // MARKER_SOF0 marker - MARKER_SOF0 = 0xc0 - - // MARKER_SOF1 marker - MARKER_SOF1 = 0xc1 - - // MARKER_SOF2 marker - MARKER_SOF2 = 0xc2 - - // MARKER_SOF3 marker - MARKER_SOF3 = 0xc3 - - // MARKER_SOF5 marker - MARKER_SOF5 = 0xc5 - - // MARKER_SOF6 marker - MARKER_SOF6 = 0xc6 - - // MARKER_SOF7 marker - MARKER_SOF7 = 0xc7 - - // MARKER_SOF9 marker - MARKER_SOF9 = 0xc9 - - // MARKER_SOF10 marker - MARKER_SOF10 = 0xca - - // MARKER_SOF11 marker - MARKER_SOF11 = 0xcb - - // MARKER_SOF13 marker - MARKER_SOF13 = 0xcd - - // MARKER_SOF14 marker - MARKER_SOF14 = 0xce - - // MARKER_SOF15 marker - MARKER_SOF15 = 0xcf -) - -var ( - jpegLogger = log.NewLogger("jpegstructure.jpeg") - jpegMagicStandard = []byte{0xff, MARKER_SOI, 0xff} - jpegMagic2000 = []byte{0xff, 0x4f, 0xff} - - markerLen = map[byte]int{ - 0x00: 0, - 0x01: 0, - 0xd0: 0, - 0xd1: 0, - 0xd2: 0, - 0xd3: 0, - 0xd4: 0, - 0xd5: 0, - 0xd6: 0, - 0xd7: 0, - 0xd8: 0, - 0xd9: 0, - 0xda: 0, - - // J2C - 0x30: 0, - 0x31: 0, - 0x32: 0, - 0x33: 0, - 0x34: 0, - 0x35: 0, - 0x36: 0, - 0x37: 0, - 0x38: 0, - 0x39: 0, - 0x3a: 0, - 0x3b: 0, - 0x3c: 0, - 0x3d: 0, - 0x3e: 0, - 0x3f: 0, - 0x4f: 0, - 0x92: 0, - 0x93: 0, - - // J2C extensions - 0x74: 4, - 0x75: 4, - 0x77: 4, - } - - markerNames = map[byte]string{ - MARKER_SOI: "SOI", - MARKER_EOI: "EOI", - MARKER_SOS: "SOS", - MARKER_SOD: "SOD", - MARKER_DQT: "DQT", - MARKER_APP0: "APP0", - MARKER_APP1: "APP1", - MARKER_APP2: "APP2", - MARKER_APP3: "APP3", - MARKER_APP4: "APP4", - MARKER_APP5: "APP5", - MARKER_APP6: "APP6", - MARKER_APP7: "APP7", - MARKER_APP8: "APP8", - MARKER_APP10: "APP10", - MARKER_APP12: "APP12", - MARKER_APP13: "APP13", - MARKER_APP14: "APP14", - MARKER_APP15: "APP15", - MARKER_COM: "COM", - MARKER_CME: "CME", - MARKER_SIZ: "SIZ", - - MARKER_DHT: "DHT", - MARKER_JPG: "JPG", - MARKER_DAC: "DAC", - - MARKER_SOF0: "SOF0", - MARKER_SOF1: "SOF1", - MARKER_SOF2: "SOF2", - MARKER_SOF3: "SOF3", - MARKER_SOF5: "SOF5", - MARKER_SOF6: "SOF6", - MARKER_SOF7: "SOF7", - MARKER_SOF9: "SOF9", - MARKER_SOF10: "SOF10", - MARKER_SOF11: "SOF11", - MARKER_SOF13: "SOF13", - MARKER_SOF14: "SOF14", - MARKER_SOF15: "SOF15", - } -) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/media_parser.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/media_parser.go deleted file mode 100644 index dd4c73af9..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/media_parser.go +++ /dev/null @@ -1,128 +0,0 @@ -package jpegstructure - -import ( - "bufio" - "bytes" - "io" - "os" - - "github.com/dsoprea/go-logging" - "github.com/dsoprea/go-utility/image" -) - -// JpegMediaParser is a `riimage.MediaParser` that knows how to parse JPEG -// images. -type JpegMediaParser struct { -} - -// NewJpegMediaParser returns a new JpegMediaParser. -func NewJpegMediaParser() *JpegMediaParser { - - // TODO(dustin): Add test - - return new(JpegMediaParser) -} - -// Parse parses a JPEG uses an `io.ReadSeeker`. Even if it fails, it will return -// the list of segments encountered prior to the failure. -func (jmp *JpegMediaParser) Parse(rs io.ReadSeeker, size int) (ec riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - s := bufio.NewScanner(rs) - - // Since each segment can be any size, our buffer must allowed to grow as - // large as the file. - buffer := []byte{} - s.Buffer(buffer, size) - - js := NewJpegSplitter(nil) - s.Split(js.Split) - - for s.Scan() != false { - } - - // Always return the segments that were parsed, at least until there was an - // error. - ec = js.Segments() - - log.PanicIf(s.Err()) - - return ec, nil -} - -// ParseFile parses a JPEG file. Even if it fails, it will return the list of -// segments encountered prior to the failure. -func (jmp *JpegMediaParser) ParseFile(filepath string) (ec riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - f, err := os.Open(filepath) - log.PanicIf(err) - - defer f.Close() - - stat, err := f.Stat() - log.PanicIf(err) - - size := stat.Size() - - sl, err := jmp.Parse(f, int(size)) - - // Always return the segments that were parsed, at least until there was an - // error. - ec = sl - - log.PanicIf(err) - - return ec, nil -} - -// ParseBytes parses a JPEG byte-slice. Even if it fails, it will return the -// list of segments encountered prior to the failure. -func (jmp *JpegMediaParser) ParseBytes(data []byte) (ec riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - br := bytes.NewReader(data) - - sl, err := jmp.Parse(br, len(data)) - - // Always return the segments that were parsed, at least until there was an - // error. - ec = sl - - log.PanicIf(err) - - return ec, nil -} - -// LooksLikeFormat indicates whether the data looks like a JPEG image. -func (jmp *JpegMediaParser) LooksLikeFormat(data []byte) bool { - if len(data) < 4 { - return false - } - - l := len(data) - if data[0] != 0xff || data[1] != MARKER_SOI || data[l-2] != 0xff || data[l-1] != MARKER_EOI { - return false - } - - return true -} - -var ( - // Enforce interface conformance. - _ riimage.MediaParser = new(JpegMediaParser) -) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/segment.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/segment.go deleted file mode 100644 index d6a1c42bb..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/segment.go +++ /dev/null @@ -1,349 +0,0 @@ -package jpegstructure - -import ( - "bytes" - "errors" - "fmt" - - "crypto/sha1" - "encoding/hex" - - "github.com/dsoprea/go-exif/v2" - "github.com/dsoprea/go-iptc" - "github.com/dsoprea/go-logging" - "github.com/dsoprea/go-photoshop-info-format" - "github.com/dsoprea/go-utility/image" -) - -const ( - pirIptcImageResourceId = uint16(0x0404) -) - -var ( - // exifPrefix is the prefix found at the top of an EXIF slice. This is JPEG- - // specific. - exifPrefix = []byte{'E', 'x', 'i', 'f', 0, 0} - - xmpPrefix = []byte("http://ns.adobe.com/xap/1.0/\000") - - ps30Prefix = []byte("Photoshop 3.0\000") -) - -var ( - // ErrNoXmp is returned if XMP data was requested but not found. - ErrNoXmp = errors.New("no XMP data") - - // ErrNoIptc is returned if IPTC data was requested but not found. - ErrNoIptc = errors.New("no IPTC data") - - // ErrNoPhotoshopData is returned if Photoshop info was requested but not - // found. - ErrNoPhotoshopData = errors.New("no photoshop data") -) - -// SofSegment has info read from a SOF segment. -type SofSegment struct { - // BitsPerSample is the bits-per-sample. - BitsPerSample byte - - // Width is the image width. - Width uint16 - - // Height is the image height. - Height uint16 - - // ComponentCount is the number of color components. - ComponentCount byte -} - -// String returns a string representation of the SOF segment. -func (ss SofSegment) String() string { - - // TODO(dustin): Add test - - return fmt.Sprintf("SOF", ss.BitsPerSample, ss.Width, ss.Height, ss.ComponentCount) -} - -// SegmentVisitor describes a segment-visitor struct. -type SegmentVisitor interface { - // HandleSegment is triggered for each segment encountered as well as the - // scan-data. - HandleSegment(markerId byte, markerName string, counter int, lastIsScanData bool) error -} - -// SofSegmentVisitor describes a visitor that is only called for each SOF -// segment. -type SofSegmentVisitor interface { - // HandleSof is called for each encountered SOF segment. - HandleSof(sof *SofSegment) error -} - -// Segment describes a single segment. -type Segment struct { - MarkerId byte - MarkerName string - Offset int - Data []byte - - photoshopInfo map[uint16]photoshopinfo.Photoshop30InfoRecord - iptcTags map[iptc.StreamTagKey][]iptc.TagData -} - -// SetExif encodes and sets EXIF data into this segment. -func (s *Segment) SetExif(ib *exif.IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - ibe := exif.NewIfdByteEncoder() - - exifData, err := ibe.EncodeToExif(ib) - log.PanicIf(err) - - l := len(exifPrefix) - - s.Data = make([]byte, l+len(exifData)) - copy(s.Data[0:], exifPrefix) - copy(s.Data[l:], exifData) - - return nil -} - -// Exif returns an `exif.Ifd` instance for the EXIF data we currently have. -func (s *Segment) Exif() (rootIfd *exif.Ifd, data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - l := len(exifPrefix) - - rawExif := s.Data[l:] - - jpegLogger.Debugf(nil, "Attempting to parse (%d) byte EXIF blob (Exif).", len(rawExif)) - - im := exif.NewIfdMappingWithStandard() - ti := exif.NewTagIndex() - - _, index, err := exif.Collect(im, ti, rawExif) - log.PanicIf(err) - - return index.RootIfd, rawExif, nil -} - -// FlatExif parses the EXIF data and just returns a list of tags. -func (s *Segment) FlatExif() (exifTags []exif.ExifTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - l := len(exifPrefix) - - rawExif := s.Data[l:] - - jpegLogger.Debugf(nil, "Attempting to parse (%d) byte EXIF blob (FlatExif).", len(rawExif)) - - exifTags, err = exif.GetFlatExifData(rawExif) - log.PanicIf(err) - - return exifTags, nil -} - -// EmbeddedString returns a string of properties that can be embedded into an -// longer string of properties. -func (s *Segment) EmbeddedString() string { - h := sha1.New() - h.Write(s.Data) - - // TODO(dustin): Add test - - digestString := hex.EncodeToString(h.Sum(nil)) - - return fmt.Sprintf("OFFSET=(0x%08x %10d) ID=(0x%02x) NAME=[%-5s] SIZE=(%10d) SHA1=[%s]", s.Offset, s.Offset, s.MarkerId, markerNames[s.MarkerId], len(s.Data), digestString) -} - -// String returns a descriptive string. -func (s *Segment) String() string { - - // TODO(dustin): Add test - - return fmt.Sprintf("Segment<%s>", s.EmbeddedString()) -} - -// IsExif returns true if EXIF data. -func (s *Segment) IsExif() bool { - if s.MarkerId != MARKER_APP1 { - return false - } - - // TODO(dustin): Add test - - l := len(exifPrefix) - - if len(s.Data) < l { - return false - } - - if bytes.Equal(s.Data[:l], exifPrefix) == false { - return false - } - - return true -} - -// IsXmp returns true if XMP data. -func (s *Segment) IsXmp() bool { - if s.MarkerId != MARKER_APP1 { - return false - } - - // TODO(dustin): Add test - - l := len(xmpPrefix) - - if len(s.Data) < l { - return false - } - - if bytes.Equal(s.Data[:l], xmpPrefix) == false { - return false - } - - return true -} - -// FormattedXmp returns a formatted XML string. This only makes sense for a -// segment comprised of XML data (like XMP). -func (s *Segment) FormattedXmp() (formatted string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - if s.IsXmp() != true { - log.Panicf("not an XMP segment") - } - - l := len(xmpPrefix) - - raw := string(s.Data[l:]) - - formatted, err = FormatXml(raw) - log.PanicIf(err) - - return formatted, nil -} - -func (s *Segment) parsePhotoshopInfo() (photoshopInfo map[uint16]photoshopinfo.Photoshop30InfoRecord, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if s.photoshopInfo != nil { - return s.photoshopInfo, nil - } - - if s.MarkerId != MARKER_APP13 { - return nil, ErrNoPhotoshopData - } - - l := len(ps30Prefix) - - if len(s.Data) < l { - return nil, ErrNoPhotoshopData - } - - if bytes.Equal(s.Data[:l], ps30Prefix) == false { - return nil, ErrNoPhotoshopData - } - - data := s.Data[l:] - b := bytes.NewBuffer(data) - - // Parse it. - - pirIndex, err := photoshopinfo.ReadPhotoshop30Info(b) - log.PanicIf(err) - - s.photoshopInfo = pirIndex - - return s.photoshopInfo, nil -} - -// IsIptc returns true if XMP data. -func (s *Segment) IsIptc() bool { - // TODO(dustin): Add test - - // There's a cost to determining if there's IPTC data, so we won't do it - // more than once. - if s.iptcTags != nil { - return true - } - - photoshopInfo, err := s.parsePhotoshopInfo() - if err != nil { - if err == ErrNoPhotoshopData { - return false - } - - log.Panic(err) - } - - // Bail if the Photoshop info doesn't have IPTC data. - - _, found := photoshopInfo[pirIptcImageResourceId] - if found == false { - return false - } - - return true -} - -// Iptc parses Photoshop info (if present) and then parses the IPTC info inside -// it (if present). -func (s *Segment) Iptc() (tags map[iptc.StreamTagKey][]iptc.TagData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Cache the parse. - if s.iptcTags != nil { - return s.iptcTags, nil - } - - photoshopInfo, err := s.parsePhotoshopInfo() - log.PanicIf(err) - - iptcPir, found := photoshopInfo[pirIptcImageResourceId] - if found == false { - return nil, ErrNoIptc - } - - b := bytes.NewBuffer(iptcPir.Data) - - tags, err = iptc.ParseStream(b) - log.PanicIf(err) - - s.iptcTags = tags - - return tags, nil -} - -var ( - // Enforce interface conformance. - _ riimage.MediaContext = new(Segment) -) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/segment_list.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/segment_list.go deleted file mode 100644 index 1b5b06a3a..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/segment_list.go +++ /dev/null @@ -1,395 +0,0 @@ -package jpegstructure - -import ( - "bytes" - "fmt" - "io" - - "crypto/sha1" - "encoding/binary" - - "github.com/dsoprea/go-exif/v2" - "github.com/dsoprea/go-iptc" - "github.com/dsoprea/go-logging" -) - -// SegmentList contains a slice of segments. -type SegmentList struct { - segments []*Segment -} - -// NewSegmentList returns a new SegmentList struct. -func NewSegmentList(segments []*Segment) (sl *SegmentList) { - if segments == nil { - segments = make([]*Segment, 0) - } - - return &SegmentList{ - segments: segments, - } -} - -// OffsetsEqual returns true is all segments have the same marker-IDs and were -// found at the same offsets. -func (sl *SegmentList) OffsetsEqual(o *SegmentList) bool { - if len(o.segments) != len(sl.segments) { - return false - } - - for i, s := range o.segments { - if s.MarkerId != sl.segments[i].MarkerId || s.Offset != sl.segments[i].Offset { - return false - } - } - - return true -} - -// Segments returns the underlying slice of segments. -func (sl *SegmentList) Segments() []*Segment { - return sl.segments -} - -// Add adds another segment. -func (sl *SegmentList) Add(s *Segment) { - sl.segments = append(sl.segments, s) -} - -// Print prints segment info. -func (sl *SegmentList) Print() { - if len(sl.segments) == 0 { - fmt.Printf("No segments.\n") - } else { - exifIndex, _, err := sl.FindExif() - if err != nil { - if err == exif.ErrNoExif { - exifIndex = -1 - } else { - log.Panic(err) - } - } - - xmpIndex, _, err := sl.FindXmp() - if err != nil { - if err == ErrNoXmp { - xmpIndex = -1 - } else { - log.Panic(err) - } - } - - iptcIndex, _, err := sl.FindIptc() - if err != nil { - if err == ErrNoIptc { - iptcIndex = -1 - } else { - log.Panic(err) - } - } - - for i, s := range sl.segments { - fmt.Printf("%2d: %s", i, s.EmbeddedString()) - - if i == exifIndex { - fmt.Printf(" [EXIF]") - } else if i == xmpIndex { - fmt.Printf(" [XMP]") - } else if i == iptcIndex { - fmt.Printf(" [IPTC]") - } - - fmt.Printf("\n") - } - } -} - -// Validate checks that all of the markers are actually located at all of the -// recorded offsets. -func (sl *SegmentList) Validate(data []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(sl.segments) < 2 { - log.Panicf("minimum segments not found") - } - - if sl.segments[0].MarkerId != MARKER_SOI { - log.Panicf("first segment not SOI") - } else if sl.segments[len(sl.segments)-1].MarkerId != MARKER_EOI { - log.Panicf("last segment not EOI") - } - - lastOffset := 0 - for i, s := range sl.segments { - if lastOffset != 0 && s.Offset <= lastOffset { - log.Panicf("segment offset not greater than the last: SEGMENT=(%d) (0x%08x) <= (0x%08x)", i, s.Offset, lastOffset) - } - - // The scan-data doesn't start with a marker. - if s.MarkerId == 0x0 { - continue - } - - o := s.Offset - if bytes.Compare(data[o:o+2], []byte{0xff, s.MarkerId}) != 0 { - log.Panicf("segment offset does not point to the start of a segment: SEGMENT=(%d) (0x%08x)", i, s.Offset) - } - - lastOffset = o - } - - return nil -} - -// FindExif returns the the segment that hosts the EXIF data (if present). -func (sl *SegmentList) FindExif() (index int, segment *Segment, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for i, s := range sl.segments { - if s.IsExif() == true { - return i, s, nil - } - } - - return -1, nil, exif.ErrNoExif -} - -// FindXmp returns the the segment that hosts the XMP data (if present). -func (sl *SegmentList) FindXmp() (index int, segment *Segment, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for i, s := range sl.segments { - if s.IsXmp() == true { - return i, s, nil - } - } - - return -1, nil, ErrNoXmp -} - -// FindIptc returns the the segment that hosts the IPTC data (if present). -func (sl *SegmentList) FindIptc() (index int, segment *Segment, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for i, s := range sl.segments { - if s.IsIptc() == true { - return i, s, nil - } - } - - return -1, nil, ErrNoIptc -} - -// Exif returns an `exif.Ifd` instance for the EXIF data we currently have. -func (sl *SegmentList) Exif() (rootIfd *exif.Ifd, rawExif []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - _, s, err := sl.FindExif() - log.PanicIf(err) - - rootIfd, rawExif, err = s.Exif() - log.PanicIf(err) - - return rootIfd, rawExif, nil -} - -// Iptc returns embedded IPTC data if present. -func (sl *SegmentList) Iptc() (tags map[iptc.StreamTagKey][]iptc.TagData, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add comment and return data. - - _, s, err := sl.FindIptc() - log.PanicIf(err) - - tags, err = s.Iptc() - log.PanicIf(err) - - return tags, nil -} - -// ConstructExifBuilder returns an `exif.IfdBuilder` instance (needed for -// modifying) preloaded with all existing tags. -func (sl *SegmentList) ConstructExifBuilder() (rootIb *exif.IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rootIfd, _, err := sl.Exif() - log.PanicIf(err) - - ib := exif.NewIfdBuilderFromExistingChain(rootIfd) - - return ib, nil -} - -// DumpExif returns an unstructured list of tags (useful when just reviewing). -func (sl *SegmentList) DumpExif() (segmentIndex int, segment *Segment, exifTags []exif.ExifTag, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - segmentIndex, s, err := sl.FindExif() - if err != nil { - if err == exif.ErrNoExif { - return 0, nil, nil, err - } - - log.Panic(err) - } - - exifTags, err = s.FlatExif() - log.PanicIf(err) - - return segmentIndex, s, exifTags, nil -} - -func makeEmptyExifSegment() (s *Segment) { - - // TODO(dustin): Add test - - return &Segment{ - MarkerId: MARKER_APP1, - } -} - -// SetExif encodes and sets EXIF data into the given segment. If `index` is -1, -// append a new segment. -func (sl *SegmentList) SetExif(ib *exif.IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - _, s, err := sl.FindExif() - if err != nil { - if log.Is(err, exif.ErrNoExif) == false { - log.Panic(err) - } - - s = makeEmptyExifSegment() - - prefix := sl.segments[:1] - - // Install it near the beginning where we know it's safe. We can't - // insert it after the EOI segment, and there might be more than one - // depending on implementation and/or lax adherence to the standard. - tail := append([]*Segment{s}, sl.segments[1:]...) - - sl.segments = append(prefix, tail...) - } - - err = s.SetExif(ib) - log.PanicIf(err) - - return nil -} - -// DropExif will drop the EXIF data if present. -func (sl *SegmentList) DropExif() (wasDropped bool, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - i, _, err := sl.FindExif() - if err == nil { - // Found. - sl.segments = append(sl.segments[:i], sl.segments[i+1:]...) - - return true, nil - } else if log.Is(err, exif.ErrNoExif) == false { - log.Panic(err) - } - - // Not found. - return false, nil -} - -// Write writes the segment data to the given `io.Writer`. -func (sl *SegmentList) Write(w io.Writer) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - offset := 0 - - for i, s := range sl.segments { - h := sha1.New() - h.Write(s.Data) - - // The scan-data will have a marker-ID of (0) because it doesn't have a - // marker-ID or length. - if s.MarkerId != 0 { - _, err := w.Write([]byte{0xff}) - log.PanicIf(err) - - offset++ - - _, err = w.Write([]byte{s.MarkerId}) - log.PanicIf(err) - - offset++ - - sizeLen, found := markerLen[s.MarkerId] - if found == false || sizeLen == 2 { - sizeLen = 2 - l := uint16(len(s.Data) + sizeLen) - - err = binary.Write(w, binary.BigEndian, &l) - log.PanicIf(err) - - offset += 2 - } else if sizeLen == 4 { - l := uint32(len(s.Data) + sizeLen) - - err = binary.Write(w, binary.BigEndian, &l) - log.PanicIf(err) - - offset += 4 - } else if sizeLen != 0 { - log.Panicf("not a supported marker-size: SEGMENT-INDEX=(%d) MARKER-ID=(0x%02x) MARKER-SIZE-LEN=(%d)", i, s.MarkerId, sizeLen) - } - } - - _, err := w.Write(s.Data) - log.PanicIf(err) - - offset += len(s.Data) - } - - return nil -} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/splitter.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/splitter.go deleted file mode 100644 index 8e9c7c020..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/splitter.go +++ /dev/null @@ -1,437 +0,0 @@ -package jpegstructure - -import ( - "bufio" - "bytes" - "io" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -// JpegSplitter uses the Go stream splitter to divide the JPEG stream into -// segments. -type JpegSplitter struct { - lastMarkerId byte - lastMarkerName string - counter int - lastIsScanData bool - visitor interface{} - - currentOffset int - segments *SegmentList - - scandataOffset int -} - -// NewJpegSplitter returns a new JpegSplitter. -func NewJpegSplitter(visitor interface{}) *JpegSplitter { - return &JpegSplitter{ - segments: NewSegmentList(nil), - visitor: visitor, - } -} - -// Segments returns all found segments. -func (js *JpegSplitter) Segments() *SegmentList { - return js.segments -} - -// MarkerId returns the ID of the last processed marker. -func (js *JpegSplitter) MarkerId() byte { - return js.lastMarkerId -} - -// MarkerName returns the name of the last-processed marker. -func (js *JpegSplitter) MarkerName() string { - return js.lastMarkerName -} - -// Counter returns the number of processed segments. -func (js *JpegSplitter) Counter() int { - return js.counter -} - -// IsScanData returns whether the last processed segment was scan-data. -func (js *JpegSplitter) IsScanData() bool { - return js.lastIsScanData -} - -func (js *JpegSplitter) processScanData(data []byte) (advanceBytes int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Search through the segment, past all 0xff's therein, until we encounter - // the EOI segment. - - dataLength := -1 - for i := js.scandataOffset; i < len(data); i++ { - thisByte := data[i] - - if i == 0 { - continue - } - - lastByte := data[i-1] - if lastByte != 0xff { - continue - } - - if thisByte == 0x00 || thisByte >= 0xd0 && thisByte <= 0xd8 { - continue - } - - // After all of the other checks, this means that we're on the EOF - // segment. - if thisByte != MARKER_EOI { - continue - } - - dataLength = i - 1 - break - } - - if dataLength == -1 { - // On the next pass, start on the last byte of this pass, just in case - // the first byte of the two-byte sequence is here. - js.scandataOffset = len(data) - 1 - - jpegLogger.Debugf(nil, "Scan-data not fully available (%d).", len(data)) - return 0, nil - } - - js.lastIsScanData = true - js.lastMarkerId = 0 - js.lastMarkerName = "" - - // Note that we don't increment the counter since this isn't an actual - // segment. - - jpegLogger.Debugf(nil, "End of scan-data.") - - err = js.handleSegment(0x0, "!SCANDATA", 0x0, data[:dataLength]) - log.PanicIf(err) - - return dataLength, nil -} - -func (js *JpegSplitter) readSegment(data []byte) (count int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if js.counter == 0 { - // Verify magic bytes. - - if len(data) < 3 { - jpegLogger.Debugf(nil, "Not enough (1)") - return 0, nil - } - - if data[0] == jpegMagic2000[0] && data[1] == jpegMagic2000[1] && data[2] == jpegMagic2000[2] { - // TODO(dustin): Revisit JPEG2000 support. - log.Panicf("JPEG2000 not supported") - } - - if data[0] != jpegMagicStandard[0] || data[1] != jpegMagicStandard[1] || data[2] != jpegMagicStandard[2] { - log.Panicf("file does not look like a JPEG: (%02x) (%02x) (%02x)", data[0], data[1], data[2]) - } - } - - chunkLength := len(data) - - jpegLogger.Debugf(nil, "SPLIT: LEN=(%d) COUNTER=(%d)", chunkLength, js.counter) - - if js.scanDataIsNext() == true { - // If the last segment was the SOS, we're currently sitting on scan data. - // Search for the EOI marker afterward in order to know how much data - // there is. Return this as its own token. - // - // REF: https://stackoverflow.com/questions/26715684/parsing-jpeg-sos-marker - - advanceBytes, err := js.processScanData(data) - log.PanicIf(err) - - // This will either return 0 and implicitly request that we need more - // data and then need to run again or will return an actual byte count - // to progress by. - - return advanceBytes, nil - } else if js.lastMarkerId == MARKER_EOI { - // We have more data following the EOI, which is unexpected. There - // might be non-standard cruft at the end of the file. Terminate the - // parse because the file-structure is, technically, complete at this - // point. - - return 0, io.EOF - } else { - js.lastIsScanData = false - } - - // If we're here, we're supposed to be sitting on the 0xff bytes at the - // beginning of a segment (just before the marker). - - if data[0] != 0xff { - log.Panicf("not on new segment marker @ (%d): (%02X)", js.currentOffset, data[0]) - } - - i := 0 - found := false - for ; i < chunkLength; i++ { - jpegLogger.Debugf(nil, "Prefix check: (%d) %02X", i, data[i]) - - if data[i] != 0xff { - found = true - break - } - } - - jpegLogger.Debugf(nil, "Skipped over leading 0xFF bytes: (%d)", i) - - if found == false || i >= chunkLength { - jpegLogger.Debugf(nil, "Not enough (3)") - return 0, nil - } - - markerId := data[i] - - js.lastMarkerName = markerNames[markerId] - - sizeLen, found := markerLen[markerId] - jpegLogger.Debugf(nil, "MARKER-ID=%x SIZELEN=%v FOUND=%v", markerId, sizeLen, found) - - i++ - - b := bytes.NewBuffer(data[i:]) - payloadLength := 0 - - // marker-ID + size => 2 + - headerSize := 2 + sizeLen - - if found == false { - - // It's not one of the static-length markers. Read the length. - // - // The length is an unsigned 16-bit network/big-endian. - - // marker-ID + size => 2 + 2 - headerSize = 2 + 2 - - if i+2 >= chunkLength { - jpegLogger.Debugf(nil, "Not enough (4)") - return 0, nil - } - - l := uint16(0) - err = binary.Read(b, binary.BigEndian, &l) - log.PanicIf(err) - - if l <= 2 { - log.Panicf("length of size read for non-special marker (%02x) is unexpectedly not more than two.", markerId) - } - - // (l includes the bytes of the length itself.) - payloadLength = int(l) - 2 - jpegLogger.Debugf(nil, "DataLength (dynamically-sized segment): (%d)", payloadLength) - - i += 2 - } else if sizeLen > 0 { - - // Accommodates the non-zero markers in our marker index, which only - // represent J2C extensions. - // - // The length is an unsigned 32-bit network/big-endian. - - // TODO(dustin): !! This needs to be tested, but we need an image. - - if sizeLen != 4 { - log.Panicf("known non-zero marker is not four bytes, which is not currently handled: M=(%x)", markerId) - } - - if i+4 >= chunkLength { - jpegLogger.Debugf(nil, "Not enough (5)") - return 0, nil - } - - l := uint32(0) - err = binary.Read(b, binary.BigEndian, &l) - log.PanicIf(err) - - payloadLength = int(l) - 4 - jpegLogger.Debugf(nil, "DataLength (four-byte-length segment): (%u)", l) - - i += 4 - } - - jpegLogger.Debugf(nil, "PAYLOAD-LENGTH: %d", payloadLength) - - payload := data[i:] - - if payloadLength < 0 { - log.Panicf("payload length less than zero: (%d)", payloadLength) - } - - i += int(payloadLength) - - if i > chunkLength { - jpegLogger.Debugf(nil, "Not enough (6)") - return 0, nil - } - - jpegLogger.Debugf(nil, "Found whole segment.") - - js.lastMarkerId = markerId - - payloadWindow := payload[:payloadLength] - err = js.handleSegment(markerId, js.lastMarkerName, headerSize, payloadWindow) - log.PanicIf(err) - - js.counter++ - - jpegLogger.Debugf(nil, "Returning advance of (%d)", i) - - return i, nil -} - -func (js *JpegSplitter) scanDataIsNext() bool { - return js.lastMarkerId == MARKER_SOS -} - -// Split is the base splitting function that satisfies `bufio.SplitFunc`. -func (js *JpegSplitter) Split(data []byte, atEOF bool) (advance int, token []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - for len(data) > 0 { - currentAdvance, err := js.readSegment(data) - if err != nil { - if err == io.EOF { - // We've encountered an EOI marker. - return 0, nil, err - } - - log.Panic(err) - } - - if currentAdvance == 0 { - if len(data) > 0 && atEOF == true { - // Provide a little context in the error message. - - if js.scanDataIsNext() == true { - // Yes, we've ran into this. - - log.Panicf("scan-data is unbounded; EOI not encountered before EOF") - } else { - log.Panicf("partial segment data encountered before scan-data") - } - } - - // We don't have enough data for another segment. - break - } - - data = data[currentAdvance:] - advance += currentAdvance - } - - return advance, nil, nil -} - -func (js *JpegSplitter) parseSof(data []byte) (sof *SofSegment, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - stream := bytes.NewBuffer(data) - buffer := bufio.NewReader(stream) - - bitsPerSample, err := buffer.ReadByte() - log.PanicIf(err) - - height := uint16(0) - err = binary.Read(buffer, binary.BigEndian, &height) - log.PanicIf(err) - - width := uint16(0) - err = binary.Read(buffer, binary.BigEndian, &width) - log.PanicIf(err) - - componentCount, err := buffer.ReadByte() - log.PanicIf(err) - - sof = &SofSegment{ - BitsPerSample: bitsPerSample, - Width: width, - Height: height, - ComponentCount: componentCount, - } - - return sof, nil -} - -func (js *JpegSplitter) parseAppData(markerId byte, data []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - return nil -} - -func (js *JpegSplitter) handleSegment(markerId byte, markerName string, headerSize int, payload []byte) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - cloned := make([]byte, len(payload)) - copy(cloned, payload) - - s := &Segment{ - MarkerId: markerId, - MarkerName: markerName, - Offset: js.currentOffset, - Data: cloned, - } - - jpegLogger.Debugf(nil, "Encountered marker (0x%02x) [%s] at offset (%d)", markerId, markerName, js.currentOffset) - - js.currentOffset += headerSize + len(payload) - - js.segments.Add(s) - - sv, ok := js.visitor.(SegmentVisitor) - if ok == true { - err = sv.HandleSegment(js.lastMarkerId, js.lastMarkerName, js.counter, js.lastIsScanData) - log.PanicIf(err) - } - - if markerId >= MARKER_SOF0 && markerId <= MARKER_SOF15 { - ssv, ok := js.visitor.(SofSegmentVisitor) - if ok == true { - sof, err := js.parseSof(payload) - log.PanicIf(err) - - err = ssv.HandleSof(sof) - log.PanicIf(err) - } - } else if markerId >= MARKER_APP0 && markerId <= MARKER_APP15 { - err := js.parseAppData(markerId, payload) - log.PanicIf(err) - } - - return nil -} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/testing_common.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/testing_common.go deleted file mode 100644 index e7169c2f0..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/testing_common.go +++ /dev/null @@ -1,73 +0,0 @@ -package jpegstructure - -import ( - "os" - "path" - - "github.com/dsoprea/go-logging" -) - -var ( - testImageRelFilepath = "NDM_8901.jpg" -) - -var ( - moduleRootPath = "" - assetsPath = "" -) - -// GetModuleRootPath returns the root-path of the module. -func GetModuleRootPath() string { - if moduleRootPath == "" { - moduleRootPath = os.Getenv("JPEG_MODULE_ROOT_PATH") - if moduleRootPath != "" { - return moduleRootPath - } - - currentWd, err := os.Getwd() - log.PanicIf(err) - - currentPath := currentWd - visited := make([]string, 0) - - for { - tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") - - _, err := os.Stat(tryStampFilepath) - if err != nil && os.IsNotExist(err) != true { - log.Panic(err) - } else if err == nil { - break - } - - visited = append(visited, tryStampFilepath) - - currentPath = path.Dir(currentPath) - if currentPath == "/" { - log.Panicf("could not find module-root: %v", visited) - } - } - - moduleRootPath = currentPath - } - - return moduleRootPath -} - -// GetTestAssetsPath returns the path of the test-assets. -func GetTestAssetsPath() string { - if assetsPath == "" { - moduleRootPath := GetModuleRootPath() - assetsPath = path.Join(moduleRootPath, "assets") - } - - return assetsPath -} - -// GetTestImageFilepath returns the file-path of the common test-image. -func GetTestImageFilepath() string { - assetsPath := GetTestAssetsPath() - filepath := path.Join(assetsPath, testImageRelFilepath) - - return filepath -} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/utility.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/utility.go deleted file mode 100644 index 1c618ba6d..000000000 --- a/vendor/github.com/dsoprea/go-jpeg-image-structure/utility.go +++ /dev/null @@ -1,110 +0,0 @@ -package jpegstructure - -import ( - "bytes" - "fmt" - "sort" - "strings" - - "github.com/dsoprea/go-logging" - "github.com/go-xmlfmt/xmlfmt" -) - -// DumpBytes prints the hex for a given byte-slice. -func DumpBytes(data []byte) { - fmt.Printf("DUMP: ") - for _, x := range data { - fmt.Printf("%02x ", x) - } - - fmt.Printf("\n") -} - -// DumpBytesClause prints a Go-formatted byte-slice expression. -func DumpBytesClause(data []byte) { - fmt.Printf("DUMP: ") - - fmt.Printf("[]byte { ") - - for i, x := range data { - fmt.Printf("0x%02x", x) - - if i < len(data)-1 { - fmt.Printf(", ") - } - } - - fmt.Printf(" }\n") -} - -// DumpBytesToString returns a string of hex-encoded bytes. -func DumpBytesToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteRune(' ') - log.PanicIf(err) - } - } - - return b.String() -} - -// DumpBytesClauseToString returns a string of Go-formatted byte values. -func DumpBytesClauseToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) - log.PanicIf(err) - - if i < len(data)-1 { - _, err := b.WriteString(", ") - log.PanicIf(err) - } - } - - return b.String() -} - -// FormatXml prettifies XML data. -func FormatXml(raw string) (formatted string, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - formatted = xmlfmt.FormatXML(raw, " ", " ") - formatted = strings.TrimSpace(formatted) - - return formatted, nil -} - -// SortStringStringMap sorts a string-string dictionary and returns it as a list -// of 2-tuples. -func SortStringStringMap(data map[string]string) (sorted [][2]string) { - // Sort keys. - - sortedKeys := make([]string, len(data)) - i := 0 - for key := range data { - sortedKeys[i] = key - i++ - } - - sort.Strings(sortedKeys) - - // Build result. - - sorted = make([][2]string, len(sortedKeys)) - for i, key := range sortedKeys { - sorted[i] = [2]string{key, data[key]} - } - - return sorted -} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/.MODULE_ROOT b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/.MODULE_ROOT new file mode 100644 index 000000000..e69de29bb diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/LICENSE b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/LICENSE new file mode 100644 index 000000000..163291ed6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/LICENSE @@ -0,0 +1,9 @@ +MIT LICENSE + +Copyright 2020 Dustin Oprea + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/README.md b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/README.md new file mode 100644 index 000000000..bf60ef504 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/README.md @@ -0,0 +1,10 @@ +[![Build Status](https://travis-ci.org/dsoprea/go-jpeg-image-structure/v2.svg?branch=master)](https://travis-ci.org/dsoprea/go-jpeg-image-structure/v2) +[![codecov](https://codecov.io/gh/dsoprea/go-jpeg-image-structure/branch/master/graph/badge.svg)](https://codecov.io/gh/dsoprea/go-jpeg-image-structure) +[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-jpeg-image-structure/v2)](https://goreportcard.com/report/github.com/dsoprea/go-jpeg-image-structure/v2) +[![GoDoc](https://godoc.org/github.com/dsoprea/go-jpeg-image-structure/v2?status.svg)](https://godoc.org/github.com/dsoprea/go-jpeg-image-structure/v2) + +## Overview + +Parse raw JPEG data into individual segments of data. You can print or export this data, including hash digests for each. You can also parse/modify the EXIF data and write an updated image. + +EXIF, XMP, and IPTC data can also be extracted. The provided CLI tool can print this data as well. diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/markers.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/markers.go new file mode 100644 index 000000000..a12171bd8 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/markers.go @@ -0,0 +1,212 @@ +package jpegstructure + +import ( + "github.com/dsoprea/go-logging" +) + +const ( + // MARKER_SOI marker + MARKER_SOI = 0xd8 + + // MARKER_EOI marker + MARKER_EOI = 0xd9 + + // MARKER_SOS marker + MARKER_SOS = 0xda + + // MARKER_SOD marker + MARKER_SOD = 0x93 + + // MARKER_DQT marker + MARKER_DQT = 0xdb + + // MARKER_APP0 marker + MARKER_APP0 = 0xe0 + + // MARKER_APP1 marker + MARKER_APP1 = 0xe1 + + // MARKER_APP2 marker + MARKER_APP2 = 0xe2 + + // MARKER_APP3 marker + MARKER_APP3 = 0xe3 + + // MARKER_APP4 marker + MARKER_APP4 = 0xe4 + + // MARKER_APP5 marker + MARKER_APP5 = 0xe5 + + // MARKER_APP6 marker + MARKER_APP6 = 0xe6 + + // MARKER_APP7 marker + MARKER_APP7 = 0xe7 + + // MARKER_APP8 marker + MARKER_APP8 = 0xe8 + + // MARKER_APP10 marker + MARKER_APP10 = 0xea + + // MARKER_APP12 marker + MARKER_APP12 = 0xec + + // MARKER_APP13 marker + MARKER_APP13 = 0xed + + // MARKER_APP14 marker + MARKER_APP14 = 0xee + + // MARKER_APP15 marker + MARKER_APP15 = 0xef + + // MARKER_COM marker + MARKER_COM = 0xfe + + // MARKER_CME marker + MARKER_CME = 0x64 + + // MARKER_SIZ marker + MARKER_SIZ = 0x51 + + // MARKER_DHT marker + MARKER_DHT = 0xc4 + + // MARKER_JPG marker + MARKER_JPG = 0xc8 + + // MARKER_DAC marker + MARKER_DAC = 0xcc + + // MARKER_SOF0 marker + MARKER_SOF0 = 0xc0 + + // MARKER_SOF1 marker + MARKER_SOF1 = 0xc1 + + // MARKER_SOF2 marker + MARKER_SOF2 = 0xc2 + + // MARKER_SOF3 marker + MARKER_SOF3 = 0xc3 + + // MARKER_SOF5 marker + MARKER_SOF5 = 0xc5 + + // MARKER_SOF6 marker + MARKER_SOF6 = 0xc6 + + // MARKER_SOF7 marker + MARKER_SOF7 = 0xc7 + + // MARKER_SOF9 marker + MARKER_SOF9 = 0xc9 + + // MARKER_SOF10 marker + MARKER_SOF10 = 0xca + + // MARKER_SOF11 marker + MARKER_SOF11 = 0xcb + + // MARKER_SOF13 marker + MARKER_SOF13 = 0xcd + + // MARKER_SOF14 marker + MARKER_SOF14 = 0xce + + // MARKER_SOF15 marker + MARKER_SOF15 = 0xcf +) + +var ( + jpegLogger = log.NewLogger("jpegstructure.jpeg") + jpegMagicStandard = []byte{0xff, MARKER_SOI, 0xff} + jpegMagic2000 = []byte{0xff, 0x4f, 0xff} + + markerLen = map[byte]int{ + 0x00: 0, + 0x01: 0, + 0xd0: 0, + 0xd1: 0, + 0xd2: 0, + 0xd3: 0, + 0xd4: 0, + 0xd5: 0, + 0xd6: 0, + 0xd7: 0, + 0xd8: 0, + 0xd9: 0, + 0xda: 0, + + // J2C + 0x30: 0, + 0x31: 0, + 0x32: 0, + 0x33: 0, + 0x34: 0, + 0x35: 0, + 0x36: 0, + 0x37: 0, + 0x38: 0, + 0x39: 0, + 0x3a: 0, + 0x3b: 0, + 0x3c: 0, + 0x3d: 0, + 0x3e: 0, + 0x3f: 0, + 0x4f: 0, + 0x92: 0, + 0x93: 0, + + // J2C extensions + 0x74: 4, + 0x75: 4, + 0x77: 4, + } + + markerNames = map[byte]string{ + MARKER_SOI: "SOI", + MARKER_EOI: "EOI", + MARKER_SOS: "SOS", + MARKER_SOD: "SOD", + MARKER_DQT: "DQT", + MARKER_APP0: "APP0", + MARKER_APP1: "APP1", + MARKER_APP2: "APP2", + MARKER_APP3: "APP3", + MARKER_APP4: "APP4", + MARKER_APP5: "APP5", + MARKER_APP6: "APP6", + MARKER_APP7: "APP7", + MARKER_APP8: "APP8", + MARKER_APP10: "APP10", + MARKER_APP12: "APP12", + MARKER_APP13: "APP13", + MARKER_APP14: "APP14", + MARKER_APP15: "APP15", + MARKER_COM: "COM", + MARKER_CME: "CME", + MARKER_SIZ: "SIZ", + + MARKER_DHT: "DHT", + MARKER_JPG: "JPG", + MARKER_DAC: "DAC", + + MARKER_SOF0: "SOF0", + MARKER_SOF1: "SOF1", + MARKER_SOF2: "SOF2", + MARKER_SOF3: "SOF3", + MARKER_SOF5: "SOF5", + MARKER_SOF6: "SOF6", + MARKER_SOF7: "SOF7", + MARKER_SOF9: "SOF9", + MARKER_SOF10: "SOF10", + MARKER_SOF11: "SOF11", + MARKER_SOF13: "SOF13", + MARKER_SOF14: "SOF14", + MARKER_SOF15: "SOF15", + } +) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/media_parser.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/media_parser.go new file mode 100644 index 000000000..e6fc60bc4 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/media_parser.go @@ -0,0 +1,139 @@ +package jpegstructure + +import ( + "bufio" + "bytes" + "image" + "io" + "os" + + "image/jpeg" + + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-utility/v2/image" +) + +// JpegMediaParser is a `riimage.MediaParser` that knows how to parse JPEG +// images. +type JpegMediaParser struct { +} + +// NewJpegMediaParser returns a new JpegMediaParser. +func NewJpegMediaParser() *JpegMediaParser { + + // TODO(dustin): Add test + + return new(JpegMediaParser) +} + +// Parse parses a JPEG uses an `io.ReadSeeker`. Even if it fails, it will return +// the list of segments encountered prior to the failure. +func (jmp *JpegMediaParser) Parse(rs io.ReadSeeker, size int) (ec riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + s := bufio.NewScanner(rs) + + // Since each segment can be any size, our buffer must allowed to grow as + // large as the file. + buffer := []byte{} + s.Buffer(buffer, size) + + js := NewJpegSplitter(nil) + s.Split(js.Split) + + for s.Scan() != false { + } + + // Always return the segments that were parsed, at least until there was an + // error. + ec = js.Segments() + + log.PanicIf(s.Err()) + + return ec, nil +} + +// ParseFile parses a JPEG file. Even if it fails, it will return the list of +// segments encountered prior to the failure. +func (jmp *JpegMediaParser) ParseFile(filepath string) (ec riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + f, err := os.Open(filepath) + log.PanicIf(err) + + defer f.Close() + + stat, err := f.Stat() + log.PanicIf(err) + + size := stat.Size() + + sl, err := jmp.Parse(f, int(size)) + + // Always return the segments that were parsed, at least until there was an + // error. + ec = sl + + log.PanicIf(err) + + return ec, nil +} + +// ParseBytes parses a JPEG byte-slice. Even if it fails, it will return the +// list of segments encountered prior to the failure. +func (jmp *JpegMediaParser) ParseBytes(data []byte) (ec riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + br := bytes.NewReader(data) + + sl, err := jmp.Parse(br, len(data)) + + // Always return the segments that were parsed, at least until there was an + // error. + ec = sl + + log.PanicIf(err) + + return ec, nil +} + +// LooksLikeFormat indicates whether the data looks like a JPEG image. +func (jmp *JpegMediaParser) LooksLikeFormat(data []byte) bool { + if len(data) < 4 { + return false + } + + l := len(data) + if data[0] != 0xff || data[1] != MARKER_SOI || data[l-2] != 0xff || data[l-1] != MARKER_EOI { + return false + } + + return true +} + +// GetImage returns an image.Image-compatible struct. +func (jmp *JpegMediaParser) GetImage(r io.Reader) (img image.Image, err error) { + img, err = jpeg.Decode(r) + log.PanicIf(err) + + return img, nil +} + +var ( + // Enforce interface conformance. + _ riimage.MediaParser = new(JpegMediaParser) +) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment.go new file mode 100644 index 000000000..6b433bf1f --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment.go @@ -0,0 +1,352 @@ +package jpegstructure + +import ( + "bytes" + "errors" + "fmt" + + "crypto/sha1" + "encoding/hex" + + "github.com/dsoprea/go-exif/v3" + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-iptc" + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-photoshop-info-format" + "github.com/dsoprea/go-utility/v2/image" +) + +const ( + pirIptcImageResourceId = uint16(0x0404) +) + +var ( + // exifPrefix is the prefix found at the top of an EXIF slice. This is JPEG- + // specific. + exifPrefix = []byte{'E', 'x', 'i', 'f', 0, 0} + + xmpPrefix = []byte("http://ns.adobe.com/xap/1.0/\000") + + ps30Prefix = []byte("Photoshop 3.0\000") +) + +var ( + // ErrNoXmp is returned if XMP data was requested but not found. + ErrNoXmp = errors.New("no XMP data") + + // ErrNoIptc is returned if IPTC data was requested but not found. + ErrNoIptc = errors.New("no IPTC data") + + // ErrNoPhotoshopData is returned if Photoshop info was requested but not + // found. + ErrNoPhotoshopData = errors.New("no photoshop data") +) + +// SofSegment has info read from a SOF segment. +type SofSegment struct { + // BitsPerSample is the bits-per-sample. + BitsPerSample byte + + // Width is the image width. + Width uint16 + + // Height is the image height. + Height uint16 + + // ComponentCount is the number of color components. + ComponentCount byte +} + +// String returns a string representation of the SOF segment. +func (ss SofSegment) String() string { + + // TODO(dustin): Add test + + return fmt.Sprintf("SOF", ss.BitsPerSample, ss.Width, ss.Height, ss.ComponentCount) +} + +// SegmentVisitor describes a segment-visitor struct. +type SegmentVisitor interface { + // HandleSegment is triggered for each segment encountered as well as the + // scan-data. + HandleSegment(markerId byte, markerName string, counter int, lastIsScanData bool) error +} + +// SofSegmentVisitor describes a visitor that is only called for each SOF +// segment. +type SofSegmentVisitor interface { + // HandleSof is called for each encountered SOF segment. + HandleSof(sof *SofSegment) error +} + +// Segment describes a single segment. +type Segment struct { + MarkerId byte + MarkerName string + Offset int + Data []byte + + photoshopInfo map[uint16]photoshopinfo.Photoshop30InfoRecord + iptcTags map[iptc.StreamTagKey][]iptc.TagData +} + +// SetExif encodes and sets EXIF data into this segment. +func (s *Segment) SetExif(ib *exif.IfdBuilder) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + ibe := exif.NewIfdByteEncoder() + + exifData, err := ibe.EncodeToExif(ib) + log.PanicIf(err) + + l := len(exifPrefix) + + s.Data = make([]byte, l+len(exifData)) + copy(s.Data[0:], exifPrefix) + copy(s.Data[l:], exifData) + + return nil +} + +// Exif returns an `exif.Ifd` instance for the EXIF data we currently have. +func (s *Segment) Exif() (rootIfd *exif.Ifd, data []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + l := len(exifPrefix) + + rawExif := s.Data[l:] + + jpegLogger.Debugf(nil, "Attempting to parse (%d) byte EXIF blob (Exif).", len(rawExif)) + + im, err := exifcommon.NewIfdMappingWithStandard() + log.PanicIf(err) + + ti := exif.NewTagIndex() + + _, index, err := exif.Collect(im, ti, rawExif) + log.PanicIf(err) + + return index.RootIfd, rawExif, nil +} + +// FlatExif parses the EXIF data and just returns a list of tags. +func (s *Segment) FlatExif() (exifTags []exif.ExifTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + l := len(exifPrefix) + + rawExif := s.Data[l:] + + jpegLogger.Debugf(nil, "Attempting to parse (%d) byte EXIF blob (FlatExif).", len(rawExif)) + + exifTags, _, err = exif.GetFlatExifData(rawExif, nil) + log.PanicIf(err) + + return exifTags, nil +} + +// EmbeddedString returns a string of properties that can be embedded into an +// longer string of properties. +func (s *Segment) EmbeddedString() string { + h := sha1.New() + h.Write(s.Data) + + // TODO(dustin): Add test + + digestString := hex.EncodeToString(h.Sum(nil)) + + return fmt.Sprintf("OFFSET=(0x%08x %10d) ID=(0x%02x) NAME=[%-5s] SIZE=(%10d) SHA1=[%s]", s.Offset, s.Offset, s.MarkerId, markerNames[s.MarkerId], len(s.Data), digestString) +} + +// String returns a descriptive string. +func (s *Segment) String() string { + + // TODO(dustin): Add test + + return fmt.Sprintf("Segment<%s>", s.EmbeddedString()) +} + +// IsExif returns true if EXIF data. +func (s *Segment) IsExif() bool { + if s.MarkerId != MARKER_APP1 { + return false + } + + // TODO(dustin): Add test + + l := len(exifPrefix) + + if len(s.Data) < l { + return false + } + + if bytes.Equal(s.Data[:l], exifPrefix) == false { + return false + } + + return true +} + +// IsXmp returns true if XMP data. +func (s *Segment) IsXmp() bool { + if s.MarkerId != MARKER_APP1 { + return false + } + + // TODO(dustin): Add test + + l := len(xmpPrefix) + + if len(s.Data) < l { + return false + } + + if bytes.Equal(s.Data[:l], xmpPrefix) == false { + return false + } + + return true +} + +// FormattedXmp returns a formatted XML string. This only makes sense for a +// segment comprised of XML data (like XMP). +func (s *Segment) FormattedXmp() (formatted string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + if s.IsXmp() != true { + log.Panicf("not an XMP segment") + } + + l := len(xmpPrefix) + + raw := string(s.Data[l:]) + + formatted, err = FormatXml(raw) + log.PanicIf(err) + + return formatted, nil +} + +func (s *Segment) parsePhotoshopInfo() (photoshopInfo map[uint16]photoshopinfo.Photoshop30InfoRecord, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if s.photoshopInfo != nil { + return s.photoshopInfo, nil + } + + if s.MarkerId != MARKER_APP13 { + return nil, ErrNoPhotoshopData + } + + l := len(ps30Prefix) + + if len(s.Data) < l { + return nil, ErrNoPhotoshopData + } + + if bytes.Equal(s.Data[:l], ps30Prefix) == false { + return nil, ErrNoPhotoshopData + } + + data := s.Data[l:] + b := bytes.NewBuffer(data) + + // Parse it. + + pirIndex, err := photoshopinfo.ReadPhotoshop30Info(b) + log.PanicIf(err) + + s.photoshopInfo = pirIndex + + return s.photoshopInfo, nil +} + +// IsIptc returns true if XMP data. +func (s *Segment) IsIptc() bool { + // TODO(dustin): Add test + + // There's a cost to determining if there's IPTC data, so we won't do it + // more than once. + if s.iptcTags != nil { + return true + } + + photoshopInfo, err := s.parsePhotoshopInfo() + if err != nil { + if err == ErrNoPhotoshopData { + return false + } + + log.Panic(err) + } + + // Bail if the Photoshop info doesn't have IPTC data. + + _, found := photoshopInfo[pirIptcImageResourceId] + if found == false { + return false + } + + return true +} + +// Iptc parses Photoshop info (if present) and then parses the IPTC info inside +// it (if present). +func (s *Segment) Iptc() (tags map[iptc.StreamTagKey][]iptc.TagData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Cache the parse. + if s.iptcTags != nil { + return s.iptcTags, nil + } + + photoshopInfo, err := s.parsePhotoshopInfo() + log.PanicIf(err) + + iptcPir, found := photoshopInfo[pirIptcImageResourceId] + if found == false { + return nil, ErrNoIptc + } + + b := bytes.NewBuffer(iptcPir.Data) + + tags, err = iptc.ParseStream(b) + log.PanicIf(err) + + s.iptcTags = tags + + return tags, nil +} + +var ( + // Enforce interface conformance. + _ riimage.MediaContext = new(Segment) +) diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment_list.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment_list.go new file mode 100644 index 000000000..b4f4d5810 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/segment_list.go @@ -0,0 +1,416 @@ +package jpegstructure + +import ( + "bytes" + "fmt" + "io" + + "crypto/sha1" + "encoding/binary" + + "github.com/dsoprea/go-exif/v3" + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-iptc" + "github.com/dsoprea/go-logging" +) + +// SegmentList contains a slice of segments. +type SegmentList struct { + segments []*Segment +} + +// NewSegmentList returns a new SegmentList struct. +func NewSegmentList(segments []*Segment) (sl *SegmentList) { + if segments == nil { + segments = make([]*Segment, 0) + } + + return &SegmentList{ + segments: segments, + } +} + +// OffsetsEqual returns true is all segments have the same marker-IDs and were +// found at the same offsets. +func (sl *SegmentList) OffsetsEqual(o *SegmentList) bool { + if len(o.segments) != len(sl.segments) { + return false + } + + for i, s := range o.segments { + if s.MarkerId != sl.segments[i].MarkerId || s.Offset != sl.segments[i].Offset { + return false + } + } + + return true +} + +// Segments returns the underlying slice of segments. +func (sl *SegmentList) Segments() []*Segment { + return sl.segments +} + +// Add adds another segment. +func (sl *SegmentList) Add(s *Segment) { + sl.segments = append(sl.segments, s) +} + +// Print prints segment info. +func (sl *SegmentList) Print() { + if len(sl.segments) == 0 { + fmt.Printf("No segments.\n") + } else { + exifIndex, _, err := sl.FindExif() + if err != nil { + if err == exif.ErrNoExif { + exifIndex = -1 + } else { + log.Panic(err) + } + } + + xmpIndex, _, err := sl.FindXmp() + if err != nil { + if err == ErrNoXmp { + xmpIndex = -1 + } else { + log.Panic(err) + } + } + + iptcIndex, _, err := sl.FindIptc() + if err != nil { + if err == ErrNoIptc { + iptcIndex = -1 + } else { + log.Panic(err) + } + } + + for i, s := range sl.segments { + fmt.Printf("%2d: %s", i, s.EmbeddedString()) + + if i == exifIndex { + fmt.Printf(" [EXIF]") + } else if i == xmpIndex { + fmt.Printf(" [XMP]") + } else if i == iptcIndex { + fmt.Printf(" [IPTC]") + } + + fmt.Printf("\n") + } + } +} + +// Validate checks that all of the markers are actually located at all of the +// recorded offsets. +func (sl *SegmentList) Validate(data []byte) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if len(sl.segments) < 2 { + log.Panicf("minimum segments not found") + } + + if sl.segments[0].MarkerId != MARKER_SOI { + log.Panicf("first segment not SOI") + } else if sl.segments[len(sl.segments)-1].MarkerId != MARKER_EOI { + log.Panicf("last segment not EOI") + } + + lastOffset := 0 + for i, s := range sl.segments { + if lastOffset != 0 && s.Offset <= lastOffset { + log.Panicf("segment offset not greater than the last: SEGMENT=(%d) (0x%08x) <= (0x%08x)", i, s.Offset, lastOffset) + } + + // The scan-data doesn't start with a marker. + if s.MarkerId == 0x0 { + continue + } + + o := s.Offset + if bytes.Compare(data[o:o+2], []byte{0xff, s.MarkerId}) != 0 { + log.Panicf("segment offset does not point to the start of a segment: SEGMENT=(%d) (0x%08x)", i, s.Offset) + } + + lastOffset = o + } + + return nil +} + +// FindExif returns the the segment that hosts the EXIF data (if present). +func (sl *SegmentList) FindExif() (index int, segment *Segment, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for i, s := range sl.segments { + if s.IsExif() == true { + return i, s, nil + } + } + + return -1, nil, exif.ErrNoExif +} + +// FindXmp returns the the segment that hosts the XMP data (if present). +func (sl *SegmentList) FindXmp() (index int, segment *Segment, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for i, s := range sl.segments { + if s.IsXmp() == true { + return i, s, nil + } + } + + return -1, nil, ErrNoXmp +} + +// FindIptc returns the the segment that hosts the IPTC data (if present). +func (sl *SegmentList) FindIptc() (index int, segment *Segment, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for i, s := range sl.segments { + if s.IsIptc() == true { + return i, s, nil + } + } + + return -1, nil, ErrNoIptc +} + +// Exif returns an `exif.Ifd` instance for the EXIF data we currently have. +func (sl *SegmentList) Exif() (rootIfd *exif.Ifd, rawExif []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + _, s, err := sl.FindExif() + log.PanicIf(err) + + rootIfd, rawExif, err = s.Exif() + log.PanicIf(err) + + return rootIfd, rawExif, nil +} + +// Iptc returns embedded IPTC data if present. +func (sl *SegmentList) Iptc() (tags map[iptc.StreamTagKey][]iptc.TagData, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add comment and return data. + + _, s, err := sl.FindIptc() + log.PanicIf(err) + + tags, err = s.Iptc() + log.PanicIf(err) + + return tags, nil +} + +// ConstructExifBuilder returns an `exif.IfdBuilder` instance (needed for +// modifying) preloaded with all existing tags. +func (sl *SegmentList) ConstructExifBuilder() (rootIb *exif.IfdBuilder, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rootIfd, _, err := sl.Exif() + if log.Is(err, exif.ErrNoExif) == true { + // No EXIF. Just create a boilerplate builder. + + im := exifcommon.NewIfdMapping() + + err := exifcommon.LoadStandardIfds(im) + log.PanicIf(err) + + ti := exif.NewTagIndex() + + rootIb := + exif.NewIfdBuilder( + im, + ti, + exifcommon.IfdStandardIfdIdentity, + exifcommon.EncodeDefaultByteOrder) + + return rootIb, nil + } else if err != nil { + log.Panic(err) + } + + rootIb = exif.NewIfdBuilderFromExistingChain(rootIfd) + + return rootIb, nil +} + +// DumpExif returns an unstructured list of tags (useful when just reviewing). +func (sl *SegmentList) DumpExif() (segmentIndex int, segment *Segment, exifTags []exif.ExifTag, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + segmentIndex, s, err := sl.FindExif() + if err != nil { + if err == exif.ErrNoExif { + return 0, nil, nil, err + } + + log.Panic(err) + } + + exifTags, err = s.FlatExif() + log.PanicIf(err) + + return segmentIndex, s, exifTags, nil +} + +func makeEmptyExifSegment() (s *Segment) { + + // TODO(dustin): Add test + + return &Segment{ + MarkerId: MARKER_APP1, + } +} + +// SetExif encodes and sets EXIF data into the given segment. If `index` is -1, +// append a new segment. +func (sl *SegmentList) SetExif(ib *exif.IfdBuilder) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + _, s, err := sl.FindExif() + if err != nil { + if log.Is(err, exif.ErrNoExif) == false { + log.Panic(err) + } + + s = makeEmptyExifSegment() + + prefix := sl.segments[:1] + + // Install it near the beginning where we know it's safe. We can't + // insert it after the EOI segment, and there might be more than one + // depending on implementation and/or lax adherence to the standard. + tail := append([]*Segment{s}, sl.segments[1:]...) + + sl.segments = append(prefix, tail...) + } + + err = s.SetExif(ib) + log.PanicIf(err) + + return nil +} + +// DropExif will drop the EXIF data if present. +func (sl *SegmentList) DropExif() (wasDropped bool, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + i, _, err := sl.FindExif() + if err == nil { + // Found. + sl.segments = append(sl.segments[:i], sl.segments[i+1:]...) + + return true, nil + } else if log.Is(err, exif.ErrNoExif) == false { + log.Panic(err) + } + + // Not found. + return false, nil +} + +// Write writes the segment data to the given `io.Writer`. +func (sl *SegmentList) Write(w io.Writer) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + offset := 0 + + for i, s := range sl.segments { + h := sha1.New() + h.Write(s.Data) + + // The scan-data will have a marker-ID of (0) because it doesn't have a + // marker-ID or length. + if s.MarkerId != 0 { + _, err := w.Write([]byte{0xff}) + log.PanicIf(err) + + offset++ + + _, err = w.Write([]byte{s.MarkerId}) + log.PanicIf(err) + + offset++ + + sizeLen, found := markerLen[s.MarkerId] + if found == false || sizeLen == 2 { + sizeLen = 2 + l := uint16(len(s.Data) + sizeLen) + + err = binary.Write(w, binary.BigEndian, &l) + log.PanicIf(err) + + offset += 2 + } else if sizeLen == 4 { + l := uint32(len(s.Data) + sizeLen) + + err = binary.Write(w, binary.BigEndian, &l) + log.PanicIf(err) + + offset += 4 + } else if sizeLen != 0 { + log.Panicf("not a supported marker-size: SEGMENT-INDEX=(%d) MARKER-ID=(0x%02x) MARKER-SIZE-LEN=(%d)", i, s.MarkerId, sizeLen) + } + } + + _, err := w.Write(s.Data) + log.PanicIf(err) + + offset += len(s.Data) + } + + return nil +} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/splitter.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/splitter.go new file mode 100644 index 000000000..8e9c7c020 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/splitter.go @@ -0,0 +1,437 @@ +package jpegstructure + +import ( + "bufio" + "bytes" + "io" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +// JpegSplitter uses the Go stream splitter to divide the JPEG stream into +// segments. +type JpegSplitter struct { + lastMarkerId byte + lastMarkerName string + counter int + lastIsScanData bool + visitor interface{} + + currentOffset int + segments *SegmentList + + scandataOffset int +} + +// NewJpegSplitter returns a new JpegSplitter. +func NewJpegSplitter(visitor interface{}) *JpegSplitter { + return &JpegSplitter{ + segments: NewSegmentList(nil), + visitor: visitor, + } +} + +// Segments returns all found segments. +func (js *JpegSplitter) Segments() *SegmentList { + return js.segments +} + +// MarkerId returns the ID of the last processed marker. +func (js *JpegSplitter) MarkerId() byte { + return js.lastMarkerId +} + +// MarkerName returns the name of the last-processed marker. +func (js *JpegSplitter) MarkerName() string { + return js.lastMarkerName +} + +// Counter returns the number of processed segments. +func (js *JpegSplitter) Counter() int { + return js.counter +} + +// IsScanData returns whether the last processed segment was scan-data. +func (js *JpegSplitter) IsScanData() bool { + return js.lastIsScanData +} + +func (js *JpegSplitter) processScanData(data []byte) (advanceBytes int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Search through the segment, past all 0xff's therein, until we encounter + // the EOI segment. + + dataLength := -1 + for i := js.scandataOffset; i < len(data); i++ { + thisByte := data[i] + + if i == 0 { + continue + } + + lastByte := data[i-1] + if lastByte != 0xff { + continue + } + + if thisByte == 0x00 || thisByte >= 0xd0 && thisByte <= 0xd8 { + continue + } + + // After all of the other checks, this means that we're on the EOF + // segment. + if thisByte != MARKER_EOI { + continue + } + + dataLength = i - 1 + break + } + + if dataLength == -1 { + // On the next pass, start on the last byte of this pass, just in case + // the first byte of the two-byte sequence is here. + js.scandataOffset = len(data) - 1 + + jpegLogger.Debugf(nil, "Scan-data not fully available (%d).", len(data)) + return 0, nil + } + + js.lastIsScanData = true + js.lastMarkerId = 0 + js.lastMarkerName = "" + + // Note that we don't increment the counter since this isn't an actual + // segment. + + jpegLogger.Debugf(nil, "End of scan-data.") + + err = js.handleSegment(0x0, "!SCANDATA", 0x0, data[:dataLength]) + log.PanicIf(err) + + return dataLength, nil +} + +func (js *JpegSplitter) readSegment(data []byte) (count int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if js.counter == 0 { + // Verify magic bytes. + + if len(data) < 3 { + jpegLogger.Debugf(nil, "Not enough (1)") + return 0, nil + } + + if data[0] == jpegMagic2000[0] && data[1] == jpegMagic2000[1] && data[2] == jpegMagic2000[2] { + // TODO(dustin): Revisit JPEG2000 support. + log.Panicf("JPEG2000 not supported") + } + + if data[0] != jpegMagicStandard[0] || data[1] != jpegMagicStandard[1] || data[2] != jpegMagicStandard[2] { + log.Panicf("file does not look like a JPEG: (%02x) (%02x) (%02x)", data[0], data[1], data[2]) + } + } + + chunkLength := len(data) + + jpegLogger.Debugf(nil, "SPLIT: LEN=(%d) COUNTER=(%d)", chunkLength, js.counter) + + if js.scanDataIsNext() == true { + // If the last segment was the SOS, we're currently sitting on scan data. + // Search for the EOI marker afterward in order to know how much data + // there is. Return this as its own token. + // + // REF: https://stackoverflow.com/questions/26715684/parsing-jpeg-sos-marker + + advanceBytes, err := js.processScanData(data) + log.PanicIf(err) + + // This will either return 0 and implicitly request that we need more + // data and then need to run again or will return an actual byte count + // to progress by. + + return advanceBytes, nil + } else if js.lastMarkerId == MARKER_EOI { + // We have more data following the EOI, which is unexpected. There + // might be non-standard cruft at the end of the file. Terminate the + // parse because the file-structure is, technically, complete at this + // point. + + return 0, io.EOF + } else { + js.lastIsScanData = false + } + + // If we're here, we're supposed to be sitting on the 0xff bytes at the + // beginning of a segment (just before the marker). + + if data[0] != 0xff { + log.Panicf("not on new segment marker @ (%d): (%02X)", js.currentOffset, data[0]) + } + + i := 0 + found := false + for ; i < chunkLength; i++ { + jpegLogger.Debugf(nil, "Prefix check: (%d) %02X", i, data[i]) + + if data[i] != 0xff { + found = true + break + } + } + + jpegLogger.Debugf(nil, "Skipped over leading 0xFF bytes: (%d)", i) + + if found == false || i >= chunkLength { + jpegLogger.Debugf(nil, "Not enough (3)") + return 0, nil + } + + markerId := data[i] + + js.lastMarkerName = markerNames[markerId] + + sizeLen, found := markerLen[markerId] + jpegLogger.Debugf(nil, "MARKER-ID=%x SIZELEN=%v FOUND=%v", markerId, sizeLen, found) + + i++ + + b := bytes.NewBuffer(data[i:]) + payloadLength := 0 + + // marker-ID + size => 2 + + headerSize := 2 + sizeLen + + if found == false { + + // It's not one of the static-length markers. Read the length. + // + // The length is an unsigned 16-bit network/big-endian. + + // marker-ID + size => 2 + 2 + headerSize = 2 + 2 + + if i+2 >= chunkLength { + jpegLogger.Debugf(nil, "Not enough (4)") + return 0, nil + } + + l := uint16(0) + err = binary.Read(b, binary.BigEndian, &l) + log.PanicIf(err) + + if l <= 2 { + log.Panicf("length of size read for non-special marker (%02x) is unexpectedly not more than two.", markerId) + } + + // (l includes the bytes of the length itself.) + payloadLength = int(l) - 2 + jpegLogger.Debugf(nil, "DataLength (dynamically-sized segment): (%d)", payloadLength) + + i += 2 + } else if sizeLen > 0 { + + // Accommodates the non-zero markers in our marker index, which only + // represent J2C extensions. + // + // The length is an unsigned 32-bit network/big-endian. + + // TODO(dustin): !! This needs to be tested, but we need an image. + + if sizeLen != 4 { + log.Panicf("known non-zero marker is not four bytes, which is not currently handled: M=(%x)", markerId) + } + + if i+4 >= chunkLength { + jpegLogger.Debugf(nil, "Not enough (5)") + return 0, nil + } + + l := uint32(0) + err = binary.Read(b, binary.BigEndian, &l) + log.PanicIf(err) + + payloadLength = int(l) - 4 + jpegLogger.Debugf(nil, "DataLength (four-byte-length segment): (%u)", l) + + i += 4 + } + + jpegLogger.Debugf(nil, "PAYLOAD-LENGTH: %d", payloadLength) + + payload := data[i:] + + if payloadLength < 0 { + log.Panicf("payload length less than zero: (%d)", payloadLength) + } + + i += int(payloadLength) + + if i > chunkLength { + jpegLogger.Debugf(nil, "Not enough (6)") + return 0, nil + } + + jpegLogger.Debugf(nil, "Found whole segment.") + + js.lastMarkerId = markerId + + payloadWindow := payload[:payloadLength] + err = js.handleSegment(markerId, js.lastMarkerName, headerSize, payloadWindow) + log.PanicIf(err) + + js.counter++ + + jpegLogger.Debugf(nil, "Returning advance of (%d)", i) + + return i, nil +} + +func (js *JpegSplitter) scanDataIsNext() bool { + return js.lastMarkerId == MARKER_SOS +} + +// Split is the base splitting function that satisfies `bufio.SplitFunc`. +func (js *JpegSplitter) Split(data []byte, atEOF bool) (advance int, token []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + for len(data) > 0 { + currentAdvance, err := js.readSegment(data) + if err != nil { + if err == io.EOF { + // We've encountered an EOI marker. + return 0, nil, err + } + + log.Panic(err) + } + + if currentAdvance == 0 { + if len(data) > 0 && atEOF == true { + // Provide a little context in the error message. + + if js.scanDataIsNext() == true { + // Yes, we've ran into this. + + log.Panicf("scan-data is unbounded; EOI not encountered before EOF") + } else { + log.Panicf("partial segment data encountered before scan-data") + } + } + + // We don't have enough data for another segment. + break + } + + data = data[currentAdvance:] + advance += currentAdvance + } + + return advance, nil, nil +} + +func (js *JpegSplitter) parseSof(data []byte) (sof *SofSegment, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + stream := bytes.NewBuffer(data) + buffer := bufio.NewReader(stream) + + bitsPerSample, err := buffer.ReadByte() + log.PanicIf(err) + + height := uint16(0) + err = binary.Read(buffer, binary.BigEndian, &height) + log.PanicIf(err) + + width := uint16(0) + err = binary.Read(buffer, binary.BigEndian, &width) + log.PanicIf(err) + + componentCount, err := buffer.ReadByte() + log.PanicIf(err) + + sof = &SofSegment{ + BitsPerSample: bitsPerSample, + Width: width, + Height: height, + ComponentCount: componentCount, + } + + return sof, nil +} + +func (js *JpegSplitter) parseAppData(markerId byte, data []byte) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + return nil +} + +func (js *JpegSplitter) handleSegment(markerId byte, markerName string, headerSize int, payload []byte) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + cloned := make([]byte, len(payload)) + copy(cloned, payload) + + s := &Segment{ + MarkerId: markerId, + MarkerName: markerName, + Offset: js.currentOffset, + Data: cloned, + } + + jpegLogger.Debugf(nil, "Encountered marker (0x%02x) [%s] at offset (%d)", markerId, markerName, js.currentOffset) + + js.currentOffset += headerSize + len(payload) + + js.segments.Add(s) + + sv, ok := js.visitor.(SegmentVisitor) + if ok == true { + err = sv.HandleSegment(js.lastMarkerId, js.lastMarkerName, js.counter, js.lastIsScanData) + log.PanicIf(err) + } + + if markerId >= MARKER_SOF0 && markerId <= MARKER_SOF15 { + ssv, ok := js.visitor.(SofSegmentVisitor) + if ok == true { + sof, err := js.parseSof(payload) + log.PanicIf(err) + + err = ssv.HandleSof(sof) + log.PanicIf(err) + } + } else if markerId >= MARKER_APP0 && markerId <= MARKER_APP15 { + err := js.parseAppData(markerId, payload) + log.PanicIf(err) + } + + return nil +} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/testing_common.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/testing_common.go new file mode 100644 index 000000000..e7169c2f0 --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/testing_common.go @@ -0,0 +1,73 @@ +package jpegstructure + +import ( + "os" + "path" + + "github.com/dsoprea/go-logging" +) + +var ( + testImageRelFilepath = "NDM_8901.jpg" +) + +var ( + moduleRootPath = "" + assetsPath = "" +) + +// GetModuleRootPath returns the root-path of the module. +func GetModuleRootPath() string { + if moduleRootPath == "" { + moduleRootPath = os.Getenv("JPEG_MODULE_ROOT_PATH") + if moduleRootPath != "" { + return moduleRootPath + } + + currentWd, err := os.Getwd() + log.PanicIf(err) + + currentPath := currentWd + visited := make([]string, 0) + + for { + tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") + + _, err := os.Stat(tryStampFilepath) + if err != nil && os.IsNotExist(err) != true { + log.Panic(err) + } else if err == nil { + break + } + + visited = append(visited, tryStampFilepath) + + currentPath = path.Dir(currentPath) + if currentPath == "/" { + log.Panicf("could not find module-root: %v", visited) + } + } + + moduleRootPath = currentPath + } + + return moduleRootPath +} + +// GetTestAssetsPath returns the path of the test-assets. +func GetTestAssetsPath() string { + if assetsPath == "" { + moduleRootPath := GetModuleRootPath() + assetsPath = path.Join(moduleRootPath, "assets") + } + + return assetsPath +} + +// GetTestImageFilepath returns the file-path of the common test-image. +func GetTestImageFilepath() string { + assetsPath := GetTestAssetsPath() + filepath := path.Join(assetsPath, testImageRelFilepath) + + return filepath +} diff --git a/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/utility.go b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/utility.go new file mode 100644 index 000000000..1c618ba6d --- /dev/null +++ b/vendor/github.com/dsoprea/go-jpeg-image-structure/v2/utility.go @@ -0,0 +1,110 @@ +package jpegstructure + +import ( + "bytes" + "fmt" + "sort" + "strings" + + "github.com/dsoprea/go-logging" + "github.com/go-xmlfmt/xmlfmt" +) + +// DumpBytes prints the hex for a given byte-slice. +func DumpBytes(data []byte) { + fmt.Printf("DUMP: ") + for _, x := range data { + fmt.Printf("%02x ", x) + } + + fmt.Printf("\n") +} + +// DumpBytesClause prints a Go-formatted byte-slice expression. +func DumpBytesClause(data []byte) { + fmt.Printf("DUMP: ") + + fmt.Printf("[]byte { ") + + for i, x := range data { + fmt.Printf("0x%02x", x) + + if i < len(data)-1 { + fmt.Printf(", ") + } + } + + fmt.Printf(" }\n") +} + +// DumpBytesToString returns a string of hex-encoded bytes. +func DumpBytesToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteRune(' ') + log.PanicIf(err) + } + } + + return b.String() +} + +// DumpBytesClauseToString returns a string of Go-formatted byte values. +func DumpBytesClauseToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteString(", ") + log.PanicIf(err) + } + } + + return b.String() +} + +// FormatXml prettifies XML data. +func FormatXml(raw string) (formatted string, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + formatted = xmlfmt.FormatXML(raw, " ", " ") + formatted = strings.TrimSpace(formatted) + + return formatted, nil +} + +// SortStringStringMap sorts a string-string dictionary and returns it as a list +// of 2-tuples. +func SortStringStringMap(data map[string]string) (sorted [][2]string) { + // Sort keys. + + sortedKeys := make([]string, len(data)) + i := 0 + for key := range data { + sortedKeys[i] = key + i++ + } + + sort.Strings(sortedKeys) + + // Build result. + + sorted = make([][2]string, len(sortedKeys)) + for i, key := range sortedKeys { + sorted[i] = [2]string{key, data[key]} + } + + return sorted +} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/.MODULE_ROOT b/vendor/github.com/dsoprea/go-png-image-structure/.MODULE_ROOT deleted file mode 100644 index e69de29bb..000000000 diff --git a/vendor/github.com/dsoprea/go-png-image-structure/.travis.yml b/vendor/github.com/dsoprea/go-png-image-structure/.travis.yml deleted file mode 100644 index fdeab54e1..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -go: - - master - - stable - - "1.14" - - "1.13" - - "1.12" -env: - - GO111MODULE=on -install: - - go get -t ./... -script: -# v1 - - go test -v . -# v2 - - cd v2 - - go test -v . - - cd .. -after_success: - - cd v2 - - curl -s https://codecov.io/bash | bash diff --git a/vendor/github.com/dsoprea/go-png-image-structure/LICENSE b/vendor/github.com/dsoprea/go-png-image-structure/LICENSE deleted file mode 100644 index 163291ed6..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT LICENSE - -Copyright 2020 Dustin Oprea - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-png-image-structure/README.md b/vendor/github.com/dsoprea/go-png-image-structure/README.md deleted file mode 100644 index 46297a2a7..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/README.md +++ /dev/null @@ -1,8 +0,0 @@ -[![Build Status](https://travis-ci.org/dsoprea/go-png-image-structure.svg?branch=master)](https://travis-ci.org/dsoprea/go-png-image-structure) -[![codecov](https://codecov.io/gh/dsoprea/go-png-image-structure/branch/master/graph/badge.svg)](https://codecov.io/gh/dsoprea/go-png-image-structure) -[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-png-image-structure/v2)](https://goreportcard.com/report/github.com/dsoprea/go-png-image-structure/v2) -[![GoDoc](https://godoc.org/github.com/dsoprea/go-png-image-structure/v2?status.svg)](https://godoc.org/github.com/dsoprea/go-png-image-structure/v2) - -## Overview - -Parse raw PNG data into individual chunks. Parse/modify EXIF data and write an updated image. diff --git a/vendor/github.com/dsoprea/go-png-image-structure/chunk_decoder.go b/vendor/github.com/dsoprea/go-png-image-structure/chunk_decoder.go deleted file mode 100644 index 1358c3df2..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/chunk_decoder.go +++ /dev/null @@ -1,89 +0,0 @@ -package pngstructure - -import ( - "fmt" - "bytes" - - "encoding/binary" - - "github.com/dsoprea/go-logging" -) - -type ChunkDecoder struct { - -} - -func NewChunkDecoder() *ChunkDecoder { - return new(ChunkDecoder) -} - -func (cd *ChunkDecoder) Decode(c *Chunk) (decoded interface{}, err error) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - switch c.Type { - case "IHDR": - ihdr, err := cd.decodeIHDR(c) - log.PanicIf(err) - - return ihdr, nil - } - - // We don't decode this particular type. - return nil, nil -} - - -type ChunkIHDR struct { - Width uint32 - Height uint32 - BitDepth uint8 - ColorType uint8 - CompressionMethod uint8 - FilterMethod uint8 - InterlaceMethod uint8 -} - -func (ihdr *ChunkIHDR) String() string { - return fmt.Sprintf("IHDR", ihdr.Width, ihdr.Height, ihdr.BitDepth, ihdr.ColorType, ihdr.CompressionMethod, ihdr.FilterMethod, ihdr.InterlaceMethod) -} - -func (cd *ChunkDecoder) decodeIHDR(c *Chunk) (ihdr *ChunkIHDR, err error) { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - b := bytes.NewBuffer(c.Data) - - ihdr = new(ChunkIHDR) - - err = binary.Read(b, binary.BigEndian, &ihdr.Width) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.Height) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.BitDepth) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.ColorType) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.CompressionMethod) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.FilterMethod) - log.PanicIf(err) - - err = binary.Read(b, binary.BigEndian, &ihdr.InterlaceMethod) - log.PanicIf(err) - - return ihdr, nil -} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/media_parser.go b/vendor/github.com/dsoprea/go-png-image-structure/media_parser.go deleted file mode 100644 index f7467593f..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/media_parser.go +++ /dev/null @@ -1,106 +0,0 @@ -package pngstructure - -import ( - "bufio" - "bytes" - "io" - "os" - - "github.com/dsoprea/go-logging" - "github.com/dsoprea/go-utility/image" -) - -// PngMediaParser knows how to parse a PNG stream. -type PngMediaParser struct { -} - -// NewPngMediaParser returns a new `PngMediaParser` struct. -func NewPngMediaParser() *PngMediaParser { - - // TODO(dustin): Add test - - return new(PngMediaParser) -} - -// Parse parses a PNG stream given a `io.ReadSeeker`. -func (pmp *PngMediaParser) Parse(rs io.ReadSeeker, size int) (mc riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - ps := NewPngSplitter() - - err = ps.readHeader(rs) - log.PanicIf(err) - - s := bufio.NewScanner(rs) - - // Since each segment can be any size, our buffer must be allowed to grow - // as large as the file. - buffer := []byte{} - s.Buffer(buffer, size) - s.Split(ps.Split) - - for s.Scan() != false { - } - log.PanicIf(s.Err()) - - return ps.Chunks(), nil -} - -// ParseFile parses a PNG stream given a file-path. -func (pmp *PngMediaParser) ParseFile(filepath string) (mc riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - f, err := os.Open(filepath) - log.PanicIf(err) - - defer f.Close() - - stat, err := f.Stat() - log.PanicIf(err) - - size := stat.Size() - - chunks, err := pmp.Parse(f, int(size)) - log.PanicIf(err) - - return chunks, nil -} - -// ParseBytes parses a PNG stream given a byte-slice. -func (pmp *PngMediaParser) ParseBytes(data []byte) (mc riimage.MediaContext, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // TODO(dustin): Add test - - br := bytes.NewReader(data) - - chunks, err := pmp.Parse(br, len(data)) - log.PanicIf(err) - - return chunks, nil -} - -// LooksLikeFormat returns a boolean indicating whether the stream looks like a -// PNG image. -func (pmp *PngMediaParser) LooksLikeFormat(data []byte) bool { - return bytes.Compare(data[:len(PngSignature)], PngSignature[:]) == 0 -} - -var ( - // Enforce interface conformance. - _ riimage.MediaParser = new(PngMediaParser) -) diff --git a/vendor/github.com/dsoprea/go-png-image-structure/png.go b/vendor/github.com/dsoprea/go-png-image-structure/png.go deleted file mode 100644 index 203bbf562..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/png.go +++ /dev/null @@ -1,414 +0,0 @@ -package pngstructure - -import ( - "bytes" - "errors" - "fmt" - "io" - - "encoding/binary" - "hash/crc32" - - "github.com/dsoprea/go-exif/v2" - "github.com/dsoprea/go-logging" - "github.com/dsoprea/go-utility/image" -) - -var ( - PngSignature = [8]byte{137, 'P', 'N', 'G', '\r', '\n', 26, '\n'} - EXifChunkType = "eXIf" - IHDRChunkType = "IHDR" -) - -var ( - ErrNotPng = errors.New("not png data") - ErrNoExif = errors.New("file does not have EXIF") - ErrCrcFailure = errors.New("crc failure") -) - -// ChunkSlice encapsulates a slice of chunks. -type ChunkSlice struct { - chunks []*Chunk -} - -func NewChunkSlice(chunks []*Chunk) *ChunkSlice { - if len(chunks) == 0 { - log.Panicf("ChunkSlice must be initialized with at least one chunk (IHDR)") - } else if chunks[0].Type != IHDRChunkType { - log.Panicf("first chunk in any ChunkSlice must be an IHDR") - } - - return &ChunkSlice{ - chunks: chunks, - } -} - -func NewPngChunkSlice() *ChunkSlice { - - ihdrChunk := &Chunk{ - Type: IHDRChunkType, - } - - ihdrChunk.UpdateCrc32() - - return NewChunkSlice([]*Chunk{ihdrChunk}) -} - -func (cs *ChunkSlice) String() string { - return fmt.Sprintf("ChunkSlize", len(cs.chunks)) -} - -// Chunks exposes the actual slice. -func (cs *ChunkSlice) Chunks() []*Chunk { - return cs.chunks -} - -// Write encodes and writes all chunks. -func (cs *ChunkSlice) WriteTo(w io.Writer) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - _, err = w.Write(PngSignature[:]) - log.PanicIf(err) - - // TODO(dustin): !! This should respect the safe-to-copy characteristic. - for _, c := range cs.chunks { - _, err := c.WriteTo(w) - log.PanicIf(err) - } - - return nil -} - -// Index returns a map of chunk types to chunk slices, grouping all like chunks. -func (cs *ChunkSlice) Index() (index map[string][]*Chunk) { - index = make(map[string][]*Chunk) - for _, c := range cs.chunks { - if grouped, found := index[c.Type]; found == true { - index[c.Type] = append(grouped, c) - } else { - index[c.Type] = []*Chunk{c} - } - } - - return index -} - -// FindExif returns the the segment that hosts the EXIF data. -func (cs *ChunkSlice) FindExif() (chunk *Chunk, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - index := cs.Index() - - if chunks, found := index[EXifChunkType]; found == true { - return chunks[0], nil - } - - log.Panic(ErrNoExif) - - // Never called. - return nil, nil -} - -// Exif returns an `exif.Ifd` instance with the existing tags. -func (cs *ChunkSlice) Exif() (rootIfd *exif.Ifd, data []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - chunk, err := cs.FindExif() - log.PanicIf(err) - - im := exif.NewIfdMappingWithStandard() - ti := exif.NewTagIndex() - - // TODO(dustin): Refactor and support `exif.GetExifData()`. - - _, index, err := exif.Collect(im, ti, chunk.Data) - log.PanicIf(err) - - return index.RootIfd, chunk.Data, nil -} - -// ConstructExifBuilder returns an `exif.IfdBuilder` instance (needed for -// modifying) preloaded with all existing tags. -func (cs *ChunkSlice) ConstructExifBuilder() (rootIb *exif.IfdBuilder, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - rootIfd, _, err := cs.Exif() - log.PanicIf(err) - - ib := exif.NewIfdBuilderFromExistingChain(rootIfd) - - return ib, nil -} - -// SetExif encodes and sets EXIF data into this segment. -func (cs *ChunkSlice) SetExif(ib *exif.IfdBuilder) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // Encode. - - ibe := exif.NewIfdByteEncoder() - - exifData, err := ibe.EncodeToExif(ib) - log.PanicIf(err) - - // Set. - - exifChunk, err := cs.FindExif() - if err == nil { - // EXIF chunk already exists. - - exifChunk.Data = exifData - exifChunk.Length = uint32(len(exifData)) - } else { - if log.Is(err, ErrNoExif) != true { - log.Panic(err) - } - - // Add a EXIF chunk for the first time. - - exifChunk = &Chunk{ - Type: EXifChunkType, - Data: exifData, - Length: uint32(len(exifData)), - } - - // Insert it after the IHDR chunk (it's a reliably appropriate place to - // put it). - cs.chunks = append(cs.chunks[:1], append([]*Chunk{exifChunk}, cs.chunks[1:]...)...) - } - - exifChunk.UpdateCrc32() - - return nil -} - -// PngSplitter hosts the princpal `Split()` method uses by `bufio.Scanner`. -type PngSplitter struct { - chunks []*Chunk - currentOffset int - - doCheckCrc bool - crcErrors []string -} - -func (ps *PngSplitter) Chunks() *ChunkSlice { - return NewChunkSlice(ps.chunks) -} - -func (ps *PngSplitter) DoCheckCrc(doCheck bool) { - ps.doCheckCrc = doCheck -} - -func (ps *PngSplitter) CrcErrors() []string { - return ps.crcErrors -} - -func NewPngSplitter() *PngSplitter { - return &PngSplitter{ - chunks: make([]*Chunk, 0), - doCheckCrc: true, - crcErrors: make([]string, 0), - } -} - -// Chunk describes a single chunk. -type Chunk struct { - Offset int - Length uint32 - Type string - Data []byte - Crc uint32 -} - -func (c *Chunk) String() string { - return fmt.Sprintf("Chunk", c.Offset, c.Length, c.Type, c.Crc) -} - -func calculateCrc32(chunk *Chunk) uint32 { - c := crc32.NewIEEE() - - c.Write([]byte(chunk.Type)) - c.Write(chunk.Data) - - return c.Sum32() -} - -func (c *Chunk) UpdateCrc32() { - c.Crc = calculateCrc32(c) -} - -func (c *Chunk) CheckCrc32() bool { - expected := calculateCrc32(c) - return c.Crc == expected -} - -// Bytes encodes and returns the bytes for this chunk. -func (c *Chunk) Bytes() []byte { - defer func() { - if state := recover(); state != nil { - err := log.Wrap(state.(error)) - log.Panic(err) - } - }() - - if len(c.Data) != int(c.Length) { - log.Panicf("length of data not correct") - } - - preallocated := make([]byte, 0, 4+4+c.Length+4) - b := bytes.NewBuffer(preallocated) - - err := binary.Write(b, binary.BigEndian, c.Length) - log.PanicIf(err) - - _, err = b.Write([]byte(c.Type)) - log.PanicIf(err) - - if c.Data != nil { - _, err = b.Write(c.Data) - log.PanicIf(err) - } - - err = binary.Write(b, binary.BigEndian, c.Crc) - log.PanicIf(err) - - return b.Bytes() -} - -// Write encodes and writes the bytes for this chunk. -func (c *Chunk) WriteTo(w io.Writer) (count int, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - if len(c.Data) != int(c.Length) { - log.Panicf("length of data not correct") - } - - err = binary.Write(w, binary.BigEndian, c.Length) - log.PanicIf(err) - - _, err = w.Write([]byte(c.Type)) - log.PanicIf(err) - - _, err = w.Write(c.Data) - log.PanicIf(err) - - err = binary.Write(w, binary.BigEndian, c.Crc) - log.PanicIf(err) - - return 4 + len(c.Type) + len(c.Data) + 4, nil -} - -// readHeader verifies that the PNG header bytes appear next. -func (ps *PngSplitter) readHeader(r io.Reader) (err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - len_ := len(PngSignature) - header := make([]byte, len_) - - _, err = r.Read(header) - log.PanicIf(err) - - ps.currentOffset += len_ - - if bytes.Compare(header, PngSignature[:]) != 0 { - log.Panic(ErrNotPng) - } - - return nil -} - -// Split fulfills the `bufio.SplitFunc` function definition for -// `bufio.Scanner`. -func (ps *PngSplitter) Split(data []byte, atEOF bool) (advance int, token []byte, err error) { - defer func() { - if state := recover(); state != nil { - err = log.Wrap(state.(error)) - } - }() - - // We might have more than one chunk's worth, and, if `atEOF` is true, we - // won't be called again. We'll repeatedly try to read additional chunks, - // but, when we run out of the data we were given then we'll return the - // number of bytes fo rthe chunks we've already completely read. Then, - // we'll be called again from theend ofthose bytes, at which point we'll - // indicate that we don't yet have enough for another chunk, and we should - // be then called with more. - for { - len_ := len(data) - if len_ < 8 { - return advance, nil, nil - } - - length := binary.BigEndian.Uint32(data[:4]) - type_ := string(data[4:8]) - chunkSize := (8 + int(length) + 4) - - if len_ < chunkSize { - return advance, nil, nil - } - - crcIndex := 8 + length - crc := binary.BigEndian.Uint32(data[crcIndex : crcIndex+4]) - - content := make([]byte, length) - copy(content, data[8:8+length]) - - c := &Chunk{ - Length: length, - Type: type_, - Data: content, - Crc: crc, - Offset: ps.currentOffset, - } - - ps.chunks = append(ps.chunks, c) - - if c.CheckCrc32() == false { - ps.crcErrors = append(ps.crcErrors, type_) - - if ps.doCheckCrc == true { - log.Panic(ErrCrcFailure) - } - } - - advance += chunkSize - ps.currentOffset += chunkSize - - data = data[chunkSize:] - } - - return advance, nil, nil -} - -var ( - // Enforce interface conformance. - _ riimage.MediaContext = new(ChunkSlice) -) diff --git a/vendor/github.com/dsoprea/go-png-image-structure/testing_common.go b/vendor/github.com/dsoprea/go-png-image-structure/testing_common.go deleted file mode 100644 index e7dad11af..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/testing_common.go +++ /dev/null @@ -1,64 +0,0 @@ -package pngstructure - -import ( - "os" - "path" - - "github.com/dsoprea/go-logging" -) - -var ( - assetsPath = "" -) - -func getModuleRootPath() string { - moduleRootPath := os.Getenv("PNG_MODULE_ROOT_PATH") - if moduleRootPath != "" { - return moduleRootPath - } - - currentWd, err := os.Getwd() - log.PanicIf(err) - - currentPath := currentWd - visited := make([]string, 0) - - for { - tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") - - _, err := os.Stat(tryStampFilepath) - if err != nil && os.IsNotExist(err) != true { - log.Panic(err) - } else if err == nil { - break - } - - visited = append(visited, tryStampFilepath) - - currentPath = path.Dir(currentPath) - if currentPath == "/" { - log.Panicf("could not find module-root: %v", visited) - } - } - - return currentPath -} - -func getTestAssetsPath() string { - if assetsPath == "" { - moduleRootPath := getModuleRootPath() - assetsPath = path.Join(moduleRootPath, "assets") - } - - return assetsPath -} - -func getTestBasicImageFilepath() string { - assetsPath := getTestAssetsPath() - return path.Join(assetsPath, "libpng.png") -} - -func getTestExifImageFilepath() string { - assetsPath := getTestAssetsPath() - return path.Join(assetsPath, "exif.png") -} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/utility.go b/vendor/github.com/dsoprea/go-png-image-structure/utility.go deleted file mode 100644 index 9bfab14a4..000000000 --- a/vendor/github.com/dsoprea/go-png-image-structure/utility.go +++ /dev/null @@ -1,65 +0,0 @@ -package pngstructure - -import ( - "fmt" - "bytes" - - "github.com/dsoprea/go-logging" -) - -func DumpBytes(data []byte) { - fmt.Printf("DUMP: ") - for _, x := range data { - fmt.Printf("%02x ", x) - } - - fmt.Printf("\n") -} - -func DumpBytesClause(data []byte) { - fmt.Printf("DUMP: ") - - fmt.Printf("[]byte { ") - - for i, x := range data { - fmt.Printf("0x%02x", x) - - if i < len(data) - 1 { - fmt.Printf(", ") - } - } - - fmt.Printf(" }\n") -} - -func DumpBytesToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("%02x", x)) - log.PanicIf(err) - - if i < len(data) - 1 { - _, err := b.WriteRune(' ') - log.PanicIf(err) - } - } - - return b.String() -} - -func DumpBytesClauseToString(data []byte) string { - b := new(bytes.Buffer) - - for i, x := range data { - _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) - log.PanicIf(err) - - if i < len(data) - 1 { - _, err := b.WriteString(", ") - log.PanicIf(err) - } - } - - return b.String() -} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/.MODULE_ROOT b/vendor/github.com/dsoprea/go-png-image-structure/v2/.MODULE_ROOT new file mode 100644 index 000000000..e69de29bb diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/LICENSE b/vendor/github.com/dsoprea/go-png-image-structure/v2/LICENSE new file mode 100644 index 000000000..163291ed6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/LICENSE @@ -0,0 +1,9 @@ +MIT LICENSE + +Copyright 2020 Dustin Oprea + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/chunk_decoder.go b/vendor/github.com/dsoprea/go-png-image-structure/v2/chunk_decoder.go new file mode 100644 index 000000000..b5e0b1b16 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/chunk_decoder.go @@ -0,0 +1,87 @@ +package pngstructure + +import ( + "bytes" + "fmt" + + "encoding/binary" + + "github.com/dsoprea/go-logging" +) + +type ChunkDecoder struct { +} + +func NewChunkDecoder() *ChunkDecoder { + return new(ChunkDecoder) +} + +func (cd *ChunkDecoder) Decode(c *Chunk) (decoded interface{}, err error) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + switch c.Type { + case "IHDR": + ihdr, err := cd.decodeIHDR(c) + log.PanicIf(err) + + return ihdr, nil + } + + // We don't decode this particular type. + return nil, nil +} + +type ChunkIHDR struct { + Width uint32 + Height uint32 + BitDepth uint8 + ColorType uint8 + CompressionMethod uint8 + FilterMethod uint8 + InterlaceMethod uint8 +} + +func (ihdr *ChunkIHDR) String() string { + return fmt.Sprintf("IHDR", ihdr.Width, ihdr.Height, ihdr.BitDepth, ihdr.ColorType, ihdr.CompressionMethod, ihdr.FilterMethod, ihdr.InterlaceMethod) +} + +func (cd *ChunkDecoder) decodeIHDR(c *Chunk) (ihdr *ChunkIHDR, err error) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + b := bytes.NewBuffer(c.Data) + + ihdr = new(ChunkIHDR) + + err = binary.Read(b, binary.BigEndian, &ihdr.Width) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.Height) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.BitDepth) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.ColorType) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.CompressionMethod) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.FilterMethod) + log.PanicIf(err) + + err = binary.Read(b, binary.BigEndian, &ihdr.InterlaceMethod) + log.PanicIf(err) + + return ihdr, nil +} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/media_parser.go b/vendor/github.com/dsoprea/go-png-image-structure/v2/media_parser.go new file mode 100644 index 000000000..c0e287365 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/media_parser.go @@ -0,0 +1,118 @@ +package pngstructure + +import ( + "bufio" + "bytes" + "image" + "io" + "os" + + "image/png" + + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-utility/v2/image" +) + +// PngMediaParser knows how to parse a PNG stream. +type PngMediaParser struct { +} + +// NewPngMediaParser returns a new `PngMediaParser` struct. +func NewPngMediaParser() *PngMediaParser { + + // TODO(dustin): Add test + + return new(PngMediaParser) +} + +// Parse parses a PNG stream given a `io.ReadSeeker`. +func (pmp *PngMediaParser) Parse(rs io.ReadSeeker, size int) (mc riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + ps := NewPngSplitter() + + err = ps.readHeader(rs) + log.PanicIf(err) + + s := bufio.NewScanner(rs) + + // Since each segment can be any size, our buffer must be allowed to grow + // as large as the file. + buffer := []byte{} + s.Buffer(buffer, size) + s.Split(ps.Split) + + for s.Scan() != false { + } + + log.PanicIf(s.Err()) + + return ps.Chunks(), nil +} + +// ParseFile parses a PNG stream given a file-path. +func (pmp *PngMediaParser) ParseFile(filepath string) (mc riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + f, err := os.Open(filepath) + log.PanicIf(err) + + defer f.Close() + + stat, err := f.Stat() + log.PanicIf(err) + + size := stat.Size() + + chunks, err := pmp.Parse(f, int(size)) + log.PanicIf(err) + + return chunks, nil +} + +// ParseBytes parses a PNG stream given a byte-slice. +func (pmp *PngMediaParser) ParseBytes(data []byte) (mc riimage.MediaContext, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // TODO(dustin): Add test + + br := bytes.NewReader(data) + + chunks, err := pmp.Parse(br, len(data)) + log.PanicIf(err) + + return chunks, nil +} + +// LooksLikeFormat returns a boolean indicating whether the stream looks like a +// PNG image. +func (pmp *PngMediaParser) LooksLikeFormat(data []byte) bool { + return bytes.Compare(data[:len(PngSignature)], PngSignature[:]) == 0 +} + +// GetImage returns an image.Image-compatible struct. +func (pmp *PngMediaParser) GetImage(r io.Reader) (img image.Image, err error) { + img, err = png.Decode(r) + log.PanicIf(err) + + return img, nil +} + +var ( + // Enforce interface conformance. + _ riimage.MediaParser = new(PngMediaParser) +) diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/png.go b/vendor/github.com/dsoprea/go-png-image-structure/v2/png.go new file mode 100644 index 000000000..fbb022887 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/png.go @@ -0,0 +1,416 @@ +package pngstructure + +import ( + "bytes" + "errors" + "fmt" + "io" + + "encoding/binary" + "hash/crc32" + + "github.com/dsoprea/go-exif/v3" + "github.com/dsoprea/go-exif/v3/common" + "github.com/dsoprea/go-logging" + "github.com/dsoprea/go-utility/v2/image" +) + +var ( + PngSignature = [8]byte{137, 'P', 'N', 'G', '\r', '\n', 26, '\n'} + EXifChunkType = "eXIf" + IHDRChunkType = "IHDR" +) + +var ( + ErrNotPng = errors.New("not png data") + ErrCrcFailure = errors.New("crc failure") +) + +// ChunkSlice encapsulates a slice of chunks. +type ChunkSlice struct { + chunks []*Chunk +} + +func NewChunkSlice(chunks []*Chunk) *ChunkSlice { + if len(chunks) == 0 { + log.Panicf("ChunkSlice must be initialized with at least one chunk (IHDR)") + } else if chunks[0].Type != IHDRChunkType { + log.Panicf("first chunk in any ChunkSlice must be an IHDR") + } + + return &ChunkSlice{ + chunks: chunks, + } +} + +func NewPngChunkSlice() *ChunkSlice { + + ihdrChunk := &Chunk{ + Type: IHDRChunkType, + } + + ihdrChunk.UpdateCrc32() + + return NewChunkSlice([]*Chunk{ihdrChunk}) +} + +func (cs *ChunkSlice) String() string { + return fmt.Sprintf("ChunkSlize", len(cs.chunks)) +} + +// Chunks exposes the actual slice. +func (cs *ChunkSlice) Chunks() []*Chunk { + return cs.chunks +} + +// Write encodes and writes all chunks. +func (cs *ChunkSlice) WriteTo(w io.Writer) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + _, err = w.Write(PngSignature[:]) + log.PanicIf(err) + + // TODO(dustin): !! This should respect the safe-to-copy characteristic. + for _, c := range cs.chunks { + _, err := c.WriteTo(w) + log.PanicIf(err) + } + + return nil +} + +// Index returns a map of chunk types to chunk slices, grouping all like chunks. +func (cs *ChunkSlice) Index() (index map[string][]*Chunk) { + index = make(map[string][]*Chunk) + for _, c := range cs.chunks { + if grouped, found := index[c.Type]; found == true { + index[c.Type] = append(grouped, c) + } else { + index[c.Type] = []*Chunk{c} + } + } + + return index +} + +// FindExif returns the the segment that hosts the EXIF data. +func (cs *ChunkSlice) FindExif() (chunk *Chunk, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + index := cs.Index() + + if chunks, found := index[EXifChunkType]; found == true { + return chunks[0], nil + } + + log.Panic(exif.ErrNoExif) + + // Never called. + return nil, nil +} + +// Exif returns an `exif.Ifd` instance with the existing tags. +func (cs *ChunkSlice) Exif() (rootIfd *exif.Ifd, data []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + chunk, err := cs.FindExif() + log.PanicIf(err) + + im, err := exifcommon.NewIfdMappingWithStandard() + log.PanicIf(err) + + ti := exif.NewTagIndex() + + // TODO(dustin): Refactor and support `exif.GetExifData()`. + + _, index, err := exif.Collect(im, ti, chunk.Data) + log.PanicIf(err) + + return index.RootIfd, chunk.Data, nil +} + +// ConstructExifBuilder returns an `exif.IfdBuilder` instance (needed for +// modifying) preloaded with all existing tags. +func (cs *ChunkSlice) ConstructExifBuilder() (rootIb *exif.IfdBuilder, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + rootIfd, _, err := cs.Exif() + log.PanicIf(err) + + ib := exif.NewIfdBuilderFromExistingChain(rootIfd) + + return ib, nil +} + +// SetExif encodes and sets EXIF data into this segment. +func (cs *ChunkSlice) SetExif(ib *exif.IfdBuilder) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Encode. + + ibe := exif.NewIfdByteEncoder() + + exifData, err := ibe.EncodeToExif(ib) + log.PanicIf(err) + + // Set. + + exifChunk, err := cs.FindExif() + if err == nil { + // EXIF chunk already exists. + + exifChunk.Data = exifData + exifChunk.Length = uint32(len(exifData)) + } else { + if log.Is(err, exif.ErrNoExif) != true { + log.Panic(err) + } + + // Add a EXIF chunk for the first time. + + exifChunk = &Chunk{ + Type: EXifChunkType, + Data: exifData, + Length: uint32(len(exifData)), + } + + // Insert it after the IHDR chunk (it's a reliably appropriate place to + // put it). + cs.chunks = append(cs.chunks[:1], append([]*Chunk{exifChunk}, cs.chunks[1:]...)...) + } + + exifChunk.UpdateCrc32() + + return nil +} + +// PngSplitter hosts the princpal `Split()` method uses by `bufio.Scanner`. +type PngSplitter struct { + chunks []*Chunk + currentOffset int + + doCheckCrc bool + crcErrors []string +} + +func (ps *PngSplitter) Chunks() *ChunkSlice { + return NewChunkSlice(ps.chunks) +} + +func (ps *PngSplitter) DoCheckCrc(doCheck bool) { + ps.doCheckCrc = doCheck +} + +func (ps *PngSplitter) CrcErrors() []string { + return ps.crcErrors +} + +func NewPngSplitter() *PngSplitter { + return &PngSplitter{ + chunks: make([]*Chunk, 0), + doCheckCrc: true, + crcErrors: make([]string, 0), + } +} + +// Chunk describes a single chunk. +type Chunk struct { + Offset int + Length uint32 + Type string + Data []byte + Crc uint32 +} + +func (c *Chunk) String() string { + return fmt.Sprintf("Chunk", c.Offset, c.Length, c.Type, c.Crc) +} + +func calculateCrc32(chunk *Chunk) uint32 { + c := crc32.NewIEEE() + + c.Write([]byte(chunk.Type)) + c.Write(chunk.Data) + + return c.Sum32() +} + +func (c *Chunk) UpdateCrc32() { + c.Crc = calculateCrc32(c) +} + +func (c *Chunk) CheckCrc32() bool { + expected := calculateCrc32(c) + return c.Crc == expected +} + +// Bytes encodes and returns the bytes for this chunk. +func (c *Chunk) Bytes() []byte { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + if len(c.Data) != int(c.Length) { + log.Panicf("length of data not correct") + } + + preallocated := make([]byte, 0, 4+4+c.Length+4) + b := bytes.NewBuffer(preallocated) + + err := binary.Write(b, binary.BigEndian, c.Length) + log.PanicIf(err) + + _, err = b.Write([]byte(c.Type)) + log.PanicIf(err) + + if c.Data != nil { + _, err = b.Write(c.Data) + log.PanicIf(err) + } + + err = binary.Write(b, binary.BigEndian, c.Crc) + log.PanicIf(err) + + return b.Bytes() +} + +// Write encodes and writes the bytes for this chunk. +func (c *Chunk) WriteTo(w io.Writer) (count int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if len(c.Data) != int(c.Length) { + log.Panicf("length of data not correct") + } + + err = binary.Write(w, binary.BigEndian, c.Length) + log.PanicIf(err) + + _, err = w.Write([]byte(c.Type)) + log.PanicIf(err) + + _, err = w.Write(c.Data) + log.PanicIf(err) + + err = binary.Write(w, binary.BigEndian, c.Crc) + log.PanicIf(err) + + return 4 + len(c.Type) + len(c.Data) + 4, nil +} + +// readHeader verifies that the PNG header bytes appear next. +func (ps *PngSplitter) readHeader(r io.Reader) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + len_ := len(PngSignature) + header := make([]byte, len_) + + _, err = r.Read(header) + log.PanicIf(err) + + ps.currentOffset += len_ + + if bytes.Compare(header, PngSignature[:]) != 0 { + log.Panic(ErrNotPng) + } + + return nil +} + +// Split fulfills the `bufio.SplitFunc` function definition for +// `bufio.Scanner`. +func (ps *PngSplitter) Split(data []byte, atEOF bool) (advance int, token []byte, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // We might have more than one chunk's worth, and, if `atEOF` is true, we + // won't be called again. We'll repeatedly try to read additional chunks, + // but, when we run out of the data we were given then we'll return the + // number of bytes fo rthe chunks we've already completely read. Then, + // we'll be called again from theend ofthose bytes, at which point we'll + // indicate that we don't yet have enough for another chunk, and we should + // be then called with more. + for { + len_ := len(data) + if len_ < 8 { + return advance, nil, nil + } + + length := binary.BigEndian.Uint32(data[:4]) + type_ := string(data[4:8]) + chunkSize := (8 + int(length) + 4) + + if len_ < chunkSize { + return advance, nil, nil + } + + crcIndex := 8 + length + crc := binary.BigEndian.Uint32(data[crcIndex : crcIndex+4]) + + content := make([]byte, length) + copy(content, data[8:8+length]) + + c := &Chunk{ + Length: length, + Type: type_, + Data: content, + Crc: crc, + Offset: ps.currentOffset, + } + + ps.chunks = append(ps.chunks, c) + + if c.CheckCrc32() == false { + ps.crcErrors = append(ps.crcErrors, type_) + + if ps.doCheckCrc == true { + log.Panic(ErrCrcFailure) + } + } + + advance += chunkSize + ps.currentOffset += chunkSize + + data = data[chunkSize:] + } + + return advance, nil, nil +} + +var ( + // Enforce interface conformance. + _ riimage.MediaContext = new(ChunkSlice) +) diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/testing_common.go b/vendor/github.com/dsoprea/go-png-image-structure/v2/testing_common.go new file mode 100644 index 000000000..9df13a858 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/testing_common.go @@ -0,0 +1,64 @@ +package pngstructure + +import ( + "os" + "path" + + "github.com/dsoprea/go-logging" +) + +var ( + assetsPath = "" +) + +func getModuleRootPath() string { + moduleRootPath := os.Getenv("PNG_MODULE_ROOT_PATH") + if moduleRootPath != "" { + return moduleRootPath + } + + currentWd, err := os.Getwd() + log.PanicIf(err) + + currentPath := currentWd + visited := make([]string, 0) + + for { + tryStampFilepath := path.Join(currentPath, ".MODULE_ROOT") + + _, err := os.Stat(tryStampFilepath) + if err != nil && os.IsNotExist(err) != true { + log.Panic(err) + } else if err == nil { + break + } + + visited = append(visited, tryStampFilepath) + + currentPath = path.Dir(currentPath) + if currentPath == "/" { + log.Panicf("could not find module-root: %v", visited) + } + } + + return currentPath +} + +func getTestAssetsPath() string { + if assetsPath == "" { + moduleRootPath := getModuleRootPath() + assetsPath = path.Join(moduleRootPath, "assets") + } + + return assetsPath +} + +func getTestBasicImageFilepath() string { + assetsPath := getTestAssetsPath() + return path.Join(assetsPath, "libpng.png") +} + +func getTestExifImageFilepath() string { + assetsPath := getTestAssetsPath() + return path.Join(assetsPath, "exif.png") +} diff --git a/vendor/github.com/dsoprea/go-png-image-structure/v2/utility.go b/vendor/github.com/dsoprea/go-png-image-structure/v2/utility.go new file mode 100644 index 000000000..dbff145a6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-png-image-structure/v2/utility.go @@ -0,0 +1,65 @@ +package pngstructure + +import ( + "bytes" + "fmt" + + "github.com/dsoprea/go-logging" +) + +func DumpBytes(data []byte) { + fmt.Printf("DUMP: ") + for _, x := range data { + fmt.Printf("%02x ", x) + } + + fmt.Printf("\n") +} + +func DumpBytesClause(data []byte) { + fmt.Printf("DUMP: ") + + fmt.Printf("[]byte { ") + + for i, x := range data { + fmt.Printf("0x%02x", x) + + if i < len(data)-1 { + fmt.Printf(", ") + } + } + + fmt.Printf(" }\n") +} + +func DumpBytesToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteRune(' ') + log.PanicIf(err) + } + } + + return b.String() +} + +func DumpBytesClauseToString(data []byte) string { + b := new(bytes.Buffer) + + for i, x := range data { + _, err := b.WriteString(fmt.Sprintf("0x%02x", x)) + log.PanicIf(err) + + if i < len(data)-1 { + _, err := b.WriteString(", ") + log.PanicIf(err) + } + } + + return b.String() +} diff --git a/vendor/github.com/dsoprea/go-utility/LICENSE b/vendor/github.com/dsoprea/go-utility/LICENSE deleted file mode 100644 index 8941063e1..000000000 --- a/vendor/github.com/dsoprea/go-utility/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 2019 Random Ingenuity InformationWorks - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-utility/image/README.md b/vendor/github.com/dsoprea/go-utility/image/README.md deleted file mode 100644 index 1509ff666..000000000 --- a/vendor/github.com/dsoprea/go-utility/image/README.md +++ /dev/null @@ -1,9 +0,0 @@ -[![GoDoc](https://godoc.org/github.com/dsoprea/go-utility/image?status.svg)](https://godoc.org/github.com/dsoprea/go-utility/image) -[![Build Status](https://travis-ci.org/dsoprea/go-utility.svg?branch=master)](https://travis-ci.org/dsoprea/go-utility) -[![Coverage Status](https://coveralls.io/repos/github/dsoprea/go-utility/badge.svg?branch=master)](https://coveralls.io/github/dsoprea/go-utility?branch=master) -[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-utility)](https://goreportcard.com/report/github.com/dsoprea/go-utility) - -# media_parser_type - -Common image-parsing interfaces. Used for JPEG, PNG, and HEIC parsers used by -go-exif-knife. diff --git a/vendor/github.com/dsoprea/go-utility/image/media_parser_type.go b/vendor/github.com/dsoprea/go-utility/image/media_parser_type.go deleted file mode 100644 index b7956ea17..000000000 --- a/vendor/github.com/dsoprea/go-utility/image/media_parser_type.go +++ /dev/null @@ -1,34 +0,0 @@ -package riimage - -import ( - "io" - - "github.com/dsoprea/go-exif/v2" -) - -// MediaContext is an accessor that knows how to extract specific metadata from -// the media. -type MediaContext interface { - // Exif returns the EXIF's root IFD. - Exif() (rootIfd *exif.Ifd, data []byte, err error) -} - -// MediaParser prescribes a specific structure for the parser types that are -// imported from other projects. We don't use it directly, but we use this to -// impose structure. -type MediaParser interface { - // Parse parses a stream using an `io.ReadSeeker`. `mc` should *actually* be - // a `ExifContext`. - Parse(r io.ReadSeeker, size int) (mc MediaContext, err error) - - // ParseFile parses a stream using a file. `mc` should *actually* be a - // `ExifContext`. - ParseFile(filepath string) (mc MediaContext, err error) - - // ParseBytes parses a stream direct from bytes. `mc` should *actually* be - // a `ExifContext`. - ParseBytes(data []byte) (mc MediaContext, err error) - - // Parses the data to determine if it's a compatible format. - LooksLikeFormat(data []byte) bool -} diff --git a/vendor/github.com/dsoprea/go-utility/v2/LICENSE b/vendor/github.com/dsoprea/go-utility/v2/LICENSE new file mode 100644 index 000000000..8941063e1 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/LICENSE @@ -0,0 +1,7 @@ +Copyright 2019 Random Ingenuity InformationWorks + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md b/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md new file mode 100644 index 000000000..eb03fea7c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md @@ -0,0 +1,64 @@ +[![GoDoc](https://godoc.org/github.com/dsoprea/go-utility/filesystem?status.svg)](https://godoc.org/github.com/dsoprea/go-utility/filesystem) +[![Build Status](https://travis-ci.org/dsoprea/go-utility.svg?branch=master)](https://travis-ci.org/dsoprea/go-utility) +[![Coverage Status](https://coveralls.io/repos/github/dsoprea/go-utility/badge.svg?branch=master)](https://coveralls.io/github/dsoprea/go-utility?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-utility)](https://goreportcard.com/report/github.com/dsoprea/go-utility) + +# bounceback + +An `io.ReadSeeker` and `io.WriteSeeker` that returns to the right place before +reading or writing. Useful when the same file resource is being reused for reads +or writes throughout that file. + +# list_files + +A recursive path walker that supports filters. + +# seekable_buffer + +A memory structure that satisfies `io.ReadWriteSeeker`. + +# copy_bytes_between_positions + +Given an `io.ReadWriteSeeker`, copy N bytes from one position to an earlier +position. + +# read_counter, write_counter + +Wrap `io.Reader` and `io.Writer` structs in order to report how many bytes were +transferred. + +# readseekwritecloser + +Provides the ReadWriteSeekCloser interface that combines a RWS and a Closer. +Also provides a no-op wrapper to augment a plain RWS with a closer. + +# boundedreadwriteseek + +Wraps a ReadWriteSeeker such that no seeks can be at an offset less than a +specific-offset. + +# calculateseek + +Provides a reusable function with which to calculate seek offsets. + +# progress_wrapper + +Provides `io.Reader` and `io.Writer` wrappers that also trigger callbacks after +each call. The reader wrapper also invokes the callback upon EOF. + +# does_exist + +Check whether a file/directory exists using a file-path. + +# graceful_copy + +Do a copy but correctly handle short-writes and reads that might return a non- +zero read count *and* EOF. + +# readseeker_to_readerat + +A wrapper that allows an `io.ReadSeeker` to be used as a `io.ReaderAt`. + +# simplefileinfo + +An implementation of `os.FileInfo` to support testing. diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go new file mode 100644 index 000000000..1112a10ef --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go @@ -0,0 +1,273 @@ +package rifs + +import ( + "fmt" + "io" + + "github.com/dsoprea/go-logging" +) + +// BouncebackStats describes operation counts. +type BouncebackStats struct { + reads int + writes int + seeks int + syncs int +} + +func (bbs BouncebackStats) String() string { + return fmt.Sprintf( + "BouncebackStats", + bbs.reads, bbs.writes, bbs.seeks, bbs.syncs) +} + +type bouncebackBase struct { + currentPosition int64 + + stats BouncebackStats +} + +// Position returns the position that we're supposed to be at. +func (bb *bouncebackBase) Position() int64 { + + // TODO(dustin): Add test + + return bb.currentPosition +} + +// StatsReads returns the number of reads that have been attempted. +func (bb *bouncebackBase) StatsReads() int { + + // TODO(dustin): Add test + + return bb.stats.reads +} + +// StatsWrites returns the number of write operations. +func (bb *bouncebackBase) StatsWrites() int { + + // TODO(dustin): Add test + + return bb.stats.writes +} + +// StatsSeeks returns the number of seeks. +func (bb *bouncebackBase) StatsSeeks() int { + + // TODO(dustin): Add test + + return bb.stats.seeks +} + +// StatsSyncs returns the number of corrective seeks ("bounce-backs"). +func (bb *bouncebackBase) StatsSyncs() int { + + // TODO(dustin): Add test + + return bb.stats.syncs +} + +// Seek does a seek to an arbitrary place in the `io.ReadSeeker`. +func (bb *bouncebackBase) seek(s io.Seeker, offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // If the seek is relative, make sure we're where we're supposed to be *first*. + if whence != io.SeekStart { + err = bb.checkPosition(s) + log.PanicIf(err) + } + + bb.stats.seeks++ + + newPosition, err = s.Seek(offset, whence) + log.PanicIf(err) + + // Update our internal tracking. + bb.currentPosition = newPosition + + return newPosition, nil +} + +func (bb *bouncebackBase) checkPosition(s io.Seeker) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Make sure we're where we're supposed to be. + + // This should have no overhead, and enables us to collect stats. + realCurrentPosition, err := s.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + if realCurrentPosition != bb.currentPosition { + bb.stats.syncs++ + + _, err = s.Seek(bb.currentPosition, io.SeekStart) + log.PanicIf(err) + } + + return nil +} + +// BouncebackReader wraps a ReadSeeker, keeps track of our position, and +// seeks back to it before writing. This allows an underlying ReadWriteSeeker +// with an unstable position can still be used for a prolonged series of writes. +type BouncebackReader struct { + rs io.ReadSeeker + + bouncebackBase +} + +// NewBouncebackReader returns a `*BouncebackReader` struct. +func NewBouncebackReader(rs io.ReadSeeker) (br *BouncebackReader, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + initialPosition, err := rs.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + bb := bouncebackBase{ + currentPosition: initialPosition, + } + + br = &BouncebackReader{ + rs: rs, + bouncebackBase: bb, + } + + return br, nil +} + +// Seek does a seek to an arbitrary place in the `io.ReadSeeker`. +func (br *BouncebackReader) Seek(offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newPosition, err = br.bouncebackBase.seek(br.rs, offset, whence) + log.PanicIf(err) + + return newPosition, nil +} + +// Seek does a standard read. +func (br *BouncebackReader) Read(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + br.bouncebackBase.stats.reads++ + + err = br.bouncebackBase.checkPosition(br.rs) + log.PanicIf(err) + + // Do read. + + n, err = br.rs.Read(p) + if err != nil { + if err == io.EOF { + return 0, io.EOF + } + + log.Panic(err) + } + + // Update our internal tracking. + br.bouncebackBase.currentPosition += int64(n) + + return n, nil +} + +// BouncebackWriter wraps a WriteSeeker, keeps track of our position, and +// seeks back to it before writing. This allows an underlying ReadWriteSeeker +// with an unstable position can still be used for a prolonged series of writes. +type BouncebackWriter struct { + ws io.WriteSeeker + + bouncebackBase +} + +// NewBouncebackWriter returns a new `BouncebackWriter` struct. +func NewBouncebackWriter(ws io.WriteSeeker) (bw *BouncebackWriter, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + initialPosition, err := ws.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + bb := bouncebackBase{ + currentPosition: initialPosition, + } + + bw = &BouncebackWriter{ + ws: ws, + bouncebackBase: bb, + } + + return bw, nil +} + +// Seek puts us at a specific position in the internal writer for the next +// write/seek. +func (bw *BouncebackWriter) Seek(offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newPosition, err = bw.bouncebackBase.seek(bw.ws, offset, whence) + log.PanicIf(err) + + return newPosition, nil +} + +// Write performs a write against the internal `WriteSeeker` starting at the +// position that we're supposed to be at. +func (bw *BouncebackWriter) Write(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + bw.bouncebackBase.stats.writes++ + + // Make sure we're where we're supposed to be. + + realCurrentPosition, err := bw.ws.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + if realCurrentPosition != bw.bouncebackBase.currentPosition { + bw.bouncebackBase.stats.seeks++ + + _, err = bw.ws.Seek(bw.bouncebackBase.currentPosition, io.SeekStart) + log.PanicIf(err) + } + + // Do write. + + n, err = bw.ws.Write(p) + log.PanicIf(err) + + // Update our internal tracking. + bw.bouncebackBase.currentPosition += int64(n) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go new file mode 100644 index 000000000..3d2e840fa --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go @@ -0,0 +1,95 @@ +package rifs + +import ( + "io" + + "github.com/dsoprea/go-logging" +) + +// BoundedReadWriteSeekCloser wraps a RWS that is also a closer with boundaries. +// This proxies the RWS methods to the inner BRWS inside. +type BoundedReadWriteSeekCloser struct { + io.Closer + *BoundedReadWriteSeeker +} + +// NewBoundedReadWriteSeekCloser returns a new BoundedReadWriteSeekCloser. +func NewBoundedReadWriteSeekCloser(rwsc ReadWriteSeekCloser, minimumOffset int64, staticFileSize int64) (brwsc *BoundedReadWriteSeekCloser, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + bs, err := NewBoundedReadWriteSeeker(rwsc, minimumOffset, staticFileSize) + log.PanicIf(err) + + brwsc = &BoundedReadWriteSeekCloser{ + Closer: rwsc, + BoundedReadWriteSeeker: bs, + } + + return brwsc, nil +} + +// Seek forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newOffset, err = rwsc.BoundedReadWriteSeeker.Seek(offset, whence) + log.PanicIf(err) + + return newOffset, nil +} + +// Read forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Read(buffer []byte) (readCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + readCount, err = rwsc.BoundedReadWriteSeeker.Read(buffer) + if err != nil { + if err == io.EOF { + return 0, err + } + + log.Panic(err) + } + + return readCount, nil +} + +// Write forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Write(buffer []byte) (writtenCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + writtenCount, err = rwsc.BoundedReadWriteSeeker.Write(buffer) + log.PanicIf(err) + + return writtenCount, nil +} + +// Close forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Close() (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = rwsc.Closer.Close() + log.PanicIf(err) + + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go new file mode 100644 index 000000000..d29657b05 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go @@ -0,0 +1,156 @@ +package rifs + +import ( + "errors" + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +var ( + // ErrSeekBeyondBound is returned when a seek is requested beyond the + // statically-given file-size. No writes or seeks beyond boundaries are + // supported with a statically-given file size. + ErrSeekBeyondBound = errors.New("seek beyond boundary") +) + +// BoundedReadWriteSeeker is a thin filter that ensures that no seeks can be done +// to offsets smaller than the one we were given. This supports libraries that +// might be expecting to read from the front of the stream being used on data +// that is in the middle of a stream instead. +type BoundedReadWriteSeeker struct { + io.ReadWriteSeeker + + currentOffset int64 + minimumOffset int64 + + staticFileSize int64 +} + +// NewBoundedReadWriteSeeker returns a new BoundedReadWriteSeeker instance. +func NewBoundedReadWriteSeeker(rws io.ReadWriteSeeker, minimumOffset int64, staticFileSize int64) (brws *BoundedReadWriteSeeker, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if minimumOffset < 0 { + log.Panicf("BoundedReadWriteSeeker minimum offset must be zero or larger: (%d)", minimumOffset) + } + + // We'll always started at a relative offset of zero. + _, err = rws.Seek(minimumOffset, os.SEEK_SET) + log.PanicIf(err) + + brws = &BoundedReadWriteSeeker{ + ReadWriteSeeker: rws, + + currentOffset: 0, + minimumOffset: minimumOffset, + + staticFileSize: staticFileSize, + } + + return brws, nil +} + +// Seek moves the offset to the given offset. Prevents offset from ever being +// moved left of `brws.minimumOffset`. +func (brws *BoundedReadWriteSeeker) Seek(offset int64, whence int) (updatedOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + fileSize := brws.staticFileSize + + // If we weren't given a static file-size, look it up whenever it is needed. + if whence == os.SEEK_END && fileSize == 0 { + realFileSizeRaw, err := brws.ReadWriteSeeker.Seek(0, os.SEEK_END) + log.PanicIf(err) + + fileSize = realFileSizeRaw - brws.minimumOffset + } + + updatedOffset, err = CalculateSeek(brws.currentOffset, offset, whence, fileSize) + log.PanicIf(err) + + if brws.staticFileSize != 0 && updatedOffset > brws.staticFileSize { + //updatedOffset = int64(brws.staticFileSize) + + // NOTE(dustin): Presumably, this will only be disruptive to writes that are beyond the boundaries, which, if we're being used at all, should already account for the boundary and prevent this error from ever happening. So, time will tell how disruptive this is. + return 0, ErrSeekBeyondBound + } + + if updatedOffset != brws.currentOffset { + updatedRealOffset := updatedOffset + brws.minimumOffset + + _, err = brws.ReadWriteSeeker.Seek(updatedRealOffset, os.SEEK_SET) + log.PanicIf(err) + + brws.currentOffset = updatedOffset + } + + return updatedOffset, nil +} + +// Read forwards writes to the inner RWS. +func (brws *BoundedReadWriteSeeker) Read(buffer []byte) (readCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if brws.staticFileSize != 0 { + availableCount := brws.staticFileSize - brws.currentOffset + if availableCount == 0 { + return 0, io.EOF + } + + if int64(len(buffer)) > availableCount { + buffer = buffer[:availableCount] + } + } + + readCount, err = brws.ReadWriteSeeker.Read(buffer) + brws.currentOffset += int64(readCount) + + if err != nil { + if err == io.EOF { + return 0, err + } + + log.Panic(err) + } + + return readCount, nil +} + +// Write forwards writes to the inner RWS. +func (brws *BoundedReadWriteSeeker) Write(buffer []byte) (writtenCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if brws.staticFileSize != 0 { + log.Panicf("writes can not be performed if a static file-size was given") + } + + writtenCount, err = brws.ReadWriteSeeker.Write(buffer) + brws.currentOffset += int64(writtenCount) + + log.PanicIf(err) + + return writtenCount, nil +} + +// MinimumOffset returns the configured minimum-offset. +func (brws *BoundedReadWriteSeeker) MinimumOffset() int64 { + return brws.minimumOffset +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go new file mode 100644 index 000000000..cd59d727c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go @@ -0,0 +1,52 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// SeekType is a convenience type to associate the different seek-types with +// printable descriptions. +type SeekType int + +// String returns a descriptive string. +func (n SeekType) String() string { + if n == io.SeekCurrent { + return "SEEK-CURRENT" + } else if n == io.SeekEnd { + return "SEEK-END" + } else if n == io.SeekStart { + return "SEEK-START" + } + + log.Panicf("unknown seek-type: (%d)", n) + return "" +} + +// CalculateSeek calculates an offset in a file-stream given the parameters. +func CalculateSeek(currentOffset int64, delta int64, whence int, fileSize int64) (finalOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + finalOffset = 0 + } + }() + + if whence == os.SEEK_SET { + finalOffset = delta + } else if whence == os.SEEK_CUR { + finalOffset = currentOffset + delta + } else if whence == os.SEEK_END { + finalOffset = fileSize + delta + } else { + log.Panicf("whence not valid: (%d)", whence) + } + + if finalOffset < 0 { + finalOffset = 0 + } + + return finalOffset, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go new file mode 100644 index 000000000..256333d40 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go @@ -0,0 +1,15 @@ +package rifs + +import ( + "os" + "path" +) + +var ( + appPath string +) + +func init() { + goPath := os.Getenv("GOPATH") + appPath = path.Join(goPath, "src", "github.com", "dsoprea", "go-utility", "filesystem") +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go new file mode 100644 index 000000000..89ee9a92c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go @@ -0,0 +1,40 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// CopyBytesBetweenPositions will copy bytes from one position in the given RWS +// to an earlier position in the same RWS. +func CopyBytesBetweenPositions(rws io.ReadWriteSeeker, fromPosition, toPosition int64, count int) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if fromPosition <= toPosition { + log.Panicf("from position (%d) must be larger than to position (%d)", fromPosition, toPosition) + } + + br, err := NewBouncebackReader(rws) + log.PanicIf(err) + + _, err = br.Seek(fromPosition, os.SEEK_SET) + log.PanicIf(err) + + bw, err := NewBouncebackWriter(rws) + log.PanicIf(err) + + _, err = bw.Seek(toPosition, os.SEEK_SET) + log.PanicIf(err) + + written, err := io.CopyN(bw, br, int64(count)) + log.PanicIf(err) + + n = int(written) + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go new file mode 100644 index 000000000..f5e6cd20a --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go @@ -0,0 +1,19 @@ +package rifs + +import ( + "os" +) + +// DoesExist returns true if we can open the given file/path without error. We +// can't simply use `os.IsNotExist()` because we'll get a different error when +// the parent directory doesn't exist, and really the only important thing is if +// it exists *and* it's readable. +func DoesExist(filepath string) bool { + f, err := os.Open(filepath) + if err != nil { + return false + } + + f.Close() + return true +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go new file mode 100644 index 000000000..8705e5fe0 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go @@ -0,0 +1,54 @@ +package rifs + +import ( + "fmt" + "io" +) + +const ( + defaultCopyBufferSize = 1024 * 1024 +) + +// GracefulCopy willcopy while enduring lesser normal issues. +// +// - We'll ignore EOF if the read byte-count is more than zero. Only an EOF when +// zero bytes were read will terminate the loop. +// +// - Ignore short-writes. If less bytes were written than the bytes that were +// given, we'll keep trying until done. +func GracefulCopy(w io.Writer, r io.Reader, buffer []byte) (copyCount int, err error) { + if buffer == nil { + buffer = make([]byte, defaultCopyBufferSize) + } + + for { + readCount, err := r.Read(buffer) + if err != nil { + if err != io.EOF { + err = fmt.Errorf("read error: %s", err.Error()) + return 0, err + } + + // Only break on EOF if no bytes were actually read. + if readCount == 0 { + break + } + } + + writeBuffer := buffer[:readCount] + + for len(writeBuffer) > 0 { + writtenCount, err := w.Write(writeBuffer) + if err != nil { + err = fmt.Errorf("write error: %s", err.Error()) + return 0, err + } + + writeBuffer = writeBuffer[writtenCount:] + } + + copyCount += readCount + } + + return copyCount, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go new file mode 100644 index 000000000..bcdbd67cb --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go @@ -0,0 +1,143 @@ +package rifs + +import ( + "io" + "os" + "path" + + "github.com/dsoprea/go-logging" +) + +// FileListFilterPredicate is the callback predicate used for filtering. +type FileListFilterPredicate func(parent string, child os.FileInfo) (hit bool, err error) + +// VisitedFile is one visited file. +type VisitedFile struct { + Filepath string + Info os.FileInfo + Index int +} + +// ListFiles feeds a continuous list of files from a recursive folder scan. An +// optional predicate can be provided in order to filter. When done, the +// `filesC` channel is closed. If there's an error, the `errC` channel will +// receive it. +func ListFiles(rootPath string, cb FileListFilterPredicate) (filesC chan VisitedFile, count int, errC chan error) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + // Make sure the path exists. + + f, err := os.Open(rootPath) + log.PanicIf(err) + + f.Close() + + // Do our thing. + + filesC = make(chan VisitedFile, 100) + errC = make(chan error, 1) + index := 0 + + go func() { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + errC <- err + } + }() + + queue := []string{rootPath} + for len(queue) > 0 { + // Pop the next folder to process off the queue. + var thisPath string + thisPath, queue = queue[0], queue[1:] + + // Skip path if a symlink. + + fi, err := os.Lstat(thisPath) + log.PanicIf(err) + + if (fi.Mode() & os.ModeSymlink) > 0 { + continue + } + + // Read information. + + folderF, err := os.Open(thisPath) + if err != nil { + errC <- log.Wrap(err) + return + } + + // Iterate through children. + + for { + children, err := folderF.Readdir(1000) + if err == io.EOF { + break + } else if err != nil { + errC <- log.Wrap(err) + return + } + + for _, child := range children { + filepath := path.Join(thisPath, child.Name()) + + // Skip if a file symlink. + + fi, err := os.Lstat(filepath) + log.PanicIf(err) + + if (fi.Mode() & os.ModeSymlink) > 0 { + continue + } + + // If a predicate was given, determine if this child will be + // left behind. + if cb != nil { + hit, err := cb(thisPath, child) + + if err != nil { + errC <- log.Wrap(err) + return + } + + if hit == false { + continue + } + } + + index++ + + // Push file to channel. + + vf := VisitedFile{ + Filepath: filepath, + Info: child, + Index: index, + } + + filesC <- vf + + // If a folder, queue for later processing. + + if child.IsDir() == true { + queue = append(queue, filepath) + } + } + } + + folderF.Close() + } + + close(filesC) + close(errC) + }() + + return filesC, index, errC +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go new file mode 100644 index 000000000..0a064c53d --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go @@ -0,0 +1,93 @@ +package rifs + +import ( + "io" + "time" + + "github.com/dsoprea/go-logging" +) + +// ProgressFunc receives progress updates. +type ProgressFunc func(n int, duration time.Duration, isEof bool) error + +// WriteProgressWrapper wraps a reader and calls a callback after each read with +// count and duration info. +type WriteProgressWrapper struct { + w io.Writer + progressCb ProgressFunc +} + +// NewWriteProgressWrapper returns a new WPW instance. +func NewWriteProgressWrapper(w io.Writer, progressCb ProgressFunc) io.Writer { + return &WriteProgressWrapper{ + w: w, + progressCb: progressCb, + } +} + +// Write does a write and calls the callback. +func (wpw *WriteProgressWrapper) Write(buffer []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + startAt := time.Now() + + n, err = wpw.w.Write(buffer) + log.PanicIf(err) + + duration := time.Since(startAt) + + err = wpw.progressCb(n, duration, false) + log.PanicIf(err) + + return n, nil +} + +// ReadProgressWrapper wraps a reader and calls a callback after each read with +// count and duration info. +type ReadProgressWrapper struct { + r io.Reader + progressCb ProgressFunc +} + +// NewReadProgressWrapper returns a new RPW instance. +func NewReadProgressWrapper(r io.Reader, progressCb ProgressFunc) io.Reader { + return &ReadProgressWrapper{ + r: r, + progressCb: progressCb, + } +} + +// Read reads data and calls the callback. +func (rpw *ReadProgressWrapper) Read(buffer []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + startAt := time.Now() + + n, err = rpw.r.Read(buffer) + + duration := time.Since(startAt) + + if err != nil { + if err == io.EOF { + errInner := rpw.progressCb(n, duration, true) + log.PanicIf(errInner) + + return n, err + } + + log.Panic(err) + } + + err = rpw.progressCb(n, duration, false) + log.PanicIf(err) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go new file mode 100644 index 000000000..d878ca4e6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go @@ -0,0 +1,36 @@ +package rifs + +import ( + "io" +) + +// ReadCounter proxies read requests and maintains a counter of bytes read. +type ReadCounter struct { + r io.Reader + counter int +} + +// NewReadCounter returns a new `ReadCounter` struct wrapping a `Reader`. +func NewReadCounter(r io.Reader) *ReadCounter { + return &ReadCounter{ + r: r, + } +} + +// Count returns the total number of bytes read. +func (rc *ReadCounter) Count() int { + return rc.counter +} + +// Reset resets the counter to zero. +func (rc *ReadCounter) Reset() { + rc.counter = 0 +} + +// Read forwards a read to the underlying `Reader` while bumping the counter. +func (rc *ReadCounter) Read(b []byte) (n int, err error) { + n, err = rc.r.Read(b) + rc.counter += n + + return n, err +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go new file mode 100644 index 000000000..3f3ec44dd --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go @@ -0,0 +1,63 @@ +package rifs + +import ( + "io" + + "github.com/dsoprea/go-logging" +) + +// ReadSeekerToReaderAt is a wrapper that allows a ReadSeeker to masquerade as a +// ReaderAt. +type ReadSeekerToReaderAt struct { + rs io.ReadSeeker +} + +// NewReadSeekerToReaderAt returns a new ReadSeekerToReaderAt instance. +func NewReadSeekerToReaderAt(rs io.ReadSeeker) *ReadSeekerToReaderAt { + return &ReadSeekerToReaderAt{ + rs: rs, + } +} + +// ReadAt is a wrapper that satisfies the ReaderAt interface. +// +// Note that a requirement of ReadAt is that it doesn't have an effect on the +// offset in the underlying resource as well as that concurrent calls can be +// made to it. Since we're capturing the current offset in the underlying +// resource and then seeking back to it before returning, it is the +// responsibility of the caller to serialize (i.e. use a mutex with) these +// requests in order to eliminate race-conditions in the parallel-usage +// scenario. +// +// Note also that, since ReadAt() is going to be called on a particular +// instance, that instance is going to internalize a file resource, that file- +// resource is provided by the OS, and [most] OSs are only gonna support one +// file-position per resource, locking is already going to be a necessary +// internal semantic of a ReaderAt implementation. +func (rstra *ReadSeekerToReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + originalOffset, err := rstra.rs.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + defer func() { + _, err := rstra.rs.Seek(originalOffset, io.SeekStart) + log.PanicIf(err) + }() + + _, err = rstra.rs.Seek(offset, io.SeekStart) + log.PanicIf(err) + + // Note that all errors will be wrapped, here. The usage of this method is + // such that typically no specific errors would be expected as part of + // normal operation (in which case we'd check for those first and return + // them directly). + n, err = io.ReadFull(rstra.rs, p) + log.PanicIf(err) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go new file mode 100644 index 000000000..c583a8024 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go @@ -0,0 +1,29 @@ +package rifs + +import ( + "io" +) + +// ReadWriteSeekCloser satisfies `io.ReadWriteSeeker` and `io.Closer` +// interfaces. +type ReadWriteSeekCloser interface { + io.ReadWriteSeeker + io.Closer +} + +type readWriteSeekNoopCloser struct { + io.ReadWriteSeeker +} + +// ReadWriteSeekNoopCloser wraps a `io.ReadWriteSeeker` with a no-op Close() +// call. +func ReadWriteSeekNoopCloser(rws io.ReadWriteSeeker) ReadWriteSeekCloser { + return readWriteSeekNoopCloser{ + ReadWriteSeeker: rws, + } +} + +// Close does nothing but allows the RWS to satisfy `io.Closer`.:wq +func (readWriteSeekNoopCloser) Close() (err error) { + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go new file mode 100644 index 000000000..5d41bb5df --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go @@ -0,0 +1,146 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// SeekableBuffer is a simple memory structure that satisfies +// `io.ReadWriteSeeker`. +type SeekableBuffer struct { + data []byte + position int64 +} + +// NewSeekableBuffer is a factory that returns a `*SeekableBuffer`. +func NewSeekableBuffer() *SeekableBuffer { + data := make([]byte, 0) + + return &SeekableBuffer{ + data: data, + } +} + +// NewSeekableBufferWithBytes is a factory that returns a `*SeekableBuffer`. +func NewSeekableBufferWithBytes(originalData []byte) *SeekableBuffer { + data := make([]byte, len(originalData)) + copy(data, originalData) + + return &SeekableBuffer{ + data: data, + } +} + +func len64(data []byte) int64 { + return int64(len(data)) +} + +// Bytes returns the underlying slice. +func (sb *SeekableBuffer) Bytes() []byte { + return sb.data +} + +// Len returns the number of bytes currently stored. +func (sb *SeekableBuffer) Len() int { + return len(sb.data) +} + +// Write does a standard write to the internal slice. +func (sb *SeekableBuffer) Write(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // The current position we're already at is past the end of the data we + // actually have. Extend our buffer up to our current position. + if sb.position > len64(sb.data) { + extra := make([]byte, sb.position-len64(sb.data)) + sb.data = append(sb.data, extra...) + } + + positionFromEnd := len64(sb.data) - sb.position + tailCount := positionFromEnd - len64(p) + + var tailBytes []byte + if tailCount > 0 { + tailBytes = sb.data[len64(sb.data)-tailCount:] + sb.data = append(sb.data[:sb.position], p...) + } else { + sb.data = append(sb.data[:sb.position], p...) + } + + if tailBytes != nil { + sb.data = append(sb.data, tailBytes...) + } + + dataSize := len64(p) + sb.position += dataSize + + return int(dataSize), nil +} + +// Read does a standard read against the internal slice. +func (sb *SeekableBuffer) Read(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if sb.position >= len64(sb.data) { + return 0, io.EOF + } + + n = copy(p, sb.data[sb.position:]) + sb.position += int64(n) + + return n, nil +} + +// Truncate either chops or extends the internal buffer. +func (sb *SeekableBuffer) Truncate(size int64) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + sizeInt := int(size) + if sizeInt < len(sb.data)-1 { + sb.data = sb.data[:sizeInt] + } else { + new := make([]byte, sizeInt-len(sb.data)) + sb.data = append(sb.data, new...) + } + + return nil +} + +// Seek does a standard seek on the internal slice. +func (sb *SeekableBuffer) Seek(offset int64, whence int) (n int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if whence == os.SEEK_SET { + sb.position = offset + } else if whence == os.SEEK_END { + sb.position = len64(sb.data) + offset + } else if whence == os.SEEK_CUR { + sb.position += offset + } else { + log.Panicf("seek whence is not valid: (%d)", whence) + } + + if sb.position < 0 { + sb.position = 0 + } + + return sb.position, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go new file mode 100644 index 000000000..a227b0b00 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go @@ -0,0 +1,69 @@ +package rifs + +import ( + "os" + "time" +) + +// SimpleFileInfo is a simple `os.FileInfo` implementation useful for testing +// with the bare minimum. +type SimpleFileInfo struct { + filename string + isDir bool + size int64 + mode os.FileMode + modTime time.Time +} + +// NewSimpleFileInfoWithFile returns a new file-specific SimpleFileInfo. +func NewSimpleFileInfoWithFile(filename string, size int64, mode os.FileMode, modTime time.Time) *SimpleFileInfo { + return &SimpleFileInfo{ + filename: filename, + isDir: false, + size: size, + mode: mode, + modTime: modTime, + } +} + +// NewSimpleFileInfoWithDirectory returns a new directory-specific +// SimpleFileInfo. +func NewSimpleFileInfoWithDirectory(filename string, modTime time.Time) *SimpleFileInfo { + return &SimpleFileInfo{ + filename: filename, + isDir: true, + mode: os.ModeDir, + modTime: modTime, + } +} + +// Name returns the base name of the file. +func (sfi *SimpleFileInfo) Name() string { + return sfi.filename +} + +// Size returns the length in bytes for regular files; system-dependent for +// others. +func (sfi *SimpleFileInfo) Size() int64 { + return sfi.size +} + +// Mode returns the file mode bits. +func (sfi *SimpleFileInfo) Mode() os.FileMode { + return sfi.mode +} + +// ModTime returns the modification time. +func (sfi *SimpleFileInfo) ModTime() time.Time { + return sfi.modTime +} + +// IsDir returns true if a directory. +func (sfi *SimpleFileInfo) IsDir() bool { + return sfi.isDir +} + +// Sys returns internal state. +func (sfi *SimpleFileInfo) Sys() interface{} { + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go new file mode 100644 index 000000000..4b33b41a9 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go @@ -0,0 +1,17 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// GetOffset returns the current offset of the Seeker and just panics if unable +// to find it. +func GetOffset(s io.Seeker) int64 { + offsetRaw, err := s.Seek(0, os.SEEK_CUR) + log.PanicIf(err) + + return offsetRaw +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go new file mode 100644 index 000000000..dc39901d5 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go @@ -0,0 +1,36 @@ +package rifs + +import ( + "io" +) + +// WriteCounter proxies write requests and maintains a counter of bytes written. +type WriteCounter struct { + w io.Writer + counter int +} + +// NewWriteCounter returns a new `WriteCounter` struct wrapping a `Writer`. +func NewWriteCounter(w io.Writer) *WriteCounter { + return &WriteCounter{ + w: w, + } +} + +// Count returns the total number of bytes read. +func (wc *WriteCounter) Count() int { + return wc.counter +} + +// Reset resets the counter to zero. +func (wc *WriteCounter) Reset() { + wc.counter = 0 +} + +// Write forwards a write to the underlying `Writer` while bumping the counter. +func (wc *WriteCounter) Write(b []byte) (n int, err error) { + n, err = wc.w.Write(b) + wc.counter += n + + return n, err +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/image/README.md b/vendor/github.com/dsoprea/go-utility/v2/image/README.md new file mode 100644 index 000000000..1509ff666 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/image/README.md @@ -0,0 +1,9 @@ +[![GoDoc](https://godoc.org/github.com/dsoprea/go-utility/image?status.svg)](https://godoc.org/github.com/dsoprea/go-utility/image) +[![Build Status](https://travis-ci.org/dsoprea/go-utility.svg?branch=master)](https://travis-ci.org/dsoprea/go-utility) +[![Coverage Status](https://coveralls.io/repos/github/dsoprea/go-utility/badge.svg?branch=master)](https://coveralls.io/github/dsoprea/go-utility?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/dsoprea/go-utility)](https://goreportcard.com/report/github.com/dsoprea/go-utility) + +# media_parser_type + +Common image-parsing interfaces. Used for JPEG, PNG, and HEIC parsers used by +go-exif-knife. diff --git a/vendor/github.com/dsoprea/go-utility/v2/image/media_parser_type.go b/vendor/github.com/dsoprea/go-utility/v2/image/media_parser_type.go new file mode 100644 index 000000000..8776a1fdd --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/image/media_parser_type.go @@ -0,0 +1,34 @@ +package riimage + +import ( + "io" + + "github.com/dsoprea/go-exif/v3" +) + +// MediaContext is an accessor that knows how to extract specific metadata from +// the media. +type MediaContext interface { + // Exif returns the EXIF's root IFD. + Exif() (rootIfd *exif.Ifd, data []byte, err error) +} + +// MediaParser prescribes a specific structure for the parser types that are +// imported from other projects. We don't use it directly, but we use this to +// impose structure. +type MediaParser interface { + // Parse parses a stream using an `io.ReadSeeker`. `mc` should *actually* be + // a `ExifContext`. + Parse(r io.ReadSeeker, size int) (mc MediaContext, err error) + + // ParseFile parses a stream using a file. `mc` should *actually* be a + // `ExifContext`. + ParseFile(filepath string) (mc MediaContext, err error) + + // ParseBytes parses a stream direct from bytes. `mc` should *actually* be + // a `ExifContext`. + ParseBytes(data []byte) (mc MediaContext, err error) + + // Parses the data to determine if it's a compatible format. + LooksLikeFormat(data []byte) bool +} diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/LICENSE b/vendor/github.com/superseriousbusiness/exif-terminator/LICENSE new file mode 100644 index 000000000..dba13ed2d --- /dev/null +++ b/vendor/github.com/superseriousbusiness/exif-terminator/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/README.md b/vendor/github.com/superseriousbusiness/exif-terminator/README.md new file mode 100644 index 000000000..8866202d5 --- /dev/null +++ b/vendor/github.com/superseriousbusiness/exif-terminator/README.md @@ -0,0 +1,116 @@ +# exif-terminator + +`exif-terminator` removes exif data from images (jpeg and png currently supported) in a streaming manner. All you need to do is provide a reader of the image in, and exif-terminator will provide a reader of the image out. + +Hasta la vista, baby! + +```text + .,lddxococ. + ..',lxO0Oo;'. + . .. .,coodO0klc:. + .,. ..','. .. .,..'. .':llxKXk' + .;c:cc;;,... .''.,l:cc. .....:l:,,:oo:.. + .,:ll'. .,;cox0OxOKKXX0kOOxlcld0X0d;,,,'. + .:xkl. .':cdKNWWWWMMMMMMMMMMWWNXK0KWNd. + .coxo,..:ollk0KKXNWMMMMMMMMMMWWXXXOoOM0; + ,oc,. .;cloxOKXXWWMMMMMMMMMMMWNXk;;OWO' + . ..;cdOKXNNWWMMMMMMMMMMMMWO,,ONO' + ...... ....;okOO000XWWMMMMMMMMMWXx;,ONNx. +.;c;. .:l'ckl. ..';looooolldolloooodolcc:;'.;oo:. +.oxl. ;:..OO. .. .. .,' .;. +.oko. .cc.'Ok. .:; .:,..';. +.cdc. .;;lc.,Ox. . .',,'..','. .dN0; .. .c:,,':. +.:oc. ,dxkl.,0x. . .. . .oNMMKc.. ...:l. +.:o:. cKXKl.,Ox. .. .lKWMMMXo,. ...''. +.:l; c0KKo.,0x. ...........';:lk0OKNNXKkl,..,;cxd' +.::' ;k00l.;0d. .. .,cloooddddxxddol;:ddloxdc,:odOWNc +.;,. ,ONKc.;0d. 'l,.. .:clllllllokKOl::cllclkKx'.lolxx' +.,. '0W0:.;0d. .:l,. .,:ccc:::oOXNXOkxdook0NWNx,,;c;. +... .kX0c.;0d. .loc' .,::;;;;lk0kddoooooddooO0o',ld; +.. .oOkk:cKd. .... .;:,',;cxK0o::ldkOkkOkxod:';oKx. +.. :dlOolKO, '::'.';:oOK0xdddoollooxOx::ccOx. +.. ';:o,.xKo. .,;'...';lddolooodkkkdol:,::lc. +.. ...:..oOl. ........';:codxxOXKKKk;':;:kl +.. .,..lOc. .. ....,codxkxxxxxo:,,;lKO. .,;'.. +... .. ck: ';,'. .;:cllloc,;;;colOK; .;odxxoc;. +...,.... . :x; .;:cc;'. .,;::c:'..,kXk:xNc .':oook00x:. + . cKx. .'.. ':clllc,...'';:::cc:;.,kOo:xNx. .'codddoox + .. ,xxl;',col:;. .:cccccc;;;:lxkkOOkdc,,lolcxWO' ;kNKc.' + .,. .c' ':dkO0O; .. .;ccccccc:::cldxkxoll:;oolcdN0:.. .xWNk; + .:' .c',xXNKkOXo .,. .,:cccccllc::lloooolc:;lo:;oXKc,::. .kWWX + ,' .cONMWMWkco, ', .';::ccclolc:llolollcccodo;:KXl..cl,. ;KWN + '. .xWWWWMKc;; ....;' ',;::::coolclloooollc:,:o;;0Xx, .,:;... ,0Ko + . ,kKNWWXd,cdd0NXKk:,;;;'';::::coollllllllllc;;ccl0Nkc. ..';loOx' + 'lxXWMXOOXNMMMMWWNNNWXkc;;;;;:cllccccccccc::lllkNWXd,. .cxO0Ol' + ,xKNWWXkkXWM0dxKNWWWMWNX0OOkl;;:c::cccc:,...:oONMMXOo;. :kOkOkl; + .;,;:;...,::. .;lokXKKNMMMWNOc,;;;,::;'...lOKNWNKkol:,..cKdcO0do + .:;... .. .,:okO0KNN0:.',,''''. ':xNMWKkxxOKXd,.cNk,:l:o +``` + +## Why? + +Exif removal is a pain in the arse. Most other libraries seem to parse the whole image into memory, then remove the exif data, then encode the image again. + +`exif-terminator` differs in that it removes exif data *while scanning through the image bytes*, and it doesn't do any reencoding of the image. Bytes of exif data are simply all set to 0, and the image data is piped back out again into the returned reader. + +## Example + +```go +package test + +import ( + "io" + "os" + + terminator "github.com/superseriousbusiness/exif-terminator" +) + +func main() { + // open a file + sloth, err := os.Open("./images/sloth.jpg") + if err != nil { + panic(err) + } + + // get the length of the file + stat, err := sloth.Stat() + if err != nil { + panic(err) + } + + // terminate! + out, err := terminator.Terminate(sloth, int(stat.Size()), "jpeg") + if err != nil { + panic(err) + } + + // read the bytes from the reader + b, err := io.ReadAll(out) + if err != nil { + panic(err) + } + + // save the file somewhere + if err := os.WriteFile("./images/sloth-clean.jpg", b, 0666); err != nil { + panic(err) + } +} +``` + +## Credits + +### Libraries + +`exif-terminator` borrows heavily from the two [`dsoprea`](https://github.com/dsoprea) libraries credited below. In fact, it's basically a hack on top of those libraries. Thanks `dsoprea`! + +- [dsoprea/go-jpeg-image-structure](https://github.com/dsoprea/go-jpeg-image-structure): jpeg structure parsing. [MIT License](https://spdx.org/licenses/MIT.html). +- [dsoprea/go-png-image-structure](https://github.com/dsoprea/go-png-image-structure): png structure parsing. [MIT License](https://spdx.org/licenses/MIT.html). +- [stretchr/testify](https://github.com/stretchr/testify); test framework. [MIT License](https://spdx.org/licenses/MIT.html). + +## License + +![the gnu AGPL logo](https://www.gnu.org/graphics/agplv3-155x51.png) + +`exif-terminator` is free software, licensed under the [GNU AGPL v3 LICENSE](LICENSE). + +Copyright (C) 2022 SuperSeriousBusiness. diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go b/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go new file mode 100644 index 000000000..224a9b646 --- /dev/null +++ b/vendor/github.com/superseriousbusiness/exif-terminator/jpeg.go @@ -0,0 +1,138 @@ +/* + exif-terminator + Copyright (C) 2022 SuperSeriousBusiness admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package terminator + +import ( + "encoding/binary" + "fmt" + "io" + + jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2" +) + +var markerLen = map[byte]int{ + 0x00: 0, + 0x01: 0, + 0xd0: 0, + 0xd1: 0, + 0xd2: 0, + 0xd3: 0, + 0xd4: 0, + 0xd5: 0, + 0xd6: 0, + 0xd7: 0, + 0xd8: 0, + 0xd9: 0, + 0xda: 0, + + // J2C + 0x30: 0, + 0x31: 0, + 0x32: 0, + 0x33: 0, + 0x34: 0, + 0x35: 0, + 0x36: 0, + 0x37: 0, + 0x38: 0, + 0x39: 0, + 0x3a: 0, + 0x3b: 0, + 0x3c: 0, + 0x3d: 0, + 0x3e: 0, + 0x3f: 0, + 0x4f: 0, + 0x92: 0, + 0x93: 0, + + // J2C extensions + 0x74: 4, + 0x75: 4, + 0x77: 4, +} + +type jpegVisitor struct { + js *jpegstructure.JpegSplitter + writer io.Writer +} + +// HandleSegment satisfies the visitor interface{} of the jpegstructure library. +// +// We don't really care about any of the parameters, since all we're interested +// in here is the very last segment that was scanned. +func (v *jpegVisitor) HandleSegment(_ byte, _ string, _ int, _ bool) error { + // all we want to do here is get the last segment that was scanned, and then manipulate it + segmentList := v.js.Segments() + segments := segmentList.Segments() + lastSegment := segments[len(segments)-1] + return v.writeSegment(lastSegment) +} + +func (v *jpegVisitor) writeSegment(s *jpegstructure.Segment) error { + w := v.writer + + defer func() { + // whatever happens, when we finished then evict data from the segment; + // once we've written it we don't want it in memory anymore + s.Data = s.Data[:0] + }() + + // The scan-data will have a marker-ID of (0) because it doesn't have a marker-ID or length. + if s.MarkerId != 0 { + if _, err := w.Write([]byte{0xff, s.MarkerId}); err != nil { + return err + } + + sizeLen, found := markerLen[s.MarkerId] + if !found || sizeLen == 2 { + sizeLen = 2 + l := uint16(len(s.Data) + sizeLen) + + if err := binary.Write(w, binary.BigEndian, &l); err != nil { + return err + } + + } else if sizeLen == 4 { + l := uint32(len(s.Data) + sizeLen) + + if err := binary.Write(w, binary.BigEndian, &l); err != nil { + return err + } + + } else if sizeLen != 0 { + return fmt.Errorf("not a supported marker-size: MARKER-ID=(0x%02x) MARKER-SIZE-LEN=(%d)", s.MarkerId, sizeLen) + } + } + + if s.IsExif() { + // if this segment is exif data, write blank bytes + blank := make([]byte, len(s.Data)) + if _, err := w.Write(blank); err != nil { + return err + } + } else { + // otherwise write the data + if _, err := w.Write(s.Data); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/png.go b/vendor/github.com/superseriousbusiness/exif-terminator/png.go new file mode 100644 index 000000000..4a1ac5bf1 --- /dev/null +++ b/vendor/github.com/superseriousbusiness/exif-terminator/png.go @@ -0,0 +1,93 @@ +/* + exif-terminator + Copyright (C) 2022 SuperSeriousBusiness admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package terminator + +import ( + "encoding/binary" + "io" + + pngstructure "github.com/dsoprea/go-png-image-structure/v2" +) + +type pngVisitor struct { + ps *pngstructure.PngSplitter + writer io.Writer + lastWrittenChunk int +} + +func (v *pngVisitor) split(data []byte, atEOF bool) (int, []byte, error) { + // execute the ps split function to read in data + advance, token, err := v.ps.Split(data, atEOF) + if err != nil { + return advance, token, err + } + + // if we haven't written anything at all yet, then write the png header back into the writer first + if v.lastWrittenChunk == -1 { + if _, err := v.writer.Write(pngstructure.PngSignature[:]); err != nil { + return advance, token, err + } + } + + // check if the splitter has any new chunks in it that we haven't written yet + chunkSlice := v.ps.Chunks() + chunks := chunkSlice.Chunks() + for i, chunk := range chunks { + // look through all the chunks in the splitter + if i > v.lastWrittenChunk { + // we've got a chunk we haven't written yet! write it... + if err := v.writeChunk(chunk); err != nil { + return advance, token, err + } + // then remove the data + chunk.Data = chunk.Data[:0] + // and update + v.lastWrittenChunk = i + } + } + + return advance, token, err +} + +func (v *pngVisitor) writeChunk(chunk *pngstructure.Chunk) error { + if err := binary.Write(v.writer, binary.BigEndian, chunk.Length); err != nil { + return err + } + + if _, err := v.writer.Write([]byte(chunk.Type)); err != nil { + return err + } + + if chunk.Type == pngstructure.EXifChunkType { + blank := make([]byte, len(chunk.Data)) + if _, err := v.writer.Write(blank); err != nil { + return err + } + } else { + if _, err := v.writer.Write(chunk.Data); err != nil { + return err + } + } + + if err := binary.Write(v.writer, binary.BigEndian, chunk.Crc); err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go b/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go new file mode 100644 index 000000000..b6225f6dc --- /dev/null +++ b/vendor/github.com/superseriousbusiness/exif-terminator/terminator.go @@ -0,0 +1,116 @@ +/* + exif-terminator + Copyright (C) 2022 SuperSeriousBusiness admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package terminator + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + + jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2" + pngstructure "github.com/dsoprea/go-png-image-structure/v2" +) + +func Terminate(in io.Reader, fileSize int, mediaType string) (io.Reader, error) { + // to avoid keeping too much stuff in memory we want to pipe data directly + pipeReader, pipeWriter := io.Pipe() + + // we don't know ahead of time how long segments might be: they could be as large as + // the file itself, so unfortunately we need to allocate a buffer here that'scanner as large + // as the file + scanner := bufio.NewScanner(in) + scanner.Buffer([]byte{}, fileSize) + var err error + + switch mediaType { + case "image/jpeg", "jpeg", "jpg": + err = terminateJpeg(scanner, pipeWriter) + case "image/png", "png": + // for pngs we need to skip the header bytes, so read them in + // and check we're really dealing with a png here + header := make([]byte, len(pngstructure.PngSignature)) + if _, headerError := in.Read(header); headerError != nil { + err = headerError + break + } + + if !bytes.Equal(header, pngstructure.PngSignature[:]) { + err = errors.New("could not decode png: invalid header") + break + } + + err = terminatePng(scanner, pipeWriter) + default: + err = fmt.Errorf("mediaType %s cannot be processed", mediaType) + } + + return pipeReader, err +} + +func terminateJpeg(scanner *bufio.Scanner, writer io.WriteCloser) error { + // jpeg visitor is where the spicy hack of streaming the de-exifed data is contained + v := &jpegVisitor{ + writer: writer, + } + + // provide the visitor to the splitter so that it triggers on every section scan + js := jpegstructure.NewJpegSplitter(v) + + // the visitor also needs to read back the list of segments: for this it needs + // to know what jpeg splitter it's attached to, so give it a pointer to the splitter + v.js = js + + // use the jpeg splitters 'split' function, which satisfies the bufio.SplitFunc interface + scanner.Split(js.Split) + + scanAndClose(scanner, writer) + return nil +} + +func terminatePng(scanner *bufio.Scanner, writer io.WriteCloser) error { + ps := pngstructure.NewPngSplitter() + + v := &pngVisitor{ + ps: ps, + writer: writer, + lastWrittenChunk: -1, + } + + // use the png visitor's 'split' function, which satisfies the bufio.SplitFunc interface + scanner.Split(v.split) + + scanAndClose(scanner, writer) + return nil +} + +func scanAndClose(scanner *bufio.Scanner, writer io.WriteCloser) { + // scan asynchronously until there's nothing left to scan, and then close the writer + // so that the reader on the other side knows that we're done + // + // due to the nature of io.Pipe, writing won't actually work + // until the pipeReader starts being read by the caller, which + // is why we do this asynchronously + go func() { + for scanner.Scan() { + } + writer.Close() + }() +} diff --git a/vendor/github.com/superseriousbusiness/exifremove/LICENSE b/vendor/github.com/superseriousbusiness/exifremove/LICENSE deleted file mode 100644 index 3ba9c83c6..000000000 --- a/vendor/github.com/superseriousbusiness/exifremove/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 scott lee davis - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/exifremove.go b/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/exifremove.go deleted file mode 100644 index d9e7e2ad1..000000000 --- a/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/exifremove.go +++ /dev/null @@ -1,140 +0,0 @@ -package exifremove - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "image/jpeg" - "image/png" - - "github.com/dsoprea/go-exif" - jpegstructure "github.com/dsoprea/go-jpeg-image-structure" - pngstructure "github.com/dsoprea/go-png-image-structure" - "github.com/h2non/filetype" -) - -func Remove(data []byte) ([]byte, error) { - - const ( - JpegMediaType = "jpeg" - PngMediaType = "png" - OtherMediaType = "other" - StartBytes = 0 - EndBytes = 0 - ) - - type MediaContext struct { - MediaType string - RootIfd *exif.Ifd - RawExif []byte - Media interface{} - } - - filtered := []byte{} - - head := make([]byte, 261) - _, err := bytes.NewReader(data).Read(head) - if err != nil { - return nil, fmt.Errorf("could not read first 261 bytes of data: %s", err) - } - imagetype, err := filetype.Match(head) - if err != nil { - return nil, fmt.Errorf("error matching first 261 bytes of image to valid type: %s", err) - } - - switch imagetype.MIME.Subtype { - case "jpeg": - jmp := jpegstructure.NewJpegMediaParser() - sl, err := jmp.ParseBytes(data) - if err != nil { - return nil, err - } - - _, rawExif, err := sl.Exif() - if err != nil { - return data, nil - } - - startExifBytes := StartBytes - endExifBytes := EndBytes - - if bytes.Contains(data, rawExif) { - for i := 0; i < len(data)-len(rawExif); i++ { - if bytes.Compare(data[i:i+len(rawExif)], rawExif) == 0 { - startExifBytes = i - endExifBytes = i + len(rawExif) - break - } - } - fill := make([]byte, len(data[startExifBytes:endExifBytes])) - copy(data[startExifBytes:endExifBytes], fill) - } - - filtered = data - - _, err = jpeg.Decode(bytes.NewReader(filtered)) - if err != nil { - return nil, errors.New("EXIF removal corrupted " + err.Error()) - } - case "png": - pmp := pngstructure.NewPngMediaParser() - cs, err := pmp.ParseBytes(data) - if err != nil { - return nil, err - } - - _, rawExif, err := cs.Exif() - if err != nil { - return data, nil - } - - startExifBytes := StartBytes - endExifBytes := EndBytes - - if bytes.Contains(data, rawExif) { - for i := 0; i < len(data)-len(rawExif); i++ { - if bytes.Compare(data[i:i+len(rawExif)], rawExif) == 0 { - startExifBytes = i - endExifBytes = i + len(rawExif) - break - } - } - fill := make([]byte, len(data[startExifBytes:endExifBytes])) - copy(data[startExifBytes:endExifBytes], fill) - } - - filtered = data - - chunks := readPNGChunks(bytes.NewReader(filtered)) - - for _, chunk := range chunks { - if !chunk.CRCIsValid() { - offset := int(chunk.Offset) + 8 + int(chunk.Length) - crc := chunk.CalculateCRC() - - buf := new(bytes.Buffer) - binary.Write(buf, binary.BigEndian, crc) - crcBytes := buf.Bytes() - - copy(filtered[offset:], crcBytes) - } - } - - chunks = readPNGChunks(bytes.NewReader(filtered)) - for _, chunk := range chunks { - if !chunk.CRCIsValid() { - return nil, errors.New("EXIF removal failed CRC") - } - } - - _, err = png.Decode(bytes.NewReader(filtered)) - if err != nil { - return nil, errors.New("EXIF removal corrupted " + err.Error()) - } - default: - return nil, errors.New("filetype not recognised") - } - - return filtered, nil -} diff --git a/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/png_crc_fix.go b/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/png_crc_fix.go deleted file mode 100644 index 390e5e515..000000000 --- a/vendor/github.com/superseriousbusiness/exifremove/pkg/exifremove/png_crc_fix.go +++ /dev/null @@ -1,104 +0,0 @@ -package exifremove - -// borrowed heavily from https://github.com/landaire/png-crc-fix/blob/master/main.go - -import ( - "bytes" - "encoding/binary" - "fmt" - "hash/crc32" - "io" - "os" -) - -const chunkStartOffset = 8 -const endChunk = "IEND" - -type pngChunk struct { - Offset int64 - Length uint32 - Type [4]byte - Data []byte - CRC uint32 -} - -func (p pngChunk) String() string { - return fmt.Sprintf("%s@%x - %X - Valid CRC? %v", p.Type, p.Offset, p.CRC, p.CRCIsValid()) -} - -func (p pngChunk) Bytes() []byte { - var buffer bytes.Buffer - - binary.Write(&buffer, binary.BigEndian, p.Type) - buffer.Write(p.Data) - - return buffer.Bytes() -} - -func (p pngChunk) CRCIsValid() bool { - return p.CRC == p.CalculateCRC() -} - -func (p pngChunk) CalculateCRC() uint32 { - crcTable := crc32.MakeTable(crc32.IEEE) - - return crc32.Checksum(p.Bytes(), crcTable) -} - -func (p pngChunk) CRCOffset() int64 { - return p.Offset + int64(8+p.Length) -} - -func readPNGChunks(reader io.ReadSeeker) []pngChunk { - chunks := []pngChunk{} - - reader.Seek(chunkStartOffset, os.SEEK_SET) - - readChunk := func() (*pngChunk, error) { - var chunk pngChunk - chunk.Offset, _ = reader.Seek(0, os.SEEK_CUR) - - binary.Read(reader, binary.BigEndian, &chunk.Length) - - chunk.Data = make([]byte, chunk.Length) - - err := binary.Read(reader, binary.BigEndian, &chunk.Type) - if err != nil { - goto read_error - } - - if read, err := reader.Read(chunk.Data); read == 0 || err != nil { - goto read_error - } - - err = binary.Read(reader, binary.BigEndian, &chunk.CRC) - if err != nil { - goto read_error - } - - return &chunk, nil - - read_error: - return nil, fmt.Errorf("Read error") - } - - chunk, err := readChunk() - if err != nil { - return chunks - } - - chunks = append(chunks, *chunk) - - // Read the first chunk - for string(chunks[len(chunks)-1].Type[:]) != endChunk { - - chunk, err := readChunk() - if err != nil { - break - } - - chunks = append(chunks, *chunk) - } - - return chunks -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8637feaec..ff65f1ab7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -13,8 +13,6 @@ codeberg.org/gruf/go-format # codeberg.org/gruf/go-hashenc v1.0.1 ## explicit; go 1.16 codeberg.org/gruf/go-hashenc -# codeberg.org/gruf/go-logger v1.3.2 -## explicit; go 1.14 # codeberg.org/gruf/go-mutexes v1.0.1 ## explicit; go 1.14 codeberg.org/gruf/go-mutexes @@ -49,32 +47,30 @@ github.com/coreos/go-oidc/v3/oidc # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b -## explicit; go 1.13 -github.com/dsoprea/go-exif -# github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b -## explicit; go 1.13 -github.com/dsoprea/go-exif/v2 -github.com/dsoprea/go-exif/v2/common -github.com/dsoprea/go-exif/v2/undefined +# github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b +## explicit; go 1.12 +github.com/dsoprea/go-exif/v3 +github.com/dsoprea/go-exif/v3/common +github.com/dsoprea/go-exif/v3/undefined # github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 ## explicit; go 1.13 github.com/dsoprea/go-iptc -# github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836 -## explicit; go 1.13 -github.com/dsoprea/go-jpeg-image-structure +# github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 +## explicit; go 1.12 +github.com/dsoprea/go-jpeg-image-structure/v2 # github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd ## explicit; go 1.13 github.com/dsoprea/go-logging # github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d ## explicit; go 1.13 github.com/dsoprea/go-photoshop-info-format -# github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d -## explicit; go 1.13 -github.com/dsoprea/go-png-image-structure -# github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e +# github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d +## explicit; go 1.12 +github.com/dsoprea/go-png-image-structure/v2 +# github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e ## explicit; go 1.12 -github.com/dsoprea/go-utility/image +github.com/dsoprea/go-utility/v2/filesystem +github.com/dsoprea/go-utility/v2/image # github.com/fsnotify/fsnotify v1.5.1 ## explicit; go 1.13 github.com/fsnotify/fsnotify @@ -469,9 +465,9 @@ github.com/superseriousbusiness/activity/streams/values/rfc2045 github.com/superseriousbusiness/activity/streams/values/rfc5988 github.com/superseriousbusiness/activity/streams/values/string github.com/superseriousbusiness/activity/streams/vocab -# github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 -## explicit; go 1.16 -github.com/superseriousbusiness/exifremove/pkg/exifremove +# github.com/superseriousbusiness/exif-terminator v0.1.0 +## explicit; go 1.17 +github.com/superseriousbusiness/exif-terminator # github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB ## explicit; go 1.13 github.com/superseriousbusiness/oauth2/v4 -- cgit v1.3 From c157b1b20b38cc331cfd1673433d077719feef3f Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 23 Jan 2022 14:41:58 +0100 Subject: rework data function to provide filesize --- internal/federation/dereferencing/account.go | 4 ++-- internal/federation/dereferencing/media.go | 2 +- internal/media/image.go | 20 ----------------- internal/media/manager_test.go | 12 +++++----- internal/media/processingemoji.go | 4 ++-- internal/media/processingmedia.go | 33 ++++++++++++++++++++-------- internal/media/types.go | 2 +- internal/processing/account/update.go | 10 +++++---- internal/processing/admin/emoji.go | 5 +++-- internal/processing/media/create.go | 5 +++-- internal/transport/derefmedia.go | 12 +++++----- internal/transport/transport.go | 4 ++-- 12 files changed, 56 insertions(+), 57 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 6ea8256d5..581c95de2 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -252,7 +252,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) (io.Reader, error) { + data := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, avatarIRI) } @@ -274,7 +274,7 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * return err } - data := func(innerCtx context.Context) (io.Reader, error) { + data := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, headerIRI) } diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index c427f2507..0b19570f2 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -42,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) (io.Reader, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/media/image.go b/internal/media/image.go index b8f00024f..e5390cee5 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -30,7 +30,6 @@ import ( "github.com/buckket/go-blurhash" "github.com/nfnt/resize" - "github.com/superseriousbusiness/exifremove/pkg/exifremove" ) const ( @@ -197,22 +196,3 @@ func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) { small: out.Bytes(), }, nil } - -// purgeExif is a little wrapper for the action of removing exif data from an image. -// Only pass pngs or jpegs to this function. -func purgeExif(data []byte) ([]byte, error) { - if len(data) == 0 { - return nil, errors.New("passed image was not valid") - } - - clean, err := exifremove.Remove(data) - if err != nil { - return nil, fmt.Errorf("could not purge exif from image: %s", err) - } - - if len(clean) == 0 { - return nil, errors.New("purged image was not valid") - } - - return clean, nil -} diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 5380b83b1..960f34843 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -39,13 +39,13 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), nil + return bytes.NewBuffer(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -109,13 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), nil + return bytes.NewBuffer(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -192,9 +192,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { panic(err) } - data := func(_ context.Context) (io.Reader, error) { + data := func(_ context.Context) (io.Reader, int, error) { // load bytes from a test image - return bytes.NewReader(b), nil + return bytes.NewReader(b), len(b), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 147b6b5b3..292712427 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -163,7 +163,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { } // execute the data function to get the reader out of it - reader, err := p.data(ctx) + reader, fileSize, err := p.data(ctx) if err != nil { return fmt.Errorf("store: error executing data function: %s", err) } @@ -194,6 +194,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) p.emoji.ImageContentType = contentType + p.emoji.ImageFileSize = fileSize // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) @@ -202,7 +203,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } - p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream // if the original reader is a readcloser, close it since we're done with it now if rc, ok := reader.(io.ReadCloser); ok { diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 82db863e0..0bbe35aee 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -28,6 +28,7 @@ import ( "time" "codeberg.org/gruf/go-store/kv" + terminator "github.com/superseriousbusiness/exif-terminator" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" @@ -239,7 +240,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { } // execute the data function to get the reader out of it - reader, err := p.data(ctx) + reader, fileSize, err := p.data(ctx) if err != nil { return fmt.Errorf("store: error executing data function: %s", err) } @@ -268,22 +269,36 @@ func (p *ProcessingMedia) store(ctx context.Context) error { } extension := split[1] // something like 'jpeg' - // set some additional fields on the attachment now that - // we know more about what the underlying media actually is - if extension == mimeGif { + // concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara) + multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + + // we'll need to clean exif data from the first bytes; while we're + // here, we can also use the extension to derive the attachment type + var clean io.Reader + switch extension { + case mimeGif: p.attachment.Type = gtsmodel.FileTypeGif - } else { + clean = multiReader // nothing to clean from a gif + case mimeJpeg, mimePng: p.attachment.Type = gtsmodel.FileTypeImage + purged, err := terminator.Terminate(multiReader, fileSize, extension) + if err != nil { + return fmt.Errorf("store: exif error: %s", err) + } + clean = purged + default: + return fmt.Errorf("store: couldn't process %s", extension) } + + // now set some additional fields on the attachment since + // we know more about what the underlying media actually is p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) p.attachment.File.ContentType = contentType - - // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) - multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) + p.attachment.File.FileSize = fileSize // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil { + if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } diff --git a/internal/media/types.go b/internal/media/types.go index 0a7f60d66..b9c79d464 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -118,4 +118,4 @@ type AdditionalEmojiInfo struct { } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. -type DataFunc func(ctx context.Context) (io.Reader, error) +type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 5a0a3e5a1..758cc6600 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -140,8 +140,9 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, error) { - return avatar.Open() + dataFunc := func(ctx context.Context) (io.Reader, int, error) { + f, err := avatar.Open() + return f, int(avatar.Size), err } isAvatar := true @@ -166,8 +167,9 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, error) { - return header.Open() + dataFunc := func(ctx context.Context) (io.Reader, int, error) { + f, err := header.Open() + return f, int(header.Size), err } isHeader := true diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index e0068858b..bb9f4ecb5 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -36,8 +36,9 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin") } - data := func(innerCtx context.Context) (io.Reader, error) { - return form.Image.Open() + data := func(innerCtx context.Context) (io.Reader, int, error) { + f, err := form.Image.Open() + return f, int(form.Image.Size), err } emojiID, err := id.NewRandomULID() diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 0fda4c27b..4047278eb 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -29,8 +29,9 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { - data := func(innerCtx context.Context) (io.Reader, error) { - return form.File.Open() + data := func(innerCtx context.Context) (io.Reader, int, error) { + f, err := form.File.Open() + return f, int(form.File.Size), err } focusX, focusY, err := parseFocus(form.Focus) diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index ed32f20c6..e3c86ce1e 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -28,12 +28,12 @@ import ( "github.com/sirupsen/logrus" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) { l := logrus.WithField("func", "DereferenceMedia") l.Debugf("performing GET to %s", iri.String()) req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil) if err != nil { - return nil, err + return nil, 0, err } req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here @@ -44,14 +44,14 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.Read err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil) t.getSignerMu.Unlock() if err != nil { - return nil, err + return nil, 0, err } resp, err := t.client.Do(req) if err != nil { - return nil, err + return nil, 0, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) + return nil, 0, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) } - return resp.Body, nil + return resp.Body, int(resp.ContentLength), nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index d9650d952..9e8cd8213 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -34,8 +34,8 @@ import ( // functionality for fetching remote media. type Transport interface { pub.Transport - // DereferenceMedia fetches the given media attachment IRI. - DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, error) + // DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize. + DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. -- cgit v1.3 From 3c1eb155e482071dc8ae592519060321d347683b Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 23 Jan 2022 15:52:18 +0100 Subject: add file size checks --- internal/media/manager_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 960f34843..a3eb0360c 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -74,6 +74,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, }, attachment.FileMeta.Small) suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal(269739, attachment.File.FileSize) suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) // now make sure the attachment is in the database @@ -151,6 +152,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, }, attachment.FileMeta.Small) suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal(269739, attachment.File.FileSize) suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) // now make sure the attachment is in the database @@ -232,6 +234,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, }, attachment.FileMeta.Small) suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal(269739, attachment.File.FileSize) suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) // now make sure the attachment is in the database -- cgit v1.3 From 667e7f112ce7b5b7452c392bbbe393a4c998508d Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 13:12:17 +0100 Subject: update remote account get/deref logic --- internal/federation/dereference.go | 8 +- internal/federation/dereferencing/account.go | 286 +++++++++++++-------- internal/federation/dereferencing/account_test.go | 3 +- internal/federation/dereferencing/dereferencer.go | 33 ++- internal/federation/dereferencing/status.go | 8 +- internal/federation/federatingprotocol.go | 2 +- internal/federation/federator.go | 3 +- internal/processing/account/get.go | 8 +- internal/processing/federation/getfollowers.go | 2 +- internal/processing/federation/getfollowing.go | 2 +- internal/processing/federation/getoutbox.go | 2 +- internal/processing/federation/getstatus.go | 2 +- internal/processing/federation/getstatusreplies.go | 2 +- internal/processing/federation/getuser.go | 2 +- internal/processing/fromfederator.go | 8 +- internal/processing/search.go | 4 +- internal/typeutils/internaltofrontend.go | 23 +- 17 files changed, 241 insertions(+), 157 deletions(-) diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go index 343ddadb7..8cb23a91f 100644 --- a/internal/federation/dereference.go +++ b/internal/federation/dereference.go @@ -26,12 +26,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { - return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, refresh) -} - -func (f *federator) EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) { - return f.dereferencer.EnrichRemoteAccount(ctx, username, account) +func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { + return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh) } func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error) { diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 581c95de2..27591d857 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -26,6 +26,7 @@ import ( "io" "net/url" "strings" + "sync" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/activity/streams" @@ -44,30 +45,6 @@ func instanceAccount(account *gtsmodel.Account) bool { (account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor")) } -// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form, -// and populates it with additional fields, media, etc. -// -// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by -// the federatingDB's Create function, or during the federated authorization flow. -func (d *deref) EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) { - // if we're dealing with an instance account, we don't need to update anything - if instanceAccount(account) { - return account, nil - } - - if err := d.PopulateAccountFields(ctx, account, username, false); err != nil { - return nil, err - } - - updated, err := d.db.UpdateAccount(ctx, account) - if err != nil { - logrus.Errorf("EnrichRemoteAccount: error updating account: %s", err) - return account, nil - } - - return updated, nil -} - // GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, // puts it in the database, and returns it to a caller. The boolean indicates whether the account is new // to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before, @@ -77,60 +54,73 @@ func (d *deref) EnrichRemoteAccount(ctx context.Context, username string, accoun // the remote instance again. // // SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). -func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { +func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool, blocking bool) (*gtsmodel.Account, error) { new := true - // check if we already have the account in our db - maybeAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String()) + // check if we already have the account in our db, and just return it unless we'd doing a refresh + remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String()) if err == nil { - // we've seen this account before so it's not new new = false if !refresh { - // we're not being asked to refresh, but just in case we don't have the avatar/header cached yet.... - maybeAccount, err = d.EnrichRemoteAccount(ctx, username, maybeAccount) - return maybeAccount, new, err + // make sure the account fields are populated before returning: + // even if we're not doing a refresh, the caller might want to block + // until everything is loaded + err = d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking) + return remoteAccount, err } } - accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) - if err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err) - } - - gtsAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh) - if err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err) - } - if new { - // generate a new id since we haven't seen this account before, and do a put - ulid, err := id.NewRandomULID() + // we haven't seen this account before: dereference it from remote + accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) if err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err) + return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) } - gtsAccount.ID = ulid - if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err) } - if err := d.db.Put(ctx, gtsAccount); err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err) + ulid, err := id.NewRandomULID() + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err) } - } else { - // take the id we already have and do an update - gtsAccount.ID = maybeAccount.ID - if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil { - return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + newAccount.ID = ulid + + if err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) } - gtsAccount, err = d.db.UpdateAccount(ctx, gtsAccount) - if err != nil { - return nil, false, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err) + if err := d.db.Put(ctx, newAccount); err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err) } + + return newAccount, nil + } + + // we have seen this account before, but we have to refresh it + refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err) } - return gtsAccount, new, nil + refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err) + } + refreshedAccount.ID = remoteAccount.ID + + if err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking); err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err) + } + + updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err) + } + + return updatedAccount, nil } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -201,93 +191,177 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName()) } -// PopulateAccountFields populates any fields on the given account that weren't populated by the initial +// populateAccountFields populates any fields on the given account that weren't populated by the initial // dereferencing. This includes things like header and avatar etc. -func (d *deref) PopulateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, refresh bool) error { - l := logrus.WithFields(logrus.Fields{ - "func": "PopulateAccountFields", - "requestingUsername": requestingUsername, - }) +func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, refresh bool, blocking bool) error { + // if we're dealing with an instance account, just bail, we don't need to do anything + if instanceAccount(account) { + return nil + } accountURI, err := url.Parse(account.URI) if err != nil { - return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err) + return fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %s", account.URI, err) } + if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil { - return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host) + return fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host) } t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) if err != nil { - return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err) + return fmt.Errorf("populateAccountFields: error getting transport for user: %s", err) } // fetch the header and avatar - if err := d.fetchHeaderAndAviForAccount(ctx, account, t, refresh); err != nil { - // if this doesn't work, just skip it -- we can do it later - l.Debugf("error fetching header/avi for account: %s", err) + if err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking); err != nil { + return fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err) } return nil } -// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport -// on behalf of requestingUsername. +// fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account, +// using a transport on behalf of requestingUsername. // // targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. // -// SIDE EFFECTS: remote header and avatar will be stored in local storage. -func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { +// If refresh is true, then the media will be fetched again even if it's already been fetched before. +// +// If blocking is true, then the calls to the media manager made by this function will be blocking: +// in other words, the function won't return until the header and the avatar have been fully processed. +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, refresh bool, blocking bool) error { accountURI, err := url.Parse(targetAccount.URI) if err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) + return fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err) } + if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) + return fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host) } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) - if err != nil { - return err - } + var processingMedia *media.ProcessingMedia - data := func(innerCtx context.Context) (io.Reader, int, error) { - return t.DereferenceMedia(innerCtx, avatarIRI) + // first check if we're already processing this media + d.dereferencingAvatarsLock.Lock() + if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok { + // we're already on it, no worries + processingMedia = alreadyProcessing } - - avatar := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ - RemoteURL: &targetAccount.AvatarRemoteURL, - Avatar: &avatar, - }) - if err != nil { - return err + d.dereferencingAvatarsLock.Unlock() + + if processingMedia == nil { + // we're not already processing it so start now + avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) + if err != nil { + return err + } + + data := func(innerCtx context.Context) (io.Reader, int, error) { + return t.DereferenceMedia(innerCtx, avatarIRI) + } + + avatar := true + newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ + RemoteURL: &targetAccount.AvatarRemoteURL, + Avatar: &avatar, + }) + if err != nil { + return err + } + targetAccount.AvatarMediaAttachmentID = newProcessing.AttachmentID() + + // store it in our map to indicate it's in process + d.dereferencingAvatarsLock.Lock() + d.dereferencingAvatars[targetAccount.ID] = newProcessing + d.dereferencingAvatarsLock.Unlock() + + processingMedia = newProcessing } - targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() + // block until loaded if required... + if blocking { + if err := lockAndLoad(ctx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil { + return err + } + } else { + // ...otherwise do it async + go func() { + if err := lockAndLoad(ctx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil { + logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of avatar: %s", err) + } + }() + } } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) - if err != nil { - return err - } + var processingMedia *media.ProcessingMedia - data := func(innerCtx context.Context) (io.Reader, int, error) { - return t.DereferenceMedia(innerCtx, headerIRI) + // first check if we're already processing this media + d.dereferencingHeadersLock.Lock() + if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok { + // we're already on it, no worries + processingMedia = alreadyProcessing } - - header := true - processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ - RemoteURL: &targetAccount.HeaderRemoteURL, - Header: &header, - }) - if err != nil { - return err + d.dereferencingHeadersLock.Unlock() + + if processingMedia == nil { + // we're not already processing it so start now + headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) + if err != nil { + return err + } + + data := func(innerCtx context.Context) (io.Reader, int, error) { + return t.DereferenceMedia(innerCtx, headerIRI) + } + + header := true + newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{ + RemoteURL: &targetAccount.HeaderRemoteURL, + Header: &header, + }) + if err != nil { + return err + } + targetAccount.HeaderMediaAttachmentID = newProcessing.AttachmentID() + + // store it in our map to indicate it's in process + d.dereferencingHeadersLock.Lock() + d.dereferencingHeaders[targetAccount.ID] = newProcessing + d.dereferencingHeadersLock.Unlock() + + processingMedia = newProcessing } - targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() + // block until loaded if required... + if blocking { + if err := lockAndLoad(ctx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil { + return err + } + } else { + // ...otherwise do it async + go func() { + if err := lockAndLoad(ctx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil { + logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of header: %s", err) + } + }() + } } + return nil } + +func lockAndLoad(ctx context.Context, lock *sync.Mutex, processing *media.ProcessingMedia, processingMap map[string]*media.ProcessingMedia, accountID string) error { + // whatever happens, remove the in-process media from the map + defer func() { + lock.Lock() + delete(processingMap, accountID) + lock.Unlock() + }() + + // try and load it + _, err := processing.LoadAttachment(ctx) + return err +} diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index 593ad341c..cb6f9588c 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -35,11 +35,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() { fetchingAccount := suite.testAccounts["local_account_1"] groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group") - group, new, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false) + group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false) suite.NoError(err) suite.NotNil(group) suite.NotNil(group) - suite.True(new) // group values should be set suite.Equal("https://unknown-instance.com/groups/some_group", group.URI) diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 800da6c04..daf82b91e 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -33,8 +33,7 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) - EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool, blocking bool) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) @@ -50,21 +49,29 @@ type Dereferencer interface { } type deref struct { - db db.DB - typeConverter typeutils.TypeConverter - transportController transport.Controller - mediaManager media.Manager - handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map + db db.DB + typeConverter typeutils.TypeConverter + transportController transport.Controller + mediaManager media.Manager + dereferencingAvatars map[string]*media.ProcessingMedia + dereferencingAvatarsLock *sync.Mutex + dereferencingHeaders map[string]*media.ProcessingMedia + dereferencingHeadersLock *sync.Mutex + handshakes map[string][]*url.URL + handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewDereferencer returns a Dereferencer initialized with the given parameters. func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer { return &deref{ - db: db, - typeConverter: typeConverter, - transportController: transportController, - mediaManager: mediaManager, - handshakeSync: &sync.Mutex{}, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaManager: mediaManager, + dereferencingAvatars: make(map[string]*media.ProcessingMedia), + dereferencingAvatarsLock: &sync.Mutex{}, + dereferencingHeaders: make(map[string]*media.ProcessingMedia), + dereferencingHeadersLock: &sync.Mutex{}, + handshakeSync: &sync.Mutex{}, } } diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 004d648b5..34310f4aa 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -89,7 +89,7 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat } // do this so we know we have the remote account of the status in the db - _, _, err = d.GetRemoteAccount(ctx, username, accountURI, false) + _, err = d.GetRemoteAccount(ctx, username, accountURI, false, false) if err != nil { return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err) } @@ -332,7 +332,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta if targetAccount == nil { // we didn't find the account in our database already // check if we can get the account remotely (dereference it) - if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err != nil { + if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil { errs = append(errs, err.Error()) } else { logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) @@ -394,7 +394,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{ + processingMedia, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{ CreatedAt: &a.CreatedAt, StatusID: &a.StatusID, RemoteURL: &a.RemoteURL, @@ -406,7 +406,7 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. continue } - attachment, err := media.LoadAttachment(ctx) + attachment, err := processingMedia.LoadAttachment(ctx) if err != nil { logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index f5d42a8e3..789959810 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -153,7 +153,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount, _, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false) + requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false) if err != nil { return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err) } diff --git a/internal/federation/federator.go b/internal/federation/federator.go index 7edff1118..cb63084db 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -57,8 +57,7 @@ type Federator interface { DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) - EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go index e96040db7..2571d7af1 100644 --- a/internal/processing/account/get.go +++ b/internal/processing/account/get.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "net/url" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -56,7 +57,12 @@ func (p *processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account // last-minute check to make sure we have remote account header/avi cached if targetAccount.Domain != "" { - a, err := p.federator.EnrichRemoteAccount(ctx, requestingAccount.Username, targetAccount) + targetAccountURI, err := url.Parse(targetAccount.URI) + if err != nil { + return nil, fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err) + } + + a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false) if err == nil { targetAccount = a } diff --git a/internal/processing/federation/getfollowers.go b/internal/processing/federation/getfollowers.go index 9153cde1e..c15b2b6c4 100644 --- a/internal/processing/federation/getfollowers.go +++ b/internal/processing/federation/getfollowers.go @@ -41,7 +41,7 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string, return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/federation/getfollowing.go b/internal/processing/federation/getfollowing.go index 8a3025154..d2beaada0 100644 --- a/internal/processing/federation/getfollowing.go +++ b/internal/processing/federation/getfollowing.go @@ -41,7 +41,7 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string, return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/federation/getoutbox.go b/internal/processing/federation/getoutbox.go index 0f2043447..944c0b571 100644 --- a/internal/processing/federation/getoutbox.go +++ b/internal/processing/federation/getoutbox.go @@ -42,7 +42,7 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/federation/getstatus.go b/internal/processing/federation/getstatus.go index f065eaa71..1651516b5 100644 --- a/internal/processing/federation/getstatus.go +++ b/internal/processing/federation/getstatus.go @@ -43,7 +43,7 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/federation/getstatusreplies.go b/internal/processing/federation/getstatusreplies.go index 4fc21e3ad..c6db4dd3e 100644 --- a/internal/processing/federation/getstatusreplies.go +++ b/internal/processing/federation/getstatusreplies.go @@ -43,7 +43,7 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/federation/getuser.go b/internal/processing/federation/getuser.go index a8d6bcf38..6d5b8463f 100644 --- a/internal/processing/federation/getuser.go +++ b/internal/processing/federation/getuser.go @@ -54,7 +54,7 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque // if we're not already handshaking/dereferencing a remote account, dereference it now if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) { - requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 533d00242..8b575dda8 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "net/url" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/ap" @@ -232,7 +233,12 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder return errors.New("profile was not parseable as *gtsmodel.Account") } - if _, err := p.federator.EnrichRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccount); err != nil { + incomingAccountURL, err := url.Parse(incomingAccount.URI) + if err != nil { + return err + } + + if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil { return fmt.Errorf("error enriching updated account from federator: %s", err) } diff --git a/internal/processing/search.go b/internal/processing/search.go index b03ced831..c8c302857 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -148,7 +148,7 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, if resolve { // we don't have it locally so try and dereference it - account, _, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true) + account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true) if err != nil { return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) } @@ -203,7 +203,7 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au } // we don't have it locally so try and dereference it - account, _, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true) + account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true) if err != nil { return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err) } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 8236fb3ae..52e89b7d2 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -96,35 +96,32 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A lastStatusAt = lastPosted.Format(time.RFC3339) } - // build the avatar and header URLs + // set account avatar fields if available var aviURL string var aviURLStatic string if a.AvatarMediaAttachmentID != "" { - // make sure avi is pinned to this account if a.AvatarMediaAttachment == nil { avi, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID) - if err != nil { - return nil, fmt.Errorf("error retrieving avatar: %s", err) + if err == nil { + a.AvatarMediaAttachment = avi + aviURL = a.AvatarMediaAttachment.URL + aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL } - a.AvatarMediaAttachment = avi } - aviURL = a.AvatarMediaAttachment.URL - aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL } + // set account header fields if available var headerURL string var headerURLStatic string if a.HeaderMediaAttachmentID != "" { - // make sure header is pinned to this account if a.HeaderMediaAttachment == nil { avi, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID) - if err != nil { - return nil, fmt.Errorf("error retrieving avatar: %s", err) + if err == nil { + a.HeaderMediaAttachment = avi + headerURL = a.HeaderMediaAttachment.URL + headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL } - a.HeaderMediaAttachment = avi } - headerURL = a.HeaderMediaAttachment.URL - headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL } // get the fields set on this account -- cgit v1.3 From f28cf793ee53e8391c9eabbfba93afbc5b59936b Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 17:35:13 +0100 Subject: upgrade go-store --- go.mod | 4 +- go.sum | 4 + vendor/codeberg.org/gruf/go-mutexes/map.go | 335 ++- .../codeberg.org/gruf/go-mutexes/mutex_timeout.go | 67 +- vendor/codeberg.org/gruf/go-nowish/LICENSE | 9 - vendor/codeberg.org/gruf/go-nowish/README.md | 3 - vendor/codeberg.org/gruf/go-nowish/clock.go | 132 - vendor/codeberg.org/gruf/go-nowish/timeout.go | 233 -- vendor/codeberg.org/gruf/go-nowish/util.go | 10 - vendor/codeberg.org/gruf/go-store/kv/iterator.go | 10 +- vendor/codeberg.org/gruf/go-store/kv/state.go | 82 +- vendor/codeberg.org/gruf/go-store/kv/store.go | 40 +- vendor/codeberg.org/gruf/go-store/storage/block.go | 86 +- vendor/codeberg.org/gruf/go-store/storage/disk.go | 67 +- .../codeberg.org/gruf/go-store/storage/errors.go | 14 + vendor/codeberg.org/gruf/go-store/storage/fs.go | 9 +- vendor/codeberg.org/gruf/go-store/storage/lock.go | 75 +- .../codeberg.org/gruf/go-store/storage/memory.go | 78 +- vendor/github.com/zeebo/blake3/.gitignore | 6 - vendor/github.com/zeebo/blake3/LICENSE | 125 - vendor/github.com/zeebo/blake3/Makefile | 11 - vendor/github.com/zeebo/blake3/README.md | 77 - vendor/github.com/zeebo/blake3/api.go | 166 -- vendor/github.com/zeebo/blake3/blake3.go | 285 --- vendor/github.com/zeebo/blake3/digest.go | 100 - vendor/github.com/zeebo/blake3/internal/alg/alg.go | 18 - .../zeebo/blake3/internal/alg/compress/compress.go | 15 - .../alg/compress/compress_pure/compress.go | 135 -- .../alg/compress/compress_sse41/impl_amd64.s | 560 ----- .../alg/compress/compress_sse41/impl_other.go | 9 - .../internal/alg/compress/compress_sse41/stubs.go | 6 - .../zeebo/blake3/internal/alg/hash/hash.go | 23 - .../internal/alg/hash/hash_avx2/impl_amd64.s | 2561 -------------------- .../internal/alg/hash/hash_avx2/impl_other.go | 13 - .../blake3/internal/alg/hash/hash_avx2/stubs.go | 9 - .../blake3/internal/alg/hash/hash_pure/hashf.go | 56 - .../blake3/internal/alg/hash/hash_pure/hashp.go | 38 - .../zeebo/blake3/internal/consts/consts.go | 29 - .../github.com/zeebo/blake3/internal/consts/cpu.go | 17 - .../zeebo/blake3/internal/consts/cpu_big.go | 5 - .../zeebo/blake3/internal/consts/cpu_little.go | 5 - .../zeebo/blake3/internal/consts/cpu_other.go | 7 - .../zeebo/blake3/internal/utils/utils.go | 60 - vendor/modules.txt | 15 +- 44 files changed, 645 insertions(+), 4964 deletions(-) delete mode 100644 vendor/codeberg.org/gruf/go-nowish/LICENSE delete mode 100644 vendor/codeberg.org/gruf/go-nowish/README.md delete mode 100644 vendor/codeberg.org/gruf/go-nowish/clock.go delete mode 100644 vendor/codeberg.org/gruf/go-nowish/timeout.go delete mode 100644 vendor/codeberg.org/gruf/go-nowish/util.go delete mode 100644 vendor/github.com/zeebo/blake3/.gitignore delete mode 100644 vendor/github.com/zeebo/blake3/LICENSE delete mode 100644 vendor/github.com/zeebo/blake3/Makefile delete mode 100644 vendor/github.com/zeebo/blake3/README.md delete mode 100644 vendor/github.com/zeebo/blake3/api.go delete mode 100644 vendor/github.com/zeebo/blake3/blake3.go delete mode 100644 vendor/github.com/zeebo/blake3/digest.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/alg.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/consts/consts.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go delete mode 100644 vendor/github.com/zeebo/blake3/internal/utils/utils.go diff --git a/go.mod b/go.mod index b2ceb76aa..9c33f7f36 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( codeberg.org/gruf/go-runners v1.2.0 - codeberg.org/gruf/go-store v1.2.2 + codeberg.org/gruf/go-store v1.3.2 github.com/ReneKroon/ttlcache v1.7.0 github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.1.0 @@ -50,7 +50,7 @@ require ( codeberg.org/gruf/go-fastpath v1.0.2 // indirect codeberg.org/gruf/go-format v1.0.3 // indirect codeberg.org/gruf/go-hashenc v1.0.1 // indirect - codeberg.org/gruf/go-mutexes v1.0.1 // indirect + codeberg.org/gruf/go-mutexes v1.1.0 // indirect codeberg.org/gruf/go-nowish v1.1.0 // indirect codeberg.org/gruf/go-pools v1.0.2 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index cc773bbfb..ccfb51a1c 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ codeberg.org/gruf/go-hashenc v1.0.1 h1:EBvNe2wW8IPMUqT1XihB6/IM6KMJDLMFBxIUvmsy1 codeberg.org/gruf/go-hashenc v1.0.1/go.mod h1:IfHhPCVScOiYmJLqdCQT9bYVS1nxNTV4ewMUvFWDPtc= codeberg.org/gruf/go-mutexes v1.0.1 h1:X9bZW74YSEplWWdCrVXAvue5ztw3w5hh+INdXTENu88= codeberg.org/gruf/go-mutexes v1.0.1/go.mod h1:y2hbGLkWVHhNyxBOIVsA3/y2QMm6RSrYsC3sLVZ4EXM= +codeberg.org/gruf/go-mutexes v1.1.0 h1:kMVWHLxdfGEZTetNVRncdBMeqS4M8dSJxSGbRYXyvKk= +codeberg.org/gruf/go-mutexes v1.1.0/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8= codeberg.org/gruf/go-nowish v1.0.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= codeberg.org/gruf/go-nowish v1.1.0 h1:rj1z0AXDhLvnxs/DazWFxYAugs6rv5vhgWJkRCgrESg= codeberg.org/gruf/go-nowish v1.1.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= @@ -72,6 +74,8 @@ codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBs codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-store v1.2.2 h1:YJPzJpZv/D3t9hQC00/u76eQDScQw4++OWjfobnjHAA= codeberg.org/gruf/go-store v1.2.2/go.mod h1:Xjw1U098th0yXF2CCx6jThQ+9FIPWAX9OGjYslO+UtE= +codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo= +codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/vendor/codeberg.org/gruf/go-mutexes/map.go b/vendor/codeberg.org/gruf/go-mutexes/map.go index ea917ee5e..cb31a9543 100644 --- a/vendor/codeberg.org/gruf/go-mutexes/map.go +++ b/vendor/codeberg.org/gruf/go-mutexes/map.go @@ -1,105 +1,322 @@ package mutexes import ( + "runtime" "sync" + "sync/atomic" ) +// locktype defines maskable mutexmap lock types. +type locktype uint8 + +const ( + // possible lock types. + lockTypeRead = locktype(1) << 0 + lockTypeWrite = locktype(1) << 1 + lockTypeMap = locktype(1) << 2 + + // possible mutexmap states. + stateUnlockd = uint8(0) + stateRLocked = uint8(1) + stateLocked = uint8(2) + stateInUse = uint8(3) +) + +// permitLockType returns if provided locktype is permitted to go ahead in current state. +func permitLockType(state uint8, lt locktype) bool { + switch state { + // Unlocked state + // (all allowed) + case stateUnlockd: + return true + + // Keys locked, no state lock. + // (don't allow map locks) + case stateInUse: + return lt&lockTypeMap == 0 + + // Read locked + // (only allow read locks) + case stateRLocked: + return lt&lockTypeRead != 0 + + // Write locked + // (none allowed) + case stateLocked: + return false + + // shouldn't reach here + default: + panic("unexpected state") + } +} + // MutexMap is a structure that allows having a map of self-evicting mutexes // by key. You do not need to worry about managing the contents of the map, // only requesting RLock/Lock for keys, and ensuring to call the returned // unlock functions. type MutexMap struct { - // NOTE: - // Individual keyed mutexes should ONLY ever - // be locked within the protection of the outer - // mapMu lock. If you lock these outside the - // protection of this, there is a chance for - // deadlocks - mus map[string]RWMutex mapMu sync.Mutex pool sync.Pool + queue []func() + evict []func() + count int32 + maxmu int32 + state uint8 } -// NewMap returns a new MutexMap instance based on supplied -// RWMutex allocator function, nil implies use default -func NewMap(newFn func() RWMutex) MutexMap { - if newFn == nil { - newFn = NewRW +// NewMap returns a new MutexMap instance with provided max no. open mutexes. +func NewMap(max int32) MutexMap { + if max < 1 { + // Default = 128 * GOMAXPROCS + procs := runtime.GOMAXPROCS(0) + max = int32(procs * 128) } return MutexMap{ - mus: make(map[string]RWMutex), - mapMu: sync.Mutex{}, + mus: make(map[string]RWMutex), pool: sync.Pool{ New: func() interface{} { - return newFn() + return NewRW() }, }, + maxmu: max, + } +} + +// acquire will either acquire a mutex from pool or alloc. +func (mm *MutexMap) acquire() RWMutex { + return mm.pool.Get().(RWMutex) +} + +// release will release provided mutex to pool. +func (mm *MutexMap) release(mu RWMutex) { + mm.pool.Put(mu) +} + +// spinLock will wait (using a mutex to sleep thread) until 'cond()' returns true, +// returning with map lock. Note that 'cond' is performed within a map lock. +func (mm *MutexMap) spinLock(cond func() bool) { + mu := mm.acquire() + defer mm.release(mu) + + for { + // Get map lock + mm.mapMu.Lock() + + // Check if return + if cond() { + return + } + + // Queue ourselves + unlock := mu.Lock() + mm.queue = append(mm.queue, unlock) + mm.mapMu.Unlock() + + // Wait on notify + mu.Lock()() + } +} + +// lockMutex will acquire a lock on the mutex at provided key, handling earlier allocated mutex if provided. Unlocks map on return. +func (mm *MutexMap) lockMutex(key string, lt locktype) func() { + var unlock func() + + // Incr counter + mm.count++ + + // Check for existing mutex at key + mu, ok := mm.mus[key] + if !ok { + // Alloc from pool + mu = mm.acquire() + mm.mus[key] = mu + + // Queue mutex for eviction + mm.evict = append(mm.evict, func() { + delete(mm.mus, key) + mm.pool.Put(mu) + }) + } + + // If no state, set in use. + // State will already have been + // set if this is from LockState{} + if mm.state == stateUnlockd { + mm.state = stateInUse + } + + switch { + // Read lock + case lt&lockTypeRead != 0: + unlock = mu.RLock() + + // Write lock + case lt&lockTypeWrite != 0: + unlock = mu.Lock() + + // shouldn't reach here + default: + panic("unexpected lock type") + } + + // Unlock map + return + mm.mapMu.Unlock() + return func() { + mm.mapMu.Lock() + unlock() + go mm.onUnlock() } } -func (mm *MutexMap) evict(key string, mu RWMutex) { - // Acquire map lock - mm.mapMu.Lock() +// onUnlock is performed as the final (async) stage of releasing an acquired key / map mutex. +func (mm *MutexMap) onUnlock() { + // Decr counter + mm.count-- + + if mm.count < 1 { + // Perform all queued evictions + for i := 0; i < len(mm.evict); i++ { + mm.evict[i]() + } - // Toggle mutex lock to - // ensure it is unused - unlock := mu.Lock() - unlock() + // Notify all waiting goroutines + for i := 0; i < len(mm.queue); i++ { + mm.queue[i]() + } - // Delete mutex key - delete(mm.mus, key) + // Reset the map state + mm.evict = nil + mm.queue = nil + mm.state = stateUnlockd + } + + // Finally, unlock mm.mapMu.Unlock() +} - // Release to pool - mm.pool.Put(mu) +// RLockMap acquires a read lock over the entire map, returning a lock state for acquiring key read locks. +// Please note that the 'unlock()' function will block until all keys locked from this state are unlocked. +func (mm *MutexMap) RLockMap() *LockState { + return mm.getMapLock(lockTypeRead) } -// RLock acquires a mutex read lock for supplied key, returning an RUnlock function -func (mm *MutexMap) RLock(key string) func() { - return mm.getLock(key, func(mu RWMutex) func() { - return mu.RLock() +// LockMap acquires a write lock over the entire map, returning a lock state for acquiring key read/write locks. +// Please note that the 'unlock()' function will block until all keys locked from this state are unlocked. +func (mm *MutexMap) LockMap() *LockState { + return mm.getMapLock(lockTypeWrite) +} + +// RLock acquires a mutex read lock for supplied key, returning an RUnlock function. +func (mm *MutexMap) RLock(key string) (runlock func()) { + return mm.getLock(key, lockTypeRead) +} + +// Lock acquires a mutex write lock for supplied key, returning an Unlock function. +func (mm *MutexMap) Lock(key string) (unlock func()) { + return mm.getLock(key, lockTypeWrite) +} + +// getLock will fetch lock of provided type, for given key, returning unlock function. +func (mm *MutexMap) getLock(key string, lt locktype) func() { + // Spin until achieve lock + mm.spinLock(func() bool { + return permitLockType(mm.state, lt) && + mm.count < mm.maxmu // not overloaded }) + + // Perform actual mutex lock + return mm.lockMutex(key, lt) } -// Lock acquires a mutex lock for supplied key, returning an Unlock function -func (mm *MutexMap) Lock(key string) func() { - return mm.getLock(key, func(mu RWMutex) func() { - return mu.Lock() +// getMapLock will acquire a map lock of provided type, returning a LockState session. +func (mm *MutexMap) getMapLock(lt locktype) *LockState { + // Spin until achieve lock + mm.spinLock(func() bool { + return permitLockType(mm.state, lt|lockTypeMap) && + mm.count < mm.maxmu // not overloaded }) + + // Incr counter + mm.count++ + + switch { + // Set read lock state + case lt&lockTypeRead != 0: + mm.state = stateRLocked + + // Set write lock state + case lt&lockTypeWrite != 0: + mm.state = stateLocked + + default: + panic("unexpected lock type") + } + + // Unlock + return + mm.mapMu.Unlock() + return &LockState{ + mmap: mm, + ltyp: lt, + } } -func (mm *MutexMap) getLock(key string, doLock func(RWMutex) func()) func() { - // Get map lock - mm.mapMu.Lock() +// LockState represents a window to a locked MutexMap. +type LockState struct { + wait sync.WaitGroup + mmap *MutexMap + done uint32 + ltyp locktype +} - // Look for mutex - mu, ok := mm.mus[key] - if ok { - // Lock and return - // its unlocker func - unlock := doLock(mu) - mm.mapMu.Unlock() - return unlock +// Lock: see MutexMap.Lock() definition. Will panic if map only read locked. +func (st *LockState) Lock(key string) (unlock func()) { + return st.getLock(key, lockTypeWrite) +} + +// RLock: see MutexMap.RLock() definition. +func (st *LockState) RLock(key string) (runlock func()) { + return st.getLock(key, lockTypeRead) +} + +// UnlockMap will close this state and release the currently locked map. +func (st *LockState) UnlockMap() { + // Set state to finished (or panic if already done) + if !atomic.CompareAndSwapUint32(&st.done, 0, 1) { + panic("called UnlockMap() on expired state") } - // Note: even though the mutex data structure is - // small, benchmarking does actually show that pooled - // alloc of mutexes here is faster + // Wait until done + st.wait.Wait() - // Acquire mu + add - mu = mm.pool.Get().(RWMutex) - mm.mus[key] = mu + // Async reset map + st.mmap.mapMu.Lock() + go st.mmap.onUnlock() +} - // Lock mutex + unlock map - unlockFn := doLock(mu) - mm.mapMu.Unlock() +// getLock: see MutexMap.getLock() definition. +func (st *LockState) getLock(key string, lt locktype) func() { + st.wait.Add(1) // track lock - return func() { - // Unlock mutex - unlockFn() + // Check if closed, or if write lock is allowed + if atomic.LoadUint32(&st.done) == 1 { + panic("map lock closed") + } else if lt&lockTypeWrite != 0 && + st.ltyp&lockTypeWrite == 0 { + panic("called .Lock() on rlocked map") + } + + // Spin until achieve map lock + st.mmap.spinLock(func() bool { + return st.mmap.count < st.mmap.maxmu + }) // i.e. not overloaded - // Release function - go mm.evict(key, mu) + // Perform actual mutex lock + unlock := st.mmap.lockMutex(key, lt) + + return func() { + unlock() + st.wait.Done() } } diff --git a/vendor/codeberg.org/gruf/go-mutexes/mutex_timeout.go b/vendor/codeberg.org/gruf/go-mutexes/mutex_timeout.go index 5da69ef25..2e7b8f802 100644 --- a/vendor/codeberg.org/gruf/go-mutexes/mutex_timeout.go +++ b/vendor/codeberg.org/gruf/go-mutexes/mutex_timeout.go @@ -3,8 +3,6 @@ package mutexes import ( "sync" "time" - - "codeberg.org/gruf/go-nowish" ) // TimeoutMutex defines a Mutex with timeouts on locks @@ -73,14 +71,6 @@ func (mu *timeoutRWMutex) RLockFunc(fn func()) func() { return mutexTimeout(mu.rd, mu.mu.RLock(), fn) } -// timeoutPool provides nowish.Timeout objects for timeout mutexes -var timeoutPool = sync.Pool{ - New: func() interface{} { - t := nowish.NewTimeout() - return &t - }, -} - // mutexTimeout performs a timed unlock, calling supplied fn if timeout is reached func mutexTimeout(d time.Duration, unlock func(), fn func()) func() { if d < 1 { @@ -88,18 +78,65 @@ func mutexTimeout(d time.Duration, unlock func(), fn func()) func() { return unlock } - // Acquire timeout obj - t := timeoutPool.Get().(*nowish.Timeout) + // Acquire timer from pool + t := timerPool.Get().(*timer) - // Start the timeout with hook - t.Start(d, fn) + // Start the timer + go t.Start(d, fn) // Return func cancelling timeout, // replacing Timeout in pool and // finally unlocking mutex return func() { + defer timerPool.Put(t) t.Cancel() - timeoutPool.Put(t) unlock() } } + +// timerPool is the global &timer{} pool. +var timerPool = sync.Pool{ + New: func() interface{} { + return newtimer() + }, +} + +// timer represents a reusable cancellable timer. +type timer struct { + t *time.Timer + c chan struct{} +} + +// newtimer returns a new timer instance. +func newtimer() *timer { + t := time.NewTimer(time.Minute) + t.Stop() + return &timer{t: t, c: make(chan struct{})} +} + +// Start will start the timer with duration 'd', performing 'fn' on timeout. +func (t *timer) Start(d time.Duration, fn func()) { + t.t.Reset(d) + select { + // Timed out + case <-t.t.C: + fn() + + // Cancelled + case <-t.c: + } +} + +// Cancel will attempt to cancel the running timer. +func (t *timer) Cancel() { + select { + // cancel successful + case t.c <- struct{}{}: + if !t.t.Stop() { + <-t.t.C + } // stop timer + + // already stopped + default: + } +} diff --git a/vendor/codeberg.org/gruf/go-nowish/LICENSE b/vendor/codeberg.org/gruf/go-nowish/LICENSE deleted file mode 100644 index b7c4417ac..000000000 --- a/vendor/codeberg.org/gruf/go-nowish/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2021 gruf - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/codeberg.org/gruf/go-nowish/README.md b/vendor/codeberg.org/gruf/go-nowish/README.md deleted file mode 100644 index 4070e5013..000000000 --- a/vendor/codeberg.org/gruf/go-nowish/README.md +++ /dev/null @@ -1,3 +0,0 @@ -a simple Go library with useful time utiities: -- Clock: a high performance clock giving a good "ish" representation of "now" (hence the name!) -- Timeout: a reusable structure for enforcing timeouts with a cancel diff --git a/vendor/codeberg.org/gruf/go-nowish/clock.go b/vendor/codeberg.org/gruf/go-nowish/clock.go deleted file mode 100644 index 781e59f18..000000000 --- a/vendor/codeberg.org/gruf/go-nowish/clock.go +++ /dev/null @@ -1,132 +0,0 @@ -package nowish - -import ( - "sync" - "sync/atomic" - "time" - "unsafe" -) - -// Start returns a new Clock instance initialized and -// started with the provided precision, along with the -// stop function for it's underlying timer -func Start(precision time.Duration) (*Clock, func()) { - c := Clock{} - return &c, c.Start(precision) -} - -type Clock struct { - // format stores the time formatting style string - format string - - // valid indicates whether the current value stored in .Format is valid - valid uint32 - - // mutex protects writes to .Format, not because it would be unsafe, but - // because we want to minimize unnnecessary allocations - mutex sync.Mutex - - // nowfmt is an unsafe pointer to the last-updated time format string - nowfmt unsafe.Pointer - - // now is an unsafe pointer to the last-updated time.Time object - now unsafe.Pointer -} - -// Start starts the clock with the provided precision, the returned -// function is the stop function for the underlying timer. For >= 2ms, -// actual precision is usually within AT LEAST 10% of requested precision, -// less than this and the actual precision very quickly deteriorates. -func (c *Clock) Start(precision time.Duration) func() { - // Create ticker from duration - tick := time.NewTicker(precision / 10) - - // Set initial time - t := time.Now() - atomic.StorePointer(&c.now, unsafe.Pointer(&t)) - - // Set initial format - s := "" - atomic.StorePointer(&c.nowfmt, unsafe.Pointer(&s)) - - // If formatting string unset, set default - c.mutex.Lock() - if c.format == "" { - c.format = time.RFC822 - } - c.mutex.Unlock() - - // Start main routine - go c.run(tick) - - // Return stop fn - return tick.Stop -} - -// run is the internal clock ticking loop. -func (c *Clock) run(tick *time.Ticker) { - for { - // Wait on tick - _, ok := <-tick.C - - // Channel closed - if !ok { - break - } - - // Update time - t := time.Now() - atomic.StorePointer(&c.now, unsafe.Pointer(&t)) - - // Invalidate format string - atomic.StoreUint32(&c.valid, 0) - } -} - -// Now returns a good (ish) estimate of the current 'now' time. -func (c *Clock) Now() time.Time { - return *(*time.Time)(atomic.LoadPointer(&c.now)) -} - -// NowFormat returns the formatted "now" time, cached until next tick and "now" updates. -func (c *Clock) NowFormat() string { - // If format still valid, return this - if atomic.LoadUint32(&c.valid) == 1 { - return *(*string)(atomic.LoadPointer(&c.nowfmt)) - } - - // Get mutex lock - c.mutex.Lock() - - // Double check still invalid - if atomic.LoadUint32(&c.valid) == 1 { - c.mutex.Unlock() - return *(*string)(atomic.LoadPointer(&c.nowfmt)) - } - - // Calculate time format - nowfmt := c.Now().Format(c.format) - - // Update the stored value and set valid! - atomic.StorePointer(&c.nowfmt, unsafe.Pointer(&nowfmt)) - atomic.StoreUint32(&c.valid, 1) - - // Unlock and return - c.mutex.Unlock() - return nowfmt -} - -// SetFormat sets the time format string used by .NowFormat(). -func (c *Clock) SetFormat(format string) { - // Get mutex lock - c.mutex.Lock() - - // Update time format - c.format = format - - // Invalidate current format string - atomic.StoreUint32(&c.valid, 0) - - // Unlock - c.mutex.Unlock() -} diff --git a/vendor/codeberg.org/gruf/go-nowish/timeout.go b/vendor/codeberg.org/gruf/go-nowish/timeout.go deleted file mode 100644 index 7fe3e1d1d..000000000 --- a/vendor/codeberg.org/gruf/go-nowish/timeout.go +++ /dev/null @@ -1,233 +0,0 @@ -package nowish - -import ( - "sync" - "sync/atomic" - "time" -) - -// Timeout provides a reusable structure for enforcing timeouts with a cancel. -type Timeout struct { - timer *time.Timer // timer is the underlying timeout-timer - cncl syncer // cncl is the cancel synchronization channel - next int64 // next is the next timeout duration to run on - state uint32 // state stores the current timeout state - mu sync.Mutex // mu protects state, and helps synchronize return of .Start() -} - -// NewTimeout returns a new Timeout instance. -func NewTimeout() Timeout { - timer := time.NewTimer(time.Minute) - timer.Stop() // don't keep it running - return Timeout{ - timer: timer, - cncl: make(syncer), - } -} - -// startTimeout is the main timeout routine, handling starting the -// timeout runner at first and upon any time extensions, and handling -// any received cancels by stopping the running timer. -func (t *Timeout) startTimeout(hook func()) { - var cancelled bool - - // Receive first timeout duration - d := atomic.SwapInt64(&t.next, 0) - - // Indicate finished starting, this - // was left locked by t.start(). - t.mu.Unlock() - - for { - // Run supplied timeout - cancelled = t.runTimeout(d) - if cancelled { - break - } - - // Check for extension or set timed out - d = atomic.SwapInt64(&t.next, 0) - if d < 1 { - if t.timedOut() { - // timeout reached - hook() - break - } else { - // already cancelled - t.cncl.wait() - cancelled = true - break - } - } - - if !t.extend() { - // already cancelled - t.cncl.wait() - cancelled = true - break - } - } - - if cancelled { - // Release the .Cancel() - defer t.cncl.notify() - } - - // Mark as done - t.reset() -} - -// runTimeout will until supplied timeout or cancel called. -func (t *Timeout) runTimeout(d int64) (cancelled bool) { - // Start the timer for 'd' - t.timer.Reset(time.Duration(d)) - - select { - // Timeout reached - case <-t.timer.C: - if !t.timingOut() { - // a sneaky cancel! - t.cncl.wait() - cancelled = true - } - - // Cancel called - case <-t.cncl.wait(): - cancelled = true - if !t.timer.Stop() { - <-t.timer.C - } - } - - return cancelled -} - -// Start starts the timer with supplied timeout. If timeout is reached before -// cancel then supplied timeout hook will be called. Panic will be called if -// Timeout is already running when calling this function. -func (t *Timeout) Start(d time.Duration, hook func()) { - if !t.start() { - t.mu.Unlock() // need to unlock - panic("timeout already started") - } - - // Start the timeout - atomic.StoreInt64(&t.next, int64(d)) - go t.startTimeout(hook) - - // Wait until start - t.mu.Lock() - t.mu.Unlock() -} - -// Extend will attempt to extend the timeout runner's time, returns false if not running. -func (t *Timeout) Extend(d time.Duration) bool { - var ok bool - if ok = t.running(); ok { - atomic.AddInt64(&t.next, int64(d)) - } - return ok -} - -// Cancel cancels the currently running timer. If a cancel is achieved, then -// this function will return after the timeout goroutine is finished. -func (t *Timeout) Cancel() { - if !t.cancel() { - return - } - t.cncl.notify() - <-t.cncl.wait() -} - -// possible timeout states. -const ( - stopped = 0 - started = 1 - timingOut = 2 - cancelled = 3 - timedOut = 4 -) - -// cas will perform a compare and swap where the compare is a provided function. -func (t *Timeout) cas(check func(uint32) bool, swap uint32) bool { - var cas bool - - t.mu.Lock() - if cas = check(t.state); cas { - t.state = swap - } - t.mu.Unlock() - - return cas -} - -// start attempts to mark the timeout state as 'started', note DOES NOT unlock Timeout.mu. -func (t *Timeout) start() bool { - var ok bool - - t.mu.Lock() - if ok = (t.state == stopped); ok { - t.state = started - } - - // don't unlock - return ok -} - -// timingOut attempts to mark the timeout state as 'timing out'. -func (t *Timeout) timingOut() bool { - return t.cas(func(u uint32) bool { - return (u == started) - }, timingOut) -} - -// timedOut attempts mark the 'timing out' state as 'timed out'. -func (t *Timeout) timedOut() bool { - return t.cas(func(u uint32) bool { - return (u == timingOut) - }, timedOut) -} - -// extend attempts to extend a 'timing out' state by moving it back to 'started'. -func (t *Timeout) extend() bool { - return t.cas(func(u uint32) bool { - return (u == started) || - (u == timingOut) - }, started) -} - -// running returns whether the state is anything other than 'stopped'. -func (t *Timeout) running() bool { - t.mu.Lock() - running := (t.state != stopped) - t.mu.Unlock() - return running -} - -// cancel attempts to mark the timeout state as 'cancelled'. -func (t *Timeout) cancel() bool { - return t.cas(func(u uint32) bool { - return (u == started) || - (u == timingOut) - }, cancelled) -} - -// reset marks the timeout state as 'stopped'. -func (t *Timeout) reset() { - t.mu.Lock() - t.state = stopped - t.mu.Unlock() -} - -// syncer provides helpful receiver methods for a synchronization channel. -type syncer (chan struct{}) - -// notify blocks on sending an empty value down channel. -func (s syncer) notify() { - s <- struct{}{} -} - -// wait returns the underlying channel for blocking until '.notify()'. -func (s syncer) wait() <-chan struct{} { - return s -} diff --git a/vendor/codeberg.org/gruf/go-nowish/util.go b/vendor/codeberg.org/gruf/go-nowish/util.go deleted file mode 100644 index 31fe9050e..000000000 --- a/vendor/codeberg.org/gruf/go-nowish/util.go +++ /dev/null @@ -1,10 +0,0 @@ -package nowish - -//nolint -type noCopy struct{} - -//nolint -func (*noCopy) Lock() {} - -//nolint -func (*noCopy) Unlock() {} diff --git a/vendor/codeberg.org/gruf/go-store/kv/iterator.go b/vendor/codeberg.org/gruf/go-store/kv/iterator.go index ddaaf60cf..da743ead1 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/iterator.go +++ b/vendor/codeberg.org/gruf/go-store/kv/iterator.go @@ -2,6 +2,7 @@ package kv import ( "codeberg.org/gruf/go-errors" + "codeberg.org/gruf/go-mutexes" "codeberg.org/gruf/go-store/storage" ) @@ -17,10 +18,10 @@ var ErrIteratorClosed = errors.New("store/kv: iterator closed") // have multiple iterators running concurrently type KVIterator struct { store *KVStore // store is the linked KVStore + state *mutexes.LockState entries []storage.StorageEntry index int key string - onClose func() } // Next attempts to set the next key-value pair, the @@ -43,13 +44,10 @@ func (i *KVIterator) Key() string { // Release releases the KVIterator and KVStore's read lock func (i *KVIterator) Release() { - // Reset key, path, entries + i.state.UnlockMap() i.store = nil i.key = "" i.entries = nil - - // Perform requested callback - i.onClose() } // Value returns the next value from the KVStore @@ -60,5 +58,5 @@ func (i *KVIterator) Value() ([]byte, error) { } // Attempt to fetch from store - return i.store.get(i.store.mutexMap.RLock, i.key) + return i.store.get(i.state.RLock, i.key) } diff --git a/vendor/codeberg.org/gruf/go-store/kv/state.go b/vendor/codeberg.org/gruf/go-store/kv/state.go index 330928bce..0b226e107 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/state.go +++ b/vendor/codeberg.org/gruf/go-store/kv/state.go @@ -2,9 +2,9 @@ package kv import ( "io" - "sync" "codeberg.org/gruf/go-errors" + "codeberg.org/gruf/go-mutexes" ) var ErrStateClosed = errors.New("store/kv: state closed") @@ -16,61 +16,42 @@ var ErrStateClosed = errors.New("store/kv: state closed") // then the state has zero guarantees type StateRO struct { store *KVStore - mutex sync.RWMutex + state *mutexes.LockState } func (st *StateRO) Get(key string) ([]byte, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return nil, ErrStateClosed } // Pass request to store - return st.store.get(st.store.mutexMap.RLock, key) + return st.store.get(st.state.RLock, key) } func (st *StateRO) GetStream(key string) (io.ReadCloser, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return nil, ErrStateClosed } // Pass request to store - return st.store.getStream(st.store.mutexMap.RLock, key) + return st.store.getStream(st.state.RLock, key) } func (st *StateRO) Has(key string) (bool, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return false, ErrStateClosed } // Pass request to store - return st.store.has(st.store.mutexMap.RLock, key) + return st.store.has(st.state.RLock, key) } func (st *StateRO) Release() { - // Get state write lock - st.mutex.Lock() - defer st.mutex.Unlock() - - // Release the store - if st.store != nil { - st.store.mutex.RUnlock() - st.store = nil - } + st.state.UnlockMap() + st.store = nil } // StateRW provides a read-write window to the store. While this @@ -80,101 +61,70 @@ func (st *StateRO) Release() { // then the state has zero guarantees type StateRW struct { store *KVStore - mutex sync.RWMutex + state *mutexes.LockState } func (st *StateRW) Get(key string) ([]byte, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return nil, ErrStateClosed } // Pass request to store - return st.store.get(st.store.mutexMap.RLock, key) + return st.store.get(st.state.RLock, key) } func (st *StateRW) GetStream(key string) (io.ReadCloser, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return nil, ErrStateClosed } // Pass request to store - return st.store.getStream(st.store.mutexMap.RLock, key) + return st.store.getStream(st.state.RLock, key) } func (st *StateRW) Put(key string, value []byte) error { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return ErrStateClosed } // Pass request to store - return st.store.put(st.store.mutexMap.Lock, key, value) + return st.store.put(st.state.Lock, key, value) } func (st *StateRW) PutStream(key string, r io.Reader) error { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return ErrStateClosed } // Pass request to store - return st.store.putStream(st.store.mutexMap.Lock, key, r) + return st.store.putStream(st.state.Lock, key, r) } func (st *StateRW) Has(key string) (bool, error) { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return false, ErrStateClosed } // Pass request to store - return st.store.has(st.store.mutexMap.RLock, key) + return st.store.has(st.state.RLock, key) } func (st *StateRW) Delete(key string) error { - // Get state read lock - st.mutex.RLock() - defer st.mutex.RUnlock() - // Check not closed if st.store == nil { return ErrStateClosed } // Pass request to store - return st.store.delete(st.store.mutexMap.Lock, key) + return st.store.delete(st.state.Lock, key) } func (st *StateRW) Release() { - // Get state write lock - st.mutex.Lock() - defer st.mutex.Unlock() - - // Release the store - if st.store != nil { - st.store.mutex.Unlock() - st.store = nil - } + st.state.UnlockMap() + st.store = nil } diff --git a/vendor/codeberg.org/gruf/go-store/kv/store.go b/vendor/codeberg.org/gruf/go-store/kv/store.go index 4c3a31140..a8741afe0 100644 --- a/vendor/codeberg.org/gruf/go-store/kv/store.go +++ b/vendor/codeberg.org/gruf/go-store/kv/store.go @@ -2,7 +2,6 @@ package kv import ( "io" - "sync" "codeberg.org/gruf/go-mutexes" "codeberg.org/gruf/go-store/storage" @@ -11,9 +10,8 @@ import ( // KVStore is a very simple, yet performant key-value store type KVStore struct { - mutexMap mutexes.MutexMap // mutexMap is a map of keys to mutexes to protect file access - mutex sync.RWMutex // mutex is the total store mutex - storage storage.Storage // storage is the underlying storage + mutex mutexes.MutexMap // mutex is a map of keys to mutexes to protect file access + storage storage.Storage // storage is the underlying storage } func OpenFile(path string, cfg *storage.DiskConfig) (*KVStore, error) { @@ -47,26 +45,19 @@ func OpenStorage(storage storage.Storage) (*KVStore, error) { // Return new KVStore return &KVStore{ - mutexMap: mutexes.NewMap(mutexes.NewRW), - mutex: sync.RWMutex{}, - storage: storage, + mutex: mutexes.NewMap(-1), + storage: storage, }, nil } // RLock acquires a read-lock on supplied key, returning unlock function. func (st *KVStore) RLock(key string) (runlock func()) { - st.mutex.RLock() - runlock = st.mutexMap.RLock(key) - st.mutex.RUnlock() - return runlock + return st.mutex.RLock(key) } // Lock acquires a write-lock on supplied key, returning unlock function. func (st *KVStore) Lock(key string) (unlock func()) { - st.mutex.Lock() - unlock = st.mutexMap.Lock(key) - st.mutex.Unlock() - return unlock + return st.mutex.Lock(key) } // Get fetches the bytes for supplied key in the store @@ -167,7 +158,7 @@ func (st *KVStore) Iterator(matchFn func(string) bool) (*KVIterator, error) { } // Get store read lock - st.mutex.RLock() + state := st.mutex.RLockMap() // Setup the walk keys function entries := []storage.StorageEntry{} @@ -184,24 +175,24 @@ func (st *KVStore) Iterator(matchFn func(string) bool) (*KVIterator, error) { // Walk keys in the storage err := st.storage.WalkKeys(storage.WalkKeysOptions{WalkFn: walkFn}) if err != nil { - st.mutex.RUnlock() + state.UnlockMap() return nil, err } // Return new iterator return &KVIterator{ store: st, + state: state, entries: entries, index: -1, key: "", - onClose: st.mutex.RUnlock, }, nil } // Read provides a read-only window to the store, holding it in a read-locked state until release func (st *KVStore) Read() *StateRO { - st.mutex.RLock() - return &StateRO{store: st} + state := st.mutex.RLockMap() + return &StateRO{store: st, state: state} } // ReadFn provides a read-only window to the store, holding it in a read-locked state until fn return. @@ -216,8 +207,8 @@ func (st *KVStore) ReadFn(fn func(*StateRO)) { // Update provides a read-write window to the store, holding it in a write-locked state until release func (st *KVStore) Update() *StateRW { - st.mutex.Lock() - return &StateRW{store: st} + state := st.mutex.LockMap() + return &StateRW{store: st, state: state} } // UpdateFn provides a read-write window to the store, holding it in a write-locked state until fn return. @@ -229,3 +220,8 @@ func (st *KVStore) UpdateFn(fn func(*StateRW)) { // Pass to fn fn(state) } + +// Close will close the underlying storage, the mutex map locking (e.g. RLock(), Lock() will still work). +func (st *KVStore) Close() error { + return st.storage.Close() +} diff --git a/vendor/codeberg.org/gruf/go-store/storage/block.go b/vendor/codeberg.org/gruf/go-store/storage/block.go index bc35b07ac..5075c7d17 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/block.go +++ b/vendor/codeberg.org/gruf/go-store/storage/block.go @@ -1,6 +1,7 @@ package storage import ( + "crypto/sha256" "io" "io/fs" "os" @@ -13,7 +14,6 @@ import ( "codeberg.org/gruf/go-hashenc" "codeberg.org/gruf/go-pools" "codeberg.org/gruf/go-store/util" - "github.com/zeebo/blake3" ) var ( @@ -77,7 +77,7 @@ func getBlockConfig(cfg *BlockConfig) BlockConfig { // BlockStorage is a Storage implementation that stores input data as chunks on // a filesystem. Each value is chunked into blocks of configured size and these -// blocks are stored with name equal to their base64-encoded BLAKE3 hash-sum. A +// blocks are stored with name equal to their base64-encoded SHA256 hash-sum. A // "node" file is finally created containing an array of hashes contained within // this value type BlockStorage struct { @@ -87,7 +87,7 @@ type BlockStorage struct { config BlockConfig // cfg is the supplied configuration for this store hashPool sync.Pool // hashPool is this store's hashEncoder pool bufpool pools.BufferPool // bufpool is this store's bytes.Buffer pool - lock *LockableFile // lock is the opened lockfile for this storage instance + lock *Lock // lock is the opened lockfile for this storage instance // NOTE: // BlockStorage does not need to lock each of the underlying block files @@ -140,11 +140,9 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) { } // Open and acquire storage lock for path - lock, err := OpenLock(pb.Join(path, LockFile)) + lock, err := OpenLock(pb.Join(path, lockFile)) if err != nil { return nil, err - } else if err := lock.Lock(); err != nil { - return nil, err } // Figure out the largest size for bufpool slices @@ -174,14 +172,23 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) { // Clean implements storage.Clean() func (st *BlockStorage) Clean() error { - nodes := map[string]*node{} + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) - // Walk nodes dir for entries + nodes := map[string]*node{} onceErr := errors.OnceError{} + + // Walk nodes dir for entries err := util.WalkDir(pb, st.nodePath, func(npath string, fsentry fs.DirEntry) { // Only deal with regular files if !fsentry.Type().IsRegular() { @@ -303,6 +310,7 @@ func (st *BlockStorage) ReadBytes(key string) ([]byte, error) { if err != nil { return nil, err } + defer rc.Close() // Read all bytes and return return io.ReadAll(rc) @@ -316,9 +324,19 @@ func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error) { return nil, err } + // Track open + st.lock.Add() + + // Check if open + if st.lock.Closed() { + st.lock.Done() + return nil, ErrClosed + } + // Attempt to open RO file file, err := open(npath, defaultFileROFlags) if err != nil { + st.lock.Done() return nil, err } defer file.Close() @@ -338,14 +356,16 @@ func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error) { nil, ) if err != nil { + st.lock.Done() return nil, err } - // Return new block reader - return util.NopReadCloser(&blockReader{ + // Prepare block reader and return + rc := util.NopReadCloser(&blockReader{ storage: st, node: &node, - }), nil + }) // we wrap the blockreader to decr lockfile waitgroup + return util.ReadCloserWithCallback(rc, st.lock.Done), nil } func (st *BlockStorage) readBlock(key string) ([]byte, error) { @@ -383,6 +403,15 @@ func (st *BlockStorage) WriteStream(key string, r io.Reader) error { return err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Check if this exists ok, err := stat(key) if err != nil { @@ -567,6 +596,15 @@ func (st *BlockStorage) Stat(key string) (bool, error) { return false, err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return false, ErrClosed + } + // Check for file on disk return stat(kpath) } @@ -579,18 +617,35 @@ func (st *BlockStorage) Remove(key string) error { return err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Attempt to remove file return os.Remove(kpath) } // Close implements Storage.Close() func (st *BlockStorage) Close() error { - defer st.lock.Close() - return st.lock.Unlock() + return st.lock.Close() } // WalkKeys implements Storage.WalkKeys() func (st *BlockStorage) WalkKeys(opts WalkKeysOptions) error { + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) @@ -800,7 +855,7 @@ var ( // encodedHashLen is the once-calculated encoded hash-sum length encodedHashLen = base64Encoding.EncodedLen( - blake3.New().Size(), + sha256.New().Size(), ) ) @@ -812,9 +867,8 @@ type hashEncoder struct { // newHashEncoder returns a new hashEncoder instance func newHashEncoder() *hashEncoder { - hash := blake3.New() return &hashEncoder{ - henc: hashenc.New(hash, base64Encoding), + henc: hashenc.New(sha256.New(), base64Encoding), ebuf: make([]byte, encodedHashLen), } } diff --git a/vendor/codeberg.org/gruf/go-store/storage/disk.go b/vendor/codeberg.org/gruf/go-store/storage/disk.go index 9b5430437..2ee00ddee 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/disk.go +++ b/vendor/codeberg.org/gruf/go-store/storage/disk.go @@ -71,7 +71,7 @@ type DiskStorage struct { path string // path is the root path of this store bufp pools.BufferPool // bufp is the buffer pool for this DiskStorage config DiskConfig // cfg is the supplied configuration for this store - lock *LockableFile // lock is the opened lockfile for this storage instance + lock *Lock // lock is the opened lockfile for this storage instance } // OpenFile opens a DiskStorage instance for given folder path and configuration @@ -118,11 +118,9 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { } // Open and acquire storage lock for path - lock, err := OpenLock(pb.Join(path, LockFile)) + lock, err := OpenLock(pb.Join(path, lockFile)) if err != nil { return nil, err - } else if err := lock.Lock(); err != nil { - return nil, err } // Return new DiskStorage @@ -136,6 +134,11 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { // Clean implements Storage.Clean() func (st *DiskStorage) Clean() error { + st.lock.Add() + defer st.lock.Done() + if st.lock.Closed() { + return ErrClosed + } return util.CleanDirs(st.path) } @@ -160,9 +163,18 @@ func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error) { return nil, err } + // Track open + st.lock.Add() + + // Check if open + if st.lock.Closed() { + return nil, ErrClosed + } + // Attempt to open file (replace ENOENT with our own) file, err := open(kpath, defaultFileROFlags) if err != nil { + st.lock.Done() return nil, errSwapNotFound(err) } @@ -170,12 +182,14 @@ func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error) { cFile, err := st.config.Compression.Reader(file) if err != nil { file.Close() // close this here, ignore error + st.lock.Done() return nil, err } // Wrap compressor to ensure file close return util.ReadCloserWithCallback(cFile, func() { file.Close() + st.lock.Done() }), nil } @@ -192,6 +206,15 @@ func (st *DiskStorage) WriteStream(key string, r io.Reader) error { return err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Ensure dirs leading up to file exist err = os.MkdirAll(path.Dir(kpath), defaultDirPerms) if err != nil { @@ -242,6 +265,15 @@ func (st *DiskStorage) Stat(key string) (bool, error) { return false, err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return false, ErrClosed + } + // Check for file on disk return stat(kpath) } @@ -254,18 +286,35 @@ func (st *DiskStorage) Remove(key string) error { return err } + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Attempt to remove file return os.Remove(kpath) } // Close implements Storage.Close() func (st *DiskStorage) Close() error { - defer st.lock.Close() - return st.lock.Unlock() + return st.lock.Close() } // WalkKeys implements Storage.WalkKeys() func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error { + // Track open + st.lock.Add() + defer st.lock.Done() + + // Check if open + if st.lock.Closed() { + return ErrClosed + } + // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) @@ -286,13 +335,13 @@ func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error { // filepath checks and returns a formatted filepath for given key func (st *DiskStorage) filepath(key string) (string, error) { + // Calculate transformed key path + key = st.config.Transform.KeyToPath(key) + // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) - // Calculate transformed key path - key = st.config.Transform.KeyToPath(key) - // Generated joined root path pb.AppendString(st.path) pb.AppendString(key) diff --git a/vendor/codeberg.org/gruf/go-store/storage/errors.go b/vendor/codeberg.org/gruf/go-store/storage/errors.go index 016593596..ad2b742e6 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/errors.go +++ b/vendor/codeberg.org/gruf/go-store/storage/errors.go @@ -19,6 +19,9 @@ func (e errorString) Extend(s string, a ...interface{}) errorString { } var ( + // ErrClosed is returned on operations on a closed storage + ErrClosed = errorString("store/storage: closed") + // ErrNotFound is the error returned when a key cannot be found in storage ErrNotFound = errorString("store/storage: key not found") @@ -39,6 +42,9 @@ var ( // errCorruptNodes is returned when nodes with missing blocks are found during a BlockStorage clean errCorruptNodes = errorString("store/storage: corrupted nodes") + + // ErrAlreadyLocked is returned on fail opening a storage lockfile + ErrAlreadyLocked = errorString("store/storage: storage lock already open") ) // errSwapNoop performs no error swaps @@ -61,3 +67,11 @@ func errSwapExist(err error) error { } return err } + +// errSwapUnavailable swaps syscall.EAGAIN for ErrAlreadyLocked +func errSwapUnavailable(err error) error { + if err == syscall.EAGAIN { + return ErrAlreadyLocked + } + return err +} diff --git a/vendor/codeberg.org/gruf/go-store/storage/fs.go b/vendor/codeberg.org/gruf/go-store/storage/fs.go index ff4c857c3..b1c3560d2 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/fs.go +++ b/vendor/codeberg.org/gruf/go-store/storage/fs.go @@ -8,11 +8,14 @@ import ( ) const ( - defaultDirPerms = 0755 - defaultFilePerms = 0644 + // default file permission bits + defaultDirPerms = 0755 + defaultFilePerms = 0644 + + // default file open flags defaultFileROFlags = syscall.O_RDONLY defaultFileRWFlags = syscall.O_CREAT | syscall.O_RDWR - defaultFileLockFlags = syscall.O_RDONLY | syscall.O_EXCL | syscall.O_CREAT + defaultFileLockFlags = syscall.O_RDONLY | syscall.O_CREAT ) // NOTE: diff --git a/vendor/codeberg.org/gruf/go-store/storage/lock.go b/vendor/codeberg.org/gruf/go-store/storage/lock.go index a757830cc..fae4351bf 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/lock.go +++ b/vendor/codeberg.org/gruf/go-store/storage/lock.go @@ -1,38 +1,81 @@ package storage import ( - "os" + "sync" + "sync/atomic" "syscall" "codeberg.org/gruf/go-store/util" ) -// LockFile is our standard lockfile name. -const LockFile = "store.lock" +// lockFile is our standard lockfile name. +var lockFile = "store.lock" -type LockableFile struct { - *os.File +// IsLockKey returns whether storage key is our lockfile. +func IsLockKey(key string) bool { + return key == lockFile +} + +// Lock represents a filesystem lock to ensure only one storage instance open per path. +type Lock struct { + fd int + wg sync.WaitGroup + st uint32 } // OpenLock opens a lockfile at path. -func OpenLock(path string) (*LockableFile, error) { - file, err := open(path, defaultFileLockFlags) +func OpenLock(path string) (*Lock, error) { + var fd int + + // Open the file descriptor at path + err := util.RetryOnEINTR(func() (err error) { + fd, err = syscall.Open(path, defaultFileLockFlags, defaultFilePerms) + return + }) if err != nil { return nil, err } - return &LockableFile{file}, nil + + // Get a flock on the file descriptor + err = util.RetryOnEINTR(func() error { + return syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB) + }) + if err != nil { + return nil, errSwapUnavailable(err) + } + + return &Lock{fd: fd}, nil } -func (f *LockableFile) Lock() error { - return f.flock(syscall.LOCK_EX | syscall.LOCK_NB) +// Add will add '1' to the underlying sync.WaitGroup. +func (f *Lock) Add() { + f.wg.Add(1) } -func (f *LockableFile) Unlock() error { - return f.flock(syscall.LOCK_UN | syscall.LOCK_NB) +// Done will decrememnt '1' from the underlying sync.WaitGroup. +func (f *Lock) Done() { + f.wg.Done() } -func (f *LockableFile) flock(how int) error { - return util.RetryOnEINTR(func() error { - return syscall.Flock(int(f.Fd()), how) - }) +// Close will attempt to close the lockfile and file descriptor. +func (f *Lock) Close() error { + var err error + if atomic.CompareAndSwapUint32(&f.st, 0, 1) { + // Wait until done + f.wg.Wait() + + // Ensure gets closed + defer syscall.Close(f.fd) + + // Call funlock on the file descriptor + err = util.RetryOnEINTR(func() error { + return syscall.Flock(f.fd, syscall.LOCK_UN|syscall.LOCK_NB) + }) + } + return err +} + +// Closed will return whether this lockfile has been closed (and unlocked). +func (f *Lock) Closed() bool { + return (atomic.LoadUint32(&f.st) == 1) } diff --git a/vendor/codeberg.org/gruf/go-store/storage/memory.go b/vendor/codeberg.org/gruf/go-store/storage/memory.go index 7daa4a483..2dab562d6 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/memory.go +++ b/vendor/codeberg.org/gruf/go-store/storage/memory.go @@ -14,6 +14,7 @@ type MemoryStorage struct { ow bool // overwrites fs map[string][]byte mu sync.Mutex + st uint32 } // OpenMemory opens a new MemoryStorage instance with internal map of 'size'. @@ -27,13 +28,26 @@ func OpenMemory(size int, overwrites bool) *MemoryStorage { // Clean implements Storage.Clean(). func (st *MemoryStorage) Clean() error { + st.mu.Lock() + defer st.mu.Unlock() + if st.st == 1 { + return ErrClosed + } return nil } // ReadBytes implements Storage.ReadBytes(). func (st *MemoryStorage) ReadBytes(key string) ([]byte, error) { - // Safely check store + // Lock storage st.mu.Lock() + + // Check store open + if st.st == 1 { + st.mu.Unlock() + return nil, ErrClosed + } + + // Check for key b, ok := st.fs[key] st.mu.Unlock() @@ -48,8 +62,16 @@ func (st *MemoryStorage) ReadBytes(key string) ([]byte, error) { // ReadStream implements Storage.ReadStream(). func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error) { - // Safely check store + // Lock storage st.mu.Lock() + + // Check store open + if st.st == 1 { + st.mu.Unlock() + return nil, ErrClosed + } + + // Check for key b, ok := st.fs[key] st.mu.Unlock() @@ -66,19 +88,24 @@ func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error) { // WriteBytes implements Storage.WriteBytes(). func (st *MemoryStorage) WriteBytes(key string, b []byte) error { - // Safely check store + // Lock storage st.mu.Lock() + defer st.mu.Unlock() + + // Check store open + if st.st == 1 { + return ErrClosed + } + _, ok := st.fs[key] // Check for already exist if ok && !st.ow { - st.mu.Unlock() return ErrAlreadyExists } // Write + unlock st.fs[key] = bytes.Copy(b) - st.mu.Unlock() return nil } @@ -96,43 +123,66 @@ func (st *MemoryStorage) WriteStream(key string, r io.Reader) error { // Stat implements Storage.Stat(). func (st *MemoryStorage) Stat(key string) (bool, error) { + // Lock storage st.mu.Lock() + defer st.mu.Unlock() + + // Check store open + if st.st == 1 { + return false, ErrClosed + } + + // Check for key _, ok := st.fs[key] - st.mu.Unlock() return ok, nil } // Remove implements Storage.Remove(). func (st *MemoryStorage) Remove(key string) error { - // Safely check store + // Lock storage st.mu.Lock() - _, ok := st.fs[key] + defer st.mu.Unlock() - // Check in store + // Check store open + if st.st == 1 { + return ErrClosed + } + + // Check for key + _, ok := st.fs[key] if !ok { - st.mu.Unlock() return ErrNotFound } - // Delete + unlock + // Remove from store delete(st.fs, key) - st.mu.Unlock() + return nil } // Close implements Storage.Close(). func (st *MemoryStorage) Close() error { + st.mu.Lock() + st.st = 1 + st.mu.Unlock() return nil } // WalkKeys implements Storage.WalkKeys(). func (st *MemoryStorage) WalkKeys(opts WalkKeysOptions) error { - // Safely walk storage keys + // Lock storage st.mu.Lock() + defer st.mu.Unlock() + + // Check store open + if st.st == 1 { + return ErrClosed + } + + // Walk store keys for key := range st.fs { opts.WalkFn(entry(key)) } - st.mu.Unlock() return nil } diff --git a/vendor/github.com/zeebo/blake3/.gitignore b/vendor/github.com/zeebo/blake3/.gitignore deleted file mode 100644 index c6bfdf2c3..000000000 --- a/vendor/github.com/zeebo/blake3/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.pprof -*.test -*.txt -*.out - -/upstream diff --git a/vendor/github.com/zeebo/blake3/LICENSE b/vendor/github.com/zeebo/blake3/LICENSE deleted file mode 100644 index 3a63575d3..000000000 --- a/vendor/github.com/zeebo/blake3/LICENSE +++ /dev/null @@ -1,125 +0,0 @@ -This work is released into the public domain with CC0 1.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. diff --git a/vendor/github.com/zeebo/blake3/Makefile b/vendor/github.com/zeebo/blake3/Makefile deleted file mode 100644 index f98f0f093..000000000 --- a/vendor/github.com/zeebo/blake3/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -asm: internal/alg/hash/hash_avx2/impl_amd64.s internal/alg/compress/compress_sse41/impl_amd64.s - -internal/alg/hash/hash_avx2/impl_amd64.s: avo/avx2/*.go - ( cd avo; go run ./avx2 ) > internal/alg/hash/hash_avx2/impl_amd64.s - -internal/alg/compress/compress_sse41/impl_amd64.s: avo/sse41/*.go - ( cd avo; go run ./sse41 ) > internal/alg/compress/compress_sse41/impl_amd64.s - -.PHONY: test -test: - go test -race -bench=. -benchtime=1x diff --git a/vendor/github.com/zeebo/blake3/README.md b/vendor/github.com/zeebo/blake3/README.md deleted file mode 100644 index 0a0f2e186..000000000 --- a/vendor/github.com/zeebo/blake3/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# BLAKE3 - -

- go.dev - Go Report Card - SourceGraph -

- -Pure Go implementation of [BLAKE3](https://blake3.io) with AVX2 and SSE4.1 acceleration. - -Special thanks to the excellent [avo](https://github.com/mmcloughlin/avo) making writing vectorized version much easier. - -# Benchmarks - -## Caveats - -This library makes some different design decisions than the upstream Rust crate around internal buffering. Specifically, because it does not target the embedded system space, nor does it support multithreading, it elects to do its own internal buffering. This means that a user does not have to worry about providing large enough buffers to get the best possible performance, but it does worse on smaller input sizes. So some notes: - -- The Rust benchmarks below are all single-threaded to match this Go implementation. -- I make no attempt to get precise measurements (cpu throttling, noisy environment, etc.) so please benchmark on your own systems. -- These benchmarks are run on an i7-6700K which does not support AVX-512, so Rust is limited to use AVX2 at sizes above 8 kib. -- I tried my best to make them benchmark the same thing, but who knows? :smile: - -## Charts - -In this case, both libraries are able to avoid a lot of data copying and will use vectorized instructions to hash as fast as possible, and perform similarly. - -![Large Full Buffer](/assets/large-full-buffer.svg) - -For incremental writes, you must provide the Rust version large enough buffers so that it can use vectorized instructions. This Go library performs consistently regardless of the size being sent into the update function. - -![Incremental](/assets/incremental.svg) - -The downside of internal buffering is most apparent with small sizes as most time is spent initializing the hasher state. In terms of hashing rate, the difference is 3-4x, but in an absolute sense it's ~100ns (see tables below). If you wish to hash a large number of very small strings and you care about those nanoseconds, be sure to use the Reset method to avoid re-initializing the state. - -![Small Full Buffer](/assets/small-full-buffer.svg) - -## Timing Tables - -### Small - -| Size | Full Buffer | Reset | | Full Buffer Rate | Reset Rate | -|--------|-------------|------------|-|------------------|--------------| -| 64 b | `205ns` | `86.5ns` | | `312MB/s` | `740MB/s` | -| 256 b | `364ns` | `250ns` | | `703MB/s` | `1.03GB/s` | -| 512 b | `575ns` | `468ns` | | `892MB/s` | `1.10GB/s` | -| 768 b | `795ns` | `682ns` | | `967MB/s` | `1.13GB/s` | - -### Large - -| Size | Incremental | Full Buffer | Reset | | Incremental Rate | Full Buffer Rate | Reset Rate | -|----------|-------------|-------------|------------|-|------------------|------------------|--------------| -| 1 kib | `1.02µs` | `1.01µs` | `891ns` | | `1.00GB/s` | `1.01GB/s` | `1.15GB/s` | -| 2 kib | `2.11µs` | `2.07µs` | `1.95µs` | | `968MB/s` | `990MB/s` | `1.05GB/s` | -| 4 kib | `2.28µs` | `2.15µs` | `2.05µs` | | `1.80GB/s` | `1.90GB/s` | `2.00GB/s` | -| 8 kib | `2.64µs` | `2.52µs` | `2.44µs` | | `3.11GB/s` | `3.25GB/s` | `3.36GB/s` | -| 16 kib | `4.93µs` | `4.54µs` | `4.48µs` | | `3.33GB/s` | `3.61GB/s` | `3.66GB/s` | -| 32 kib | `9.41µs` | `8.62µs` | `8.54µs` | | `3.48GB/s` | `3.80GB/s` | `3.84GB/s` | -| 64 kib | `18.2µs` | `16.7µs` | `16.6µs` | | `3.59GB/s` | `3.91GB/s` | `3.94GB/s` | -| 128 kib | `36.3µs` | `32.9µs` | `33.1µs` | | `3.61GB/s` | `3.99GB/s` | `3.96GB/s` | -| 256 kib | `72.5µs` | `65.7µs` | `66.0µs` | | `3.62GB/s` | `3.99GB/s` | `3.97GB/s` | -| 512 kib | `145µs` | `131µs` | `132µs` | | `3.60GB/s` | `4.00GB/s` | `3.97GB/s` | -| 1024 kib | `290µs` | `262µs` | `262µs` | | `3.62GB/s` | `4.00GB/s` | `4.00GB/s` | - -### No ASM - -| Size | Incremental | Full Buffer | Reset | | Incremental Rate | Full Buffer Rate | Reset Rate | -|----------|-------------|-------------|------------|-|------------------|------------------|-------------| -| 64 b | `253ns` | `254ns` | `134ns` | | `253MB/s` | `252MB/s` | `478MB/s` | -| 256 b | `553ns` | `557ns` | `441ns` | | `463MB/s` | `459MB/s` | `580MB/s` | -| 512 b | `948ns` | `953ns` | `841ns` | | `540MB/s` | `538MB/s` | `609MB/s` | -| 768 b | `1.38µs` | `1.40µs` | `1.35µs` | | `558MB/s` | `547MB/s` | `570MB/s` | -| 1 kib | `1.77µs` | `1.77µs` | `1.70µs` | | `577MB/s` | `580MB/s` | `602MB/s` | -| | | | | | | | | -| 1024 kib | `880µs` | `883µs` | `878µs` | | `596MB/s` | `595MB/s` | `598MB/s` | - -The speed caps out at around 1 kib, so most rows have been elided from the presentation. diff --git a/vendor/github.com/zeebo/blake3/api.go b/vendor/github.com/zeebo/blake3/api.go deleted file mode 100644 index 5de263f08..000000000 --- a/vendor/github.com/zeebo/blake3/api.go +++ /dev/null @@ -1,166 +0,0 @@ -// Package blake3 provides an SSE4.1/AVX2 accelerated BLAKE3 implementation. -package blake3 - -import ( - "errors" - - "github.com/zeebo/blake3/internal/consts" - "github.com/zeebo/blake3/internal/utils" -) - -// Hasher is a hash.Hash for BLAKE3. -type Hasher struct { - size int - h hasher -} - -// New returns a new Hasher that has a digest size of 32 bytes. -// -// If you need more or less output bytes than that, use Digest method. -func New() *Hasher { - return &Hasher{ - size: 32, - h: hasher{ - key: consts.IV, - }, - } -} - -// NewKeyed returns a new Hasher that uses the 32 byte input key and has -// a digest size of 32 bytes. -// -// If you need more or less output bytes than that, use the Digest method. -func NewKeyed(key []byte) (*Hasher, error) { - if len(key) != 32 { - return nil, errors.New("invalid key size") - } - - h := &Hasher{ - size: 32, - h: hasher{ - flags: consts.Flag_Keyed, - }, - } - utils.KeyFromBytes(key, &h.h.key) - - return h, nil -} - -// DeriveKey derives a key based on reusable key material of any -// length, in the given context. The key will be stored in out, using -// all of its current length. -// -// Context strings must be hardcoded constants, and the recommended -// format is "[application] [commit timestamp] [purpose]", e.g., -// "example.com 2019-12-25 16:18:03 session tokens v1". -func DeriveKey(context string, material []byte, out []byte) { - h := NewDeriveKey(context) - _, _ = h.Write(material) - _, _ = h.Digest().Read(out) -} - -// NewDeriveKey returns a Hasher that is initialized with the context -// string. See DeriveKey for details. It has a digest size of 32 bytes. -// -// If you need more or less output bytes than that, use the Digest method. -func NewDeriveKey(context string) *Hasher { - // hash the context string and use that instead of IV - h := &Hasher{ - size: 32, - h: hasher{ - key: consts.IV, - flags: consts.Flag_DeriveKeyContext, - }, - } - - var buf [32]byte - _, _ = h.WriteString(context) - _, _ = h.Digest().Read(buf[:]) - - h.Reset() - utils.KeyFromBytes(buf[:], &h.h.key) - h.h.flags = consts.Flag_DeriveKeyMaterial - - return h -} - -// Write implements part of the hash.Hash interface. It never returns an error. -func (h *Hasher) Write(p []byte) (int, error) { - h.h.update(p) - return len(p), nil -} - -// WriteString is like Write but specialized to strings to avoid allocations. -func (h *Hasher) WriteString(p string) (int, error) { - h.h.updateString(p) - return len(p), nil -} - -// Reset implements part of the hash.Hash interface. It causes the Hasher to -// act as if it was newly created. -func (h *Hasher) Reset() { - h.h.reset() -} - -// Clone returns a new Hasher with the same internal state. -// -// Modifying the resulting Hasher will not modify the original Hasher, and vice versa. -func (h *Hasher) Clone() *Hasher { - return &Hasher{size: h.size, h: h.h} -} - -// Size implements part of the hash.Hash interface. It returns the number of -// bytes the hash will output in Sum. -func (h *Hasher) Size() int { - return h.size -} - -// BlockSize implements part of the hash.Hash interface. It returns the most -// natural size to write to the Hasher. -func (h *Hasher) BlockSize() int { - // TODO: is there a downside to picking this large size? - return 8192 -} - -// Sum implements part of the hash.Hash interface. It appends the digest of -// the Hasher to the provided buffer and returns it. -func (h *Hasher) Sum(b []byte) []byte { - if top := len(b) + h.size; top <= cap(b) && top >= len(b) { - h.h.finalize(b[len(b):top]) - return b[:top] - } - - tmp := make([]byte, h.size) - h.h.finalize(tmp) - return append(b, tmp...) -} - -// Digest takes a snapshot of the hash state and returns an object that can -// be used to read and seek through 2^64 bytes of digest output. -func (h *Hasher) Digest() *Digest { - var d Digest - h.h.finalizeDigest(&d) - return &d -} - -// Sum256 returns the first 256 bits of the unkeyed digest of the data. -func Sum256(data []byte) (sum [32]byte) { - out := Sum512(data) - copy(sum[:], out[:32]) - return sum -} - -// Sum512 returns the first 512 bits of the unkeyed digest of the data. -func Sum512(data []byte) (sum [64]byte) { - if len(data) <= consts.ChunkLen { - var d Digest - compressAll(&d, data, 0, consts.IV) - _, _ = d.Read(sum[:]) - return sum - } else { - h := hasher{key: consts.IV} - h.update(data) - h.finalize(sum[:]) - return sum - } -} diff --git a/vendor/github.com/zeebo/blake3/blake3.go b/vendor/github.com/zeebo/blake3/blake3.go deleted file mode 100644 index 98dedcabe..000000000 --- a/vendor/github.com/zeebo/blake3/blake3.go +++ /dev/null @@ -1,285 +0,0 @@ -package blake3 - -import ( - "math/bits" - "unsafe" - - "github.com/zeebo/blake3/internal/alg" - "github.com/zeebo/blake3/internal/consts" - "github.com/zeebo/blake3/internal/utils" -) - -// -// hasher contains state for a blake3 hash -// - -type hasher struct { - len uint64 - chunks uint64 - flags uint32 - key [8]uint32 - stack cvstack - buf [8192]byte -} - -func (a *hasher) reset() { - a.len = 0 - a.chunks = 0 - a.stack.occ = 0 - a.stack.lvls = [8]uint8{} - a.stack.bufn = 0 -} - -func (a *hasher) update(buf []byte) { - // relies on the first two words of a string being the same as a slice - a.updateString(*(*string)(unsafe.Pointer(&buf))) -} - -func (a *hasher) updateString(buf string) { - var input *[8192]byte - - for len(buf) > 0 { - if a.len == 0 && len(buf) > 8192 { - // relies on the data pointer being the first word in the string header - input = (*[8192]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&buf))) - buf = buf[8192:] - } else if a.len < 8192 { - n := copy(a.buf[a.len:], buf) - a.len += uint64(n) - buf = buf[n:] - continue - } else { - input = &a.buf - } - - a.consume(input) - a.len = 0 - a.chunks += 8 - } -} - -func (a *hasher) consume(input *[8192]byte) { - var out chainVector - var chain [8]uint32 - alg.HashF(input, 8192, a.chunks, a.flags, &a.key, &out, &chain) - a.stack.pushN(0, &out, 8, a.flags, &a.key) -} - -func (a *hasher) finalize(p []byte) { - var d Digest - a.finalizeDigest(&d) - _, _ = d.Read(p) -} - -func (a *hasher) finalizeDigest(d *Digest) { - if a.chunks == 0 && a.len <= consts.ChunkLen { - compressAll(d, a.buf[:a.len], a.flags, a.key) - return - } - - d.chain = a.key - d.flags = a.flags | consts.Flag_ChunkEnd - - if a.len > 64 { - var buf chainVector - alg.HashF(&a.buf, a.len, a.chunks, a.flags, &a.key, &buf, &d.chain) - - if a.len > consts.ChunkLen { - complete := (a.len - 1) / consts.ChunkLen - a.stack.pushN(0, &buf, int(complete), a.flags, &a.key) - a.chunks += complete - a.len = uint64(copy(a.buf[:], a.buf[complete*consts.ChunkLen:a.len])) - } - } - - if a.len <= 64 { - d.flags |= consts.Flag_ChunkStart - } - - d.counter = a.chunks - d.blen = uint32(a.len) % 64 - - base := a.len / 64 * 64 - if a.len > 0 && d.blen == 0 { - d.blen = 64 - base -= 64 - } - - if consts.IsLittleEndian { - copy((*[64]byte)(unsafe.Pointer(&d.block[0]))[:], a.buf[base:a.len]) - } else { - var tmp [64]byte - copy(tmp[:], a.buf[base:a.len]) - utils.BytesToWords(&tmp, &d.block) - } - - for a.stack.bufn > 0 { - a.stack.flush(a.flags, &a.key) - } - - var tmp [16]uint32 - for occ := a.stack.occ; occ != 0; occ &= occ - 1 { - col := uint(bits.TrailingZeros64(occ)) % 64 - - alg.Compress(&d.chain, &d.block, d.counter, d.blen, d.flags, &tmp) - - *(*[8]uint32)(unsafe.Pointer(&d.block[0])) = a.stack.stack[col] - *(*[8]uint32)(unsafe.Pointer(&d.block[8])) = *(*[8]uint32)(unsafe.Pointer(&tmp[0])) - - if occ == a.stack.occ { - d.chain = a.key - d.counter = 0 - d.blen = consts.BlockLen - d.flags = a.flags | consts.Flag_Parent - } - } - - d.flags |= consts.Flag_Root -} - -// -// chain value stack -// - -type chainVector = [64]uint32 - -type cvstack struct { - occ uint64 // which levels in stack are occupied - lvls [8]uint8 // what level the buf input was in - bufn int // how many pairs are loaded into buf - buf [2]chainVector - stack [64][8]uint32 -} - -func (a *cvstack) pushN(l uint8, cv *chainVector, n int, flags uint32, key *[8]uint32) { - for i := 0; i < n; i++ { - a.pushL(l, cv, i) - for a.bufn == 8 { - a.flush(flags, key) - } - } -} - -func (a *cvstack) pushL(l uint8, cv *chainVector, n int) { - bit := uint64(1) << (l & 63) - if a.occ&bit == 0 { - readChain(cv, n, &a.stack[l&63]) - a.occ ^= bit - return - } - - a.lvls[a.bufn&7] = l - writeChain(&a.stack[l&63], &a.buf[0], a.bufn) - copyChain(cv, n, &a.buf[1], a.bufn) - a.bufn++ - a.occ ^= bit -} - -func (a *cvstack) flush(flags uint32, key *[8]uint32) { - var out chainVector - alg.HashP(&a.buf[0], &a.buf[1], flags|consts.Flag_Parent, key, &out, a.bufn) - - bufn, lvls := a.bufn, a.lvls - a.bufn, a.lvls = 0, [8]uint8{} - - for i := 0; i < bufn; i++ { - a.pushL(lvls[i]+1, &out, i) - } -} - -// -// helpers to deal with reading/writing transposed values -// - -func copyChain(in *chainVector, icol int, out *chainVector, ocol int) { - type u = uintptr - type p = unsafe.Pointer - type a = *uint32 - - i := p(u(p(in)) + u(icol*4)) - o := p(u(p(out)) + u(ocol*4)) - - *a(p(u(o) + 0*32)) = *a(p(u(i) + 0*32)) - *a(p(u(o) + 1*32)) = *a(p(u(i) + 1*32)) - *a(p(u(o) + 2*32)) = *a(p(u(i) + 2*32)) - *a(p(u(o) + 3*32)) = *a(p(u(i) + 3*32)) - *a(p(u(o) + 4*32)) = *a(p(u(i) + 4*32)) - *a(p(u(o) + 5*32)) = *a(p(u(i) + 5*32)) - *a(p(u(o) + 6*32)) = *a(p(u(i) + 6*32)) - *a(p(u(o) + 7*32)) = *a(p(u(i) + 7*32)) -} - -func readChain(in *chainVector, col int, out *[8]uint32) { - type u = uintptr - type p = unsafe.Pointer - type a = *uint32 - - i := p(u(p(in)) + u(col*4)) - - out[0] = *a(p(u(i) + 0*32)) - out[1] = *a(p(u(i) + 1*32)) - out[2] = *a(p(u(i) + 2*32)) - out[3] = *a(p(u(i) + 3*32)) - out[4] = *a(p(u(i) + 4*32)) - out[5] = *a(p(u(i) + 5*32)) - out[6] = *a(p(u(i) + 6*32)) - out[7] = *a(p(u(i) + 7*32)) -} - -func writeChain(in *[8]uint32, out *chainVector, col int) { - type u = uintptr - type p = unsafe.Pointer - type a = *uint32 - - o := p(u(p(out)) + u(col*4)) - - *a(p(u(o) + 0*32)) = in[0] - *a(p(u(o) + 1*32)) = in[1] - *a(p(u(o) + 2*32)) = in[2] - *a(p(u(o) + 3*32)) = in[3] - *a(p(u(o) + 4*32)) = in[4] - *a(p(u(o) + 5*32)) = in[5] - *a(p(u(o) + 6*32)) = in[6] - *a(p(u(o) + 7*32)) = in[7] -} - -// -// compress <= chunkLen bytes in one shot -// - -func compressAll(d *Digest, in []byte, flags uint32, key [8]uint32) { - var compressed [16]uint32 - - d.chain = key - d.flags = flags | consts.Flag_ChunkStart - - for len(in) > 64 { - buf := (*[64]byte)(unsafe.Pointer(&in[0])) - - var block *[16]uint32 - if consts.IsLittleEndian { - block = (*[16]uint32)(unsafe.Pointer(buf)) - } else { - block = &d.block - utils.BytesToWords(buf, block) - } - - alg.Compress(&d.chain, block, 0, consts.BlockLen, d.flags, &compressed) - - d.chain = *(*[8]uint32)(unsafe.Pointer(&compressed[0])) - d.flags &^= consts.Flag_ChunkStart - - in = in[64:] - } - - if consts.IsLittleEndian { - copy((*[64]byte)(unsafe.Pointer(&d.block[0]))[:], in) - } else { - var tmp [64]byte - copy(tmp[:], in) - utils.BytesToWords(&tmp, &d.block) - } - - d.blen = uint32(len(in)) - d.flags |= consts.Flag_ChunkEnd | consts.Flag_Root -} diff --git a/vendor/github.com/zeebo/blake3/digest.go b/vendor/github.com/zeebo/blake3/digest.go deleted file mode 100644 index 58365d5ab..000000000 --- a/vendor/github.com/zeebo/blake3/digest.go +++ /dev/null @@ -1,100 +0,0 @@ -package blake3 - -import ( - "fmt" - "io" - "unsafe" - - "github.com/zeebo/blake3/internal/alg" - "github.com/zeebo/blake3/internal/consts" - "github.com/zeebo/blake3/internal/utils" -) - -// Digest captures the state of a Hasher allowing reading and seeking through -// the output stream. -type Digest struct { - counter uint64 - chain [8]uint32 - block [16]uint32 - blen uint32 - flags uint32 - buf [16]uint32 - bufn int -} - -// Read reads data frm the hasher into out. It always fills the entire buffer and -// never errors. The stream will wrap around when reading past 2^64 bytes. -func (d *Digest) Read(p []byte) (n int, err error) { - n = len(p) - - if d.bufn > 0 { - n := d.slowCopy(p) - p = p[n:] - d.bufn -= n - } - - for len(p) >= 64 { - d.fillBuf() - - if consts.IsLittleEndian { - *(*[64]byte)(unsafe.Pointer(&p[0])) = *(*[64]byte)(unsafe.Pointer(&d.buf[0])) - } else { - utils.WordsToBytes(&d.buf, p) - } - - p = p[64:] - d.bufn = 0 - } - - if len(p) == 0 { - return n, nil - } - - d.fillBuf() - d.bufn -= d.slowCopy(p) - - return n, nil -} - -// Seek sets the position to the provided location. Only SeekStart and -// SeekCurrent are allowed. -func (d *Digest) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - case io.SeekEnd: - return 0, fmt.Errorf("seek from end not supported") - case io.SeekCurrent: - offset += int64(consts.BlockLen*d.counter) - int64(d.bufn) - default: - return 0, fmt.Errorf("invalid whence: %d", whence) - } - if offset < 0 { - return 0, fmt.Errorf("seek before start") - } - d.setPosition(uint64(offset)) - return offset, nil -} - -func (d *Digest) setPosition(pos uint64) { - d.counter = pos / consts.BlockLen - d.fillBuf() - d.bufn -= int(pos % consts.BlockLen) -} - -func (d *Digest) slowCopy(p []byte) (n int) { - off := uint(consts.BlockLen-d.bufn) % consts.BlockLen - if consts.IsLittleEndian { - n = copy(p, (*[consts.BlockLen]byte)(unsafe.Pointer(&d.buf[0]))[off:]) - } else { - var tmp [consts.BlockLen]byte - utils.WordsToBytes(&d.buf, tmp[:]) - n = copy(p, tmp[off:]) - } - return n -} - -func (d *Digest) fillBuf() { - alg.Compress(&d.chain, &d.block, d.counter, d.blen, d.flags, &d.buf) - d.counter++ - d.bufn = consts.BlockLen -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/alg.go b/vendor/github.com/zeebo/blake3/internal/alg/alg.go deleted file mode 100644 index 239fdec5b..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/alg.go +++ /dev/null @@ -1,18 +0,0 @@ -package alg - -import ( - "github.com/zeebo/blake3/internal/alg/compress" - "github.com/zeebo/blake3/internal/alg/hash" -) - -func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { - hash.HashF(input, length, counter, flags, key, out, chain) -} - -func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { - hash.HashP(left, right, flags, key, out, n) -} - -func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { - compress.Compress(chain, block, counter, blen, flags, out) -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go deleted file mode 100644 index 0b2685408..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress.go +++ /dev/null @@ -1,15 +0,0 @@ -package compress - -import ( - "github.com/zeebo/blake3/internal/alg/compress/compress_pure" - "github.com/zeebo/blake3/internal/alg/compress/compress_sse41" - "github.com/zeebo/blake3/internal/consts" -) - -func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { - if consts.HasSSE41 { - compress_sse41.Compress(chain, block, counter, blen, flags, out) - } else { - compress_pure.Compress(chain, block, counter, blen, flags, out) - } -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go deleted file mode 100644 index 66ea1fb75..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_pure/compress.go +++ /dev/null @@ -1,135 +0,0 @@ -package compress_pure - -import ( - "math/bits" - - "github.com/zeebo/blake3/internal/consts" -) - -func Compress( - chain *[8]uint32, - block *[16]uint32, - counter uint64, - blen uint32, - flags uint32, - out *[16]uint32, -) { - - *out = [16]uint32{ - chain[0], chain[1], chain[2], chain[3], - chain[4], chain[5], chain[6], chain[7], - consts.IV0, consts.IV1, consts.IV2, consts.IV3, - uint32(counter), uint32(counter >> 32), blen, flags, - } - - rcompress(out, block) -} - -func g(a, b, c, d, mx, my uint32) (uint32, uint32, uint32, uint32) { - a += b + mx - d = bits.RotateLeft32(d^a, -16) - c += d - b = bits.RotateLeft32(b^c, -12) - a += b + my - d = bits.RotateLeft32(d^a, -8) - c += d - b = bits.RotateLeft32(b^c, -7) - return a, b, c, d -} - -func rcompress(s *[16]uint32, m *[16]uint32) { - const ( - a = 10 - b = 11 - c = 12 - d = 13 - e = 14 - f = 15 - ) - - s0, s1, s2, s3 := s[0+0], s[0+1], s[0+2], s[0+3] - s4, s5, s6, s7 := s[0+4], s[0+5], s[0+6], s[0+7] - s8, s9, sa, sb := s[8+0], s[8+1], s[8+2], s[8+3] - sc, sd, se, sf := s[8+4], s[8+5], s[8+6], s[8+7] - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[0], m[1]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[2], m[3]) - s2, s6, sa, se = g(s2, s6, sa, se, m[4], m[5]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[6], m[7]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[8], m[9]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[a], m[b]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[c], m[d]) - s3, s4, s9, se = g(s3, s4, s9, se, m[e], m[f]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[2], m[6]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[3], m[a]) - s2, s6, sa, se = g(s2, s6, sa, se, m[7], m[0]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[4], m[d]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[1], m[b]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[c], m[5]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[9], m[e]) - s3, s4, s9, se = g(s3, s4, s9, se, m[f], m[8]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[3], m[4]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[a], m[c]) - s2, s6, sa, se = g(s2, s6, sa, se, m[d], m[2]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[7], m[e]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[6], m[5]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[9], m[0]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[b], m[f]) - s3, s4, s9, se = g(s3, s4, s9, se, m[8], m[1]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[a], m[7]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[c], m[9]) - s2, s6, sa, se = g(s2, s6, sa, se, m[e], m[3]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[d], m[f]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[4], m[0]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[b], m[2]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[5], m[8]) - s3, s4, s9, se = g(s3, s4, s9, se, m[1], m[6]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[c], m[d]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[9], m[b]) - s2, s6, sa, se = g(s2, s6, sa, se, m[f], m[a]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[e], m[8]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[7], m[2]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[5], m[3]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[0], m[1]) - s3, s4, s9, se = g(s3, s4, s9, se, m[6], m[4]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[9], m[e]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[b], m[5]) - s2, s6, sa, se = g(s2, s6, sa, se, m[8], m[c]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[f], m[1]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[d], m[3]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[0], m[a]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[2], m[6]) - s3, s4, s9, se = g(s3, s4, s9, se, m[4], m[7]) - - s0, s4, s8, sc = g(s0, s4, s8, sc, m[b], m[f]) - s1, s5, s9, sd = g(s1, s5, s9, sd, m[5], m[0]) - s2, s6, sa, se = g(s2, s6, sa, se, m[1], m[9]) - s3, s7, sb, sf = g(s3, s7, sb, sf, m[8], m[6]) - s0, s5, sa, sf = g(s0, s5, sa, sf, m[e], m[a]) - s1, s6, sb, sc = g(s1, s6, sb, sc, m[2], m[c]) - s2, s7, s8, sd = g(s2, s7, s8, sd, m[3], m[4]) - s3, s4, s9, se = g(s3, s4, s9, se, m[7], m[d]) - - s[8+0] = s8 ^ s[0] - s[8+1] = s9 ^ s[1] - s[8+2] = sa ^ s[2] - s[8+3] = sb ^ s[3] - s[8+4] = sc ^ s[4] - s[8+5] = sd ^ s[5] - s[8+6] = se ^ s[6] - s[8+7] = sf ^ s[7] - - s[0] = s0 ^ s8 - s[1] = s1 ^ s9 - s[2] = s2 ^ sa - s[3] = s3 ^ sb - s[4] = s4 ^ sc - s[5] = s5 ^ sd - s[6] = s6 ^ se - s[7] = s7 ^ sf -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s deleted file mode 100644 index 0fedf0b3a..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_amd64.s +++ /dev/null @@ -1,560 +0,0 @@ -// Code generated by command: go run compress.go. DO NOT EDIT. - -#include "textflag.h" - -DATA iv<>+0(SB)/4, $0x6a09e667 -DATA iv<>+4(SB)/4, $0xbb67ae85 -DATA iv<>+8(SB)/4, $0x3c6ef372 -DATA iv<>+12(SB)/4, $0xa54ff53a -DATA iv<>+16(SB)/4, $0x510e527f -DATA iv<>+20(SB)/4, $0x9b05688c -DATA iv<>+24(SB)/4, $0x1f83d9ab -DATA iv<>+28(SB)/4, $0x5be0cd19 -GLOBL iv<>(SB), RODATA|NOPTR, $32 - -DATA rot16_shuf<>+0(SB)/1, $0x02 -DATA rot16_shuf<>+1(SB)/1, $0x03 -DATA rot16_shuf<>+2(SB)/1, $0x00 -DATA rot16_shuf<>+3(SB)/1, $0x01 -DATA rot16_shuf<>+4(SB)/1, $0x06 -DATA rot16_shuf<>+5(SB)/1, $0x07 -DATA rot16_shuf<>+6(SB)/1, $0x04 -DATA rot16_shuf<>+7(SB)/1, $0x05 -DATA rot16_shuf<>+8(SB)/1, $0x0a -DATA rot16_shuf<>+9(SB)/1, $0x0b -DATA rot16_shuf<>+10(SB)/1, $0x08 -DATA rot16_shuf<>+11(SB)/1, $0x09 -DATA rot16_shuf<>+12(SB)/1, $0x0e -DATA rot16_shuf<>+13(SB)/1, $0x0f -DATA rot16_shuf<>+14(SB)/1, $0x0c -DATA rot16_shuf<>+15(SB)/1, $0x0d -DATA rot16_shuf<>+16(SB)/1, $0x12 -DATA rot16_shuf<>+17(SB)/1, $0x13 -DATA rot16_shuf<>+18(SB)/1, $0x10 -DATA rot16_shuf<>+19(SB)/1, $0x11 -DATA rot16_shuf<>+20(SB)/1, $0x16 -DATA rot16_shuf<>+21(SB)/1, $0x17 -DATA rot16_shuf<>+22(SB)/1, $0x14 -DATA rot16_shuf<>+23(SB)/1, $0x15 -DATA rot16_shuf<>+24(SB)/1, $0x1a -DATA rot16_shuf<>+25(SB)/1, $0x1b -DATA rot16_shuf<>+26(SB)/1, $0x18 -DATA rot16_shuf<>+27(SB)/1, $0x19 -DATA rot16_shuf<>+28(SB)/1, $0x1e -DATA rot16_shuf<>+29(SB)/1, $0x1f -DATA rot16_shuf<>+30(SB)/1, $0x1c -DATA rot16_shuf<>+31(SB)/1, $0x1d -GLOBL rot16_shuf<>(SB), RODATA|NOPTR, $32 - -DATA rot8_shuf<>+0(SB)/1, $0x01 -DATA rot8_shuf<>+1(SB)/1, $0x02 -DATA rot8_shuf<>+2(SB)/1, $0x03 -DATA rot8_shuf<>+3(SB)/1, $0x00 -DATA rot8_shuf<>+4(SB)/1, $0x05 -DATA rot8_shuf<>+5(SB)/1, $0x06 -DATA rot8_shuf<>+6(SB)/1, $0x07 -DATA rot8_shuf<>+7(SB)/1, $0x04 -DATA rot8_shuf<>+8(SB)/1, $0x09 -DATA rot8_shuf<>+9(SB)/1, $0x0a -DATA rot8_shuf<>+10(SB)/1, $0x0b -DATA rot8_shuf<>+11(SB)/1, $0x08 -DATA rot8_shuf<>+12(SB)/1, $0x0d -DATA rot8_shuf<>+13(SB)/1, $0x0e -DATA rot8_shuf<>+14(SB)/1, $0x0f -DATA rot8_shuf<>+15(SB)/1, $0x0c -DATA rot8_shuf<>+16(SB)/1, $0x11 -DATA rot8_shuf<>+17(SB)/1, $0x12 -DATA rot8_shuf<>+18(SB)/1, $0x13 -DATA rot8_shuf<>+19(SB)/1, $0x10 -DATA rot8_shuf<>+20(SB)/1, $0x15 -DATA rot8_shuf<>+21(SB)/1, $0x16 -DATA rot8_shuf<>+22(SB)/1, $0x17 -DATA rot8_shuf<>+23(SB)/1, $0x14 -DATA rot8_shuf<>+24(SB)/1, $0x19 -DATA rot8_shuf<>+25(SB)/1, $0x1a -DATA rot8_shuf<>+26(SB)/1, $0x1b -DATA rot8_shuf<>+27(SB)/1, $0x18 -DATA rot8_shuf<>+28(SB)/1, $0x1d -DATA rot8_shuf<>+29(SB)/1, $0x1e -DATA rot8_shuf<>+30(SB)/1, $0x1f -DATA rot8_shuf<>+31(SB)/1, $0x1c -GLOBL rot8_shuf<>(SB), RODATA|NOPTR, $32 - -// func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) -// Requires: SSE, SSE2, SSE4.1, SSSE3 -TEXT ·Compress(SB), NOSPLIT, $0-40 - MOVQ chain+0(FP), AX - MOVQ block+8(FP), CX - MOVQ counter+16(FP), DX - MOVL blen+24(FP), BX - MOVL flags+28(FP), BP - MOVQ out+32(FP), SI - MOVUPS (AX), X0 - MOVUPS 16(AX), X1 - MOVUPS iv<>+0(SB), X2 - PINSRD $0x00, DX, X3 - SHRQ $0x20, DX - PINSRD $0x01, DX, X3 - PINSRD $0x02, BX, X3 - PINSRD $0x03, BP, X3 - MOVUPS (CX), X4 - MOVUPS 16(CX), X5 - MOVUPS 32(CX), X6 - MOVUPS 48(CX), X7 - MOVUPS rot16_shuf<>+0(SB), X8 - MOVUPS rot8_shuf<>+0(SB), X9 - - // round 1 - MOVAPS X4, X10 - SHUFPS $0x88, X5, X10 - PADDD X10, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X11 - PSRLL $0x0c, X1 - PSLLL $0x14, X11 - POR X11, X1 - MOVAPS X4, X4 - SHUFPS $0xdd, X5, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X6, X5 - SHUFPS $0x88, X7, X5 - SHUFPS $0x93, X5, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X11 - PSRLL $0x0c, X1 - PSLLL $0x14, X11 - POR X11, X1 - MOVAPS X6, X6 - SHUFPS $0xdd, X7, X6 - SHUFPS $0x93, X6, X6 - PADDD X6, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x07, X1 - PSLLL $0x19, X7 - POR X7, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 2 - MOVAPS X10, X7 - SHUFPS $0xd6, X4, X7 - SHUFPS $0x39, X7, X7 - PADDD X7, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X11 - PSRLL $0x0c, X1 - PSLLL $0x14, X11 - POR X11, X1 - MOVAPS X5, X11 - SHUFPS $0xfa, X6, X11 - PSHUFD $0x0f, X10, X10 - PBLENDW $0x33, X10, X11 - PADDD X11, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X10 - PSRLL $0x07, X1 - PSLLL $0x19, X10 - POR X10, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X6, X12 - PUNPCKLLQ X4, X12 - PBLENDW $0xc0, X5, X12 - SHUFPS $0xb4, X12, X12 - PADDD X12, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X10 - PSRLL $0x0c, X1 - PSLLL $0x14, X10 - POR X10, X1 - MOVAPS X4, X10 - PUNPCKHLQ X6, X10 - MOVAPS X5, X4 - PUNPCKLLQ X10, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 3 - MOVAPS X7, X5 - SHUFPS $0xd6, X11, X5 - SHUFPS $0x39, X5, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X6 - PSRLL $0x0c, X1 - PSLLL $0x14, X6 - POR X6, X1 - MOVAPS X12, X6 - SHUFPS $0xfa, X4, X6 - PSHUFD $0x0f, X7, X7 - PBLENDW $0x33, X7, X6 - PADDD X6, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x07, X1 - PSLLL $0x19, X7 - POR X7, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X4, X10 - PUNPCKLLQ X11, X10 - PBLENDW $0xc0, X12, X10 - SHUFPS $0xb4, X10, X10 - PADDD X10, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x0c, X1 - PSLLL $0x14, X7 - POR X7, X1 - MOVAPS X11, X7 - PUNPCKHLQ X4, X7 - MOVAPS X12, X4 - PUNPCKLLQ X7, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x07, X1 - PSLLL $0x19, X7 - POR X7, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 4 - MOVAPS X5, X7 - SHUFPS $0xd6, X6, X7 - SHUFPS $0x39, X7, X7 - PADDD X7, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X11 - PSRLL $0x0c, X1 - PSLLL $0x14, X11 - POR X11, X1 - MOVAPS X10, X11 - SHUFPS $0xfa, X4, X11 - PSHUFD $0x0f, X5, X5 - PBLENDW $0x33, X5, X11 - PADDD X11, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X4, X12 - PUNPCKLLQ X6, X12 - PBLENDW $0xc0, X10, X12 - SHUFPS $0xb4, X12, X12 - PADDD X12, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x0c, X1 - PSLLL $0x14, X5 - POR X5, X1 - MOVAPS X6, X5 - PUNPCKHLQ X4, X5 - MOVAPS X10, X4 - PUNPCKLLQ X5, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 5 - MOVAPS X7, X5 - SHUFPS $0xd6, X11, X5 - SHUFPS $0x39, X5, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X6 - PSRLL $0x0c, X1 - PSLLL $0x14, X6 - POR X6, X1 - MOVAPS X12, X6 - SHUFPS $0xfa, X4, X6 - PSHUFD $0x0f, X7, X7 - PBLENDW $0x33, X7, X6 - PADDD X6, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x07, X1 - PSLLL $0x19, X7 - POR X7, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X4, X10 - PUNPCKLLQ X11, X10 - PBLENDW $0xc0, X12, X10 - SHUFPS $0xb4, X10, X10 - PADDD X10, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x0c, X1 - PSLLL $0x14, X7 - POR X7, X1 - MOVAPS X11, X7 - PUNPCKHLQ X4, X7 - MOVAPS X12, X4 - PUNPCKLLQ X7, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X7 - PSRLL $0x07, X1 - PSLLL $0x19, X7 - POR X7, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 6 - MOVAPS X5, X7 - SHUFPS $0xd6, X6, X7 - SHUFPS $0x39, X7, X7 - PADDD X7, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X11 - PSRLL $0x0c, X1 - PSLLL $0x14, X11 - POR X11, X1 - MOVAPS X10, X11 - SHUFPS $0xfa, X4, X11 - PSHUFD $0x0f, X5, X5 - PBLENDW $0x33, X5, X11 - PADDD X11, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X4, X12 - PUNPCKLLQ X6, X12 - PBLENDW $0xc0, X10, X12 - SHUFPS $0xb4, X12, X12 - PADDD X12, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x0c, X1 - PSLLL $0x14, X5 - POR X5, X1 - MOVAPS X6, X5 - PUNPCKHLQ X4, X5 - MOVAPS X10, X4 - PUNPCKLLQ X5, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // round 7 - MOVAPS X7, X5 - SHUFPS $0xd6, X11, X5 - SHUFPS $0x39, X5, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x0c, X1 - PSLLL $0x14, X5 - POR X5, X1 - MOVAPS X12, X5 - SHUFPS $0xfa, X4, X5 - PSHUFD $0x0f, X7, X6 - PBLENDW $0x33, X6, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x07, X1 - PSLLL $0x19, X5 - POR X5, X1 - PSHUFD $0x93, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x39, X2, X2 - MOVAPS X4, X5 - PUNPCKLLQ X11, X5 - PBLENDW $0xc0, X12, X5 - SHUFPS $0xb4, X5, X5 - PADDD X5, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X8, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X5 - PSRLL $0x0c, X1 - PSLLL $0x14, X5 - POR X5, X1 - MOVAPS X11, X6 - PUNPCKHLQ X4, X6 - MOVAPS X12, X4 - PUNPCKLLQ X6, X4 - SHUFPS $0x1e, X4, X4 - PADDD X4, X0 - PADDD X1, X0 - PXOR X0, X3 - PSHUFB X9, X3 - PADDD X3, X2 - PXOR X2, X1 - MOVAPS X1, X4 - PSRLL $0x07, X1 - PSLLL $0x19, X4 - POR X4, X1 - PSHUFD $0x39, X0, X0 - PSHUFD $0x4e, X3, X3 - PSHUFD $0x93, X2, X2 - - // finalize - PXOR X2, X0 - PXOR X3, X1 - MOVUPS (AX), X4 - PXOR X4, X2 - MOVUPS 16(AX), X4 - PXOR X4, X3 - MOVUPS X0, (SI) - MOVUPS X1, 16(SI) - MOVUPS X2, 32(SI) - MOVUPS X3, 48(SI) - RET diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go deleted file mode 100644 index cd63e9740..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/impl_other.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !amd64 - -package compress_sse41 - -import "github.com/zeebo/blake3/internal/alg/compress/compress_pure" - -func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) { - compress_pure.Compress(chain, block, counter, blen, flags, out) -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go b/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go deleted file mode 100644 index ffd932d3c..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/compress/compress_sse41/stubs.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build amd64 - -package compress_sse41 - -//go:noescape -func Compress(chain *[8]uint32, block *[16]uint32, counter uint64, blen uint32, flags uint32, out *[16]uint32) diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go deleted file mode 100644 index ac43abb69..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash.go +++ /dev/null @@ -1,23 +0,0 @@ -package hash - -import ( - "github.com/zeebo/blake3/internal/alg/hash/hash_avx2" - "github.com/zeebo/blake3/internal/alg/hash/hash_pure" - "github.com/zeebo/blake3/internal/consts" -) - -func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { - if consts.HasAVX2 && length > 2*consts.ChunkLen { - hash_avx2.HashF(input, length, counter, flags, key, out, chain) - } else { - hash_pure.HashF(input, length, counter, flags, key, out, chain) - } -} - -func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { - if consts.HasAVX2 && n >= 2 { - hash_avx2.HashP(left, right, flags, key, out, n) - } else { - hash_pure.HashP(left, right, flags, key, out, n) - } -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s deleted file mode 100644 index d7531664b..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_amd64.s +++ /dev/null @@ -1,2561 +0,0 @@ -// Code generated by command: go run main.go. DO NOT EDIT. - -#include "textflag.h" - -DATA iv<>+0(SB)/4, $0x6a09e667 -DATA iv<>+4(SB)/4, $0xbb67ae85 -DATA iv<>+8(SB)/4, $0x3c6ef372 -DATA iv<>+12(SB)/4, $0xa54ff53a -DATA iv<>+16(SB)/4, $0x510e527f -DATA iv<>+20(SB)/4, $0x9b05688c -DATA iv<>+24(SB)/4, $0x1f83d9ab -DATA iv<>+28(SB)/4, $0x5be0cd19 -GLOBL iv<>(SB), RODATA|NOPTR, $32 - -DATA rot16_shuf<>+0(SB)/1, $0x02 -DATA rot16_shuf<>+1(SB)/1, $0x03 -DATA rot16_shuf<>+2(SB)/1, $0x00 -DATA rot16_shuf<>+3(SB)/1, $0x01 -DATA rot16_shuf<>+4(SB)/1, $0x06 -DATA rot16_shuf<>+5(SB)/1, $0x07 -DATA rot16_shuf<>+6(SB)/1, $0x04 -DATA rot16_shuf<>+7(SB)/1, $0x05 -DATA rot16_shuf<>+8(SB)/1, $0x0a -DATA rot16_shuf<>+9(SB)/1, $0x0b -DATA rot16_shuf<>+10(SB)/1, $0x08 -DATA rot16_shuf<>+11(SB)/1, $0x09 -DATA rot16_shuf<>+12(SB)/1, $0x0e -DATA rot16_shuf<>+13(SB)/1, $0x0f -DATA rot16_shuf<>+14(SB)/1, $0x0c -DATA rot16_shuf<>+15(SB)/1, $0x0d -DATA rot16_shuf<>+16(SB)/1, $0x12 -DATA rot16_shuf<>+17(SB)/1, $0x13 -DATA rot16_shuf<>+18(SB)/1, $0x10 -DATA rot16_shuf<>+19(SB)/1, $0x11 -DATA rot16_shuf<>+20(SB)/1, $0x16 -DATA rot16_shuf<>+21(SB)/1, $0x17 -DATA rot16_shuf<>+22(SB)/1, $0x14 -DATA rot16_shuf<>+23(SB)/1, $0x15 -DATA rot16_shuf<>+24(SB)/1, $0x1a -DATA rot16_shuf<>+25(SB)/1, $0x1b -DATA rot16_shuf<>+26(SB)/1, $0x18 -DATA rot16_shuf<>+27(SB)/1, $0x19 -DATA rot16_shuf<>+28(SB)/1, $0x1e -DATA rot16_shuf<>+29(SB)/1, $0x1f -DATA rot16_shuf<>+30(SB)/1, $0x1c -DATA rot16_shuf<>+31(SB)/1, $0x1d -GLOBL rot16_shuf<>(SB), RODATA|NOPTR, $32 - -DATA rot8_shuf<>+0(SB)/1, $0x01 -DATA rot8_shuf<>+1(SB)/1, $0x02 -DATA rot8_shuf<>+2(SB)/1, $0x03 -DATA rot8_shuf<>+3(SB)/1, $0x00 -DATA rot8_shuf<>+4(SB)/1, $0x05 -DATA rot8_shuf<>+5(SB)/1, $0x06 -DATA rot8_shuf<>+6(SB)/1, $0x07 -DATA rot8_shuf<>+7(SB)/1, $0x04 -DATA rot8_shuf<>+8(SB)/1, $0x09 -DATA rot8_shuf<>+9(SB)/1, $0x0a -DATA rot8_shuf<>+10(SB)/1, $0x0b -DATA rot8_shuf<>+11(SB)/1, $0x08 -DATA rot8_shuf<>+12(SB)/1, $0x0d -DATA rot8_shuf<>+13(SB)/1, $0x0e -DATA rot8_shuf<>+14(SB)/1, $0x0f -DATA rot8_shuf<>+15(SB)/1, $0x0c -DATA rot8_shuf<>+16(SB)/1, $0x11 -DATA rot8_shuf<>+17(SB)/1, $0x12 -DATA rot8_shuf<>+18(SB)/1, $0x13 -DATA rot8_shuf<>+19(SB)/1, $0x10 -DATA rot8_shuf<>+20(SB)/1, $0x15 -DATA rot8_shuf<>+21(SB)/1, $0x16 -DATA rot8_shuf<>+22(SB)/1, $0x17 -DATA rot8_shuf<>+23(SB)/1, $0x14 -DATA rot8_shuf<>+24(SB)/1, $0x19 -DATA rot8_shuf<>+25(SB)/1, $0x1a -DATA rot8_shuf<>+26(SB)/1, $0x1b -DATA rot8_shuf<>+27(SB)/1, $0x18 -DATA rot8_shuf<>+28(SB)/1, $0x1d -DATA rot8_shuf<>+29(SB)/1, $0x1e -DATA rot8_shuf<>+30(SB)/1, $0x1f -DATA rot8_shuf<>+31(SB)/1, $0x1c -GLOBL rot8_shuf<>(SB), RODATA|NOPTR, $32 - -DATA block_len<>+0(SB)/4, $0x00000040 -DATA block_len<>+4(SB)/4, $0x00000040 -DATA block_len<>+8(SB)/4, $0x00000040 -DATA block_len<>+12(SB)/4, $0x00000040 -DATA block_len<>+16(SB)/4, $0x00000040 -DATA block_len<>+20(SB)/4, $0x00000040 -DATA block_len<>+24(SB)/4, $0x00000040 -DATA block_len<>+28(SB)/4, $0x00000040 -GLOBL block_len<>(SB), RODATA|NOPTR, $32 - -DATA zero<>+0(SB)/4, $0x00000000 -DATA zero<>+4(SB)/4, $0x00000000 -DATA zero<>+8(SB)/4, $0x00000000 -DATA zero<>+12(SB)/4, $0x00000000 -DATA zero<>+16(SB)/4, $0x00000000 -DATA zero<>+20(SB)/4, $0x00000000 -DATA zero<>+24(SB)/4, $0x00000000 -DATA zero<>+28(SB)/4, $0x00000000 -GLOBL zero<>(SB), RODATA|NOPTR, $32 - -DATA counter<>+0(SB)/8, $0x0000000000000000 -DATA counter<>+8(SB)/8, $0x0000000000000001 -DATA counter<>+16(SB)/8, $0x0000000000000002 -DATA counter<>+24(SB)/8, $0x0000000000000003 -DATA counter<>+32(SB)/8, $0x0000000000000004 -DATA counter<>+40(SB)/8, $0x0000000000000005 -DATA counter<>+48(SB)/8, $0x0000000000000006 -DATA counter<>+56(SB)/8, $0x0000000000000007 -GLOBL counter<>(SB), RODATA|NOPTR, $64 - -// func HashF(input *[8192]byte, length uint64, counter uint64, flags uint32, key *[8]uint32, out *[32]uint32, chain *[8]uint32) -// Requires: AVX, AVX2 -TEXT ·HashF(SB), $688-56 - MOVQ input+0(FP), AX - MOVQ length+8(FP), CX - MOVQ counter+16(FP), DX - MOVL flags+24(FP), BX - MOVQ key+32(FP), BP - MOVQ out+40(FP), SI - MOVQ chain+48(FP), DI - - // Allocate local space and align it - LEAQ 31(SP), R10 - MOVQ $0x000000000000001f, R8 - NOTQ R8 - ANDQ R8, R10 - - // Skip if the length is zero - XORQ R8, R8 - XORQ R9, R9 - TESTQ CX, CX - JZ skip_compute - - // Compute complete chunks and blocks - SUBQ $0x01, CX - MOVQ CX, R8 - SHRQ $0x0a, R8 - MOVQ CX, R9 - ANDQ $0x000003c0, R9 - -skip_compute: - // Load some params into the stack (avo improvment?) - MOVL BX, 64(SP) - MOVQ DX, 72(SP) - - // Load IV into vectors - VPBROADCASTD (BP), Y0 - VPBROADCASTD 4(BP), Y1 - VPBROADCASTD 8(BP), Y2 - VPBROADCASTD 12(BP), Y3 - VPBROADCASTD 16(BP), Y4 - VPBROADCASTD 20(BP), Y5 - VPBROADCASTD 24(BP), Y6 - VPBROADCASTD 28(BP), Y7 - - // Build and store counter data on the stack - VPBROADCASTQ 72(SP), Y8 - VPADDQ counter<>+0(SB), Y8, Y8 - VPBROADCASTQ 72(SP), Y9 - VPADDQ counter<>+32(SB), Y9, Y9 - VPUNPCKLDQ Y9, Y8, Y10 - VPUNPCKHDQ Y9, Y8, Y8 - VPUNPCKLDQ Y8, Y10, Y9 - VPUNPCKHDQ Y8, Y10, Y8 - VPERMQ $0xd8, Y9, Y9 - VPERMQ $0xd8, Y8, Y8 - VMOVDQU Y9, 112(SP) - VMOVDQU Y8, 144(SP) - - // Set up block flags and variables for iteration - XORQ CX, CX - ORL $0x01, 64(SP) - -loop: - // Include end flags if last block - CMPQ CX, $0x000003c0 - JNE round_setup - ORL $0x02, 64(SP) - -round_setup: - // Load and transpose message vectors - VMOVDQU (AX)(CX*1), Y8 - VMOVDQU 1024(AX)(CX*1), Y9 - VMOVDQU 2048(AX)(CX*1), Y10 - VMOVDQU 3072(AX)(CX*1), Y11 - VMOVDQU 4096(AX)(CX*1), Y12 - VMOVDQU 5120(AX)(CX*1), Y13 - VMOVDQU 6144(AX)(CX*1), Y14 - VMOVDQU 7168(AX)(CX*1), Y15 - VMOVDQA Y0, (R10) - VPUNPCKLDQ Y9, Y8, Y0 - VPUNPCKHDQ Y9, Y8, Y8 - VPUNPCKLDQ Y11, Y10, Y9 - VPUNPCKHDQ Y11, Y10, Y10 - VPUNPCKLDQ Y13, Y12, Y11 - VPUNPCKHDQ Y13, Y12, Y12 - VPUNPCKLDQ Y15, Y14, Y13 - VPUNPCKHDQ Y15, Y14, Y14 - VPUNPCKLQDQ Y9, Y0, Y15 - VPUNPCKHQDQ Y9, Y0, Y0 - VPUNPCKLQDQ Y10, Y8, Y9 - VPUNPCKHQDQ Y10, Y8, Y8 - VPUNPCKLQDQ Y13, Y11, Y10 - VPUNPCKHQDQ Y13, Y11, Y11 - VPUNPCKLQDQ Y14, Y12, Y13 - VPUNPCKHQDQ Y14, Y12, Y12 - VINSERTI128 $0x01, X10, Y15, Y14 - VPERM2I128 $0x31, Y10, Y15, Y10 - VINSERTI128 $0x01, X11, Y0, Y15 - VPERM2I128 $0x31, Y11, Y0, Y0 - VINSERTI128 $0x01, X13, Y9, Y11 - VPERM2I128 $0x31, Y13, Y9, Y9 - VINSERTI128 $0x01, X12, Y8, Y13 - VPERM2I128 $0x31, Y12, Y8, Y8 - VMOVDQU Y14, 176(SP) - VMOVDQU Y15, 208(SP) - VMOVDQU Y11, 240(SP) - VMOVDQU Y13, 272(SP) - VMOVDQU Y10, 304(SP) - VMOVDQU Y0, 336(SP) - VMOVDQU Y9, 368(SP) - VMOVDQU Y8, 400(SP) - VMOVDQU 32(AX)(CX*1), Y0 - VMOVDQU 1056(AX)(CX*1), Y8 - VMOVDQU 2080(AX)(CX*1), Y9 - VMOVDQU 3104(AX)(CX*1), Y10 - VMOVDQU 4128(AX)(CX*1), Y11 - VMOVDQU 5152(AX)(CX*1), Y12 - VMOVDQU 6176(AX)(CX*1), Y13 - VMOVDQU 7200(AX)(CX*1), Y14 - VPUNPCKLDQ Y8, Y0, Y15 - VPUNPCKHDQ Y8, Y0, Y0 - VPUNPCKLDQ Y10, Y9, Y8 - VPUNPCKHDQ Y10, Y9, Y9 - VPUNPCKLDQ Y12, Y11, Y10 - VPUNPCKHDQ Y12, Y11, Y11 - VPUNPCKLDQ Y14, Y13, Y12 - VPUNPCKHDQ Y14, Y13, Y13 - VPUNPCKLQDQ Y8, Y15, Y14 - VPUNPCKHQDQ Y8, Y15, Y8 - VPUNPCKLQDQ Y9, Y0, Y15 - VPUNPCKHQDQ Y9, Y0, Y0 - VPUNPCKLQDQ Y12, Y10, Y9 - VPUNPCKHQDQ Y12, Y10, Y10 - VPUNPCKLQDQ Y13, Y11, Y12 - VPUNPCKHQDQ Y13, Y11, Y11 - VINSERTI128 $0x01, X9, Y14, Y13 - VPERM2I128 $0x31, Y9, Y14, Y9 - VINSERTI128 $0x01, X10, Y8, Y14 - VPERM2I128 $0x31, Y10, Y8, Y8 - VINSERTI128 $0x01, X12, Y15, Y10 - VPERM2I128 $0x31, Y12, Y15, Y12 - VINSERTI128 $0x01, X11, Y0, Y15 - VPERM2I128 $0x31, Y11, Y0, Y0 - VMOVDQU Y13, 432(SP) - VMOVDQU Y14, 464(SP) - VMOVDQU Y10, 496(SP) - VMOVDQU Y15, 528(SP) - VMOVDQU Y9, 560(SP) - VMOVDQU Y8, 592(SP) - VMOVDQU Y12, 624(SP) - VMOVDQU Y0, 656(SP) - - // Load constants for the round - VMOVDQA (R10), Y0 - VMOVDQU block_len<>+0(SB), Y8 - VPBROADCASTD 64(SP), Y9 - VPBROADCASTD iv<>+0(SB), Y10 - VPBROADCASTD iv<>+4(SB), Y11 - VPBROADCASTD iv<>+8(SB), Y12 - VPBROADCASTD iv<>+12(SB), Y13 - VMOVDQU 112(SP), Y14 - VMOVDQU 144(SP), Y15 - - // Save state for partial chunk if necessary - CMPQ CX, R9 - JNE begin_rounds - VMOVDQU Y0, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, (DI) - VMOVDQU Y1, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 4(DI) - VMOVDQU Y2, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 8(DI) - VMOVDQU Y3, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 12(DI) - VMOVDQU Y4, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 16(DI) - VMOVDQU Y5, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 20(DI) - VMOVDQU Y6, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 24(DI) - VMOVDQU Y7, 80(SP) - MOVL 80(SP)(R8*4), DX - MOVL DX, 28(DI) - -begin_rounds: - // Perform the rounds - // Round 1 - VPADDD 176(SP), Y0, Y0 - VPADDD 240(SP), Y1, Y1 - VPADDD 304(SP), Y2, Y2 - VPADDD 368(SP), Y3, Y3 - VPADDD Y4, Y0, Y0 - VPXOR Y0, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y7, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y4, Y4 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y5, Y5 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y6, Y6 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y7, Y7 - VMOVDQA Y0, (R10) - VPSRLD $0x0c, Y4, Y0 - VPSLLD $0x14, Y4, Y4 - VPOR Y0, Y4, Y0 - VPSRLD $0x0c, Y5, Y4 - VPSLLD $0x14, Y5, Y5 - VPOR Y4, Y5, Y4 - VPSRLD $0x0c, Y6, Y5 - VPSLLD $0x14, Y6, Y6 - VPOR Y5, Y6, Y5 - VPSRLD $0x0c, Y7, Y6 - VPSLLD $0x14, Y7, Y7 - VPOR Y6, Y7, Y6 - VMOVDQA (R10), Y7 - VPADDD 208(SP), Y7, Y7 - VPADDD 272(SP), Y1, Y1 - VPADDD 336(SP), Y2, Y2 - VPADDD 400(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 432(SP), Y7, Y7 - VPADDD 496(SP), Y1, Y1 - VPADDD 560(SP), Y2, Y2 - VPADDD 624(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 464(SP), Y7, Y7 - VPADDD 528(SP), Y1, Y1 - VPADDD 592(SP), Y2, Y2 - VPADDD 656(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 2 - VMOVDQA (R10), Y7 - VPADDD 240(SP), Y7, Y7 - VPADDD 272(SP), Y1, Y1 - VPADDD 400(SP), Y2, Y2 - VPADDD 304(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 368(SP), Y7, Y7 - VPADDD 496(SP), Y1, Y1 - VPADDD 176(SP), Y2, Y2 - VPADDD 592(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 208(SP), Y7, Y7 - VPADDD 560(SP), Y1, Y1 - VPADDD 464(SP), Y2, Y2 - VPADDD 656(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 528(SP), Y7, Y7 - VPADDD 336(SP), Y1, Y1 - VPADDD 624(SP), Y2, Y2 - VPADDD 432(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 3 - VMOVDQA (R10), Y7 - VPADDD 272(SP), Y7, Y7 - VPADDD 496(SP), Y1, Y1 - VPADDD 592(SP), Y2, Y2 - VPADDD 400(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 304(SP), Y7, Y7 - VPADDD 560(SP), Y1, Y1 - VPADDD 240(SP), Y2, Y2 - VPADDD 624(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 368(SP), Y7, Y7 - VPADDD 464(SP), Y1, Y1 - VPADDD 528(SP), Y2, Y2 - VPADDD 432(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 336(SP), Y7, Y7 - VPADDD 176(SP), Y1, Y1 - VPADDD 656(SP), Y2, Y2 - VPADDD 208(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 4 - VMOVDQA (R10), Y7 - VPADDD 496(SP), Y7, Y7 - VPADDD 560(SP), Y1, Y1 - VPADDD 624(SP), Y2, Y2 - VPADDD 592(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 400(SP), Y7, Y7 - VPADDD 464(SP), Y1, Y1 - VPADDD 272(SP), Y2, Y2 - VPADDD 656(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 304(SP), Y7, Y7 - VPADDD 528(SP), Y1, Y1 - VPADDD 336(SP), Y2, Y2 - VPADDD 208(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 176(SP), Y7, Y7 - VPADDD 240(SP), Y1, Y1 - VPADDD 432(SP), Y2, Y2 - VPADDD 368(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 5 - VMOVDQA (R10), Y7 - VPADDD 560(SP), Y7, Y7 - VPADDD 464(SP), Y1, Y1 - VPADDD 656(SP), Y2, Y2 - VPADDD 624(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 592(SP), Y7, Y7 - VPADDD 528(SP), Y1, Y1 - VPADDD 496(SP), Y2, Y2 - VPADDD 432(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 400(SP), Y7, Y7 - VPADDD 336(SP), Y1, Y1 - VPADDD 176(SP), Y2, Y2 - VPADDD 368(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 240(SP), Y7, Y7 - VPADDD 272(SP), Y1, Y1 - VPADDD 208(SP), Y2, Y2 - VPADDD 304(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 6 - VMOVDQA (R10), Y7 - VPADDD 464(SP), Y7, Y7 - VPADDD 528(SP), Y1, Y1 - VPADDD 432(SP), Y2, Y2 - VPADDD 656(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 624(SP), Y7, Y7 - VPADDD 336(SP), Y1, Y1 - VPADDD 560(SP), Y2, Y2 - VPADDD 208(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 592(SP), Y7, Y7 - VPADDD 176(SP), Y1, Y1 - VPADDD 240(SP), Y2, Y2 - VPADDD 304(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 272(SP), Y7, Y7 - VPADDD 496(SP), Y1, Y1 - VPADDD 368(SP), Y2, Y2 - VPADDD 400(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Round 7 - VMOVDQA (R10), Y7 - VPADDD 528(SP), Y7, Y7 - VPADDD 336(SP), Y1, Y1 - VPADDD 208(SP), Y2, Y2 - VPADDD 432(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 656(SP), Y7, Y7 - VPADDD 176(SP), Y1, Y1 - VPADDD 464(SP), Y2, Y2 - VPADDD 368(SP), Y3, Y3 - VPADDD Y0, Y7, Y7 - VPXOR Y7, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y5, Y2, Y2 - VPXOR Y2, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y6, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y14, Y10, Y10 - VPXOR Y10, Y0, Y0 - VPADDD Y15, Y11, Y11 - VPXOR Y11, Y4, Y4 - VPADDD Y8, Y12, Y12 - VPXOR Y12, Y5, Y5 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y6, Y6 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VMOVDQA (R10), Y7 - VPADDD 624(SP), Y7, Y7 - VPADDD 240(SP), Y1, Y1 - VPADDD 272(SP), Y2, Y2 - VPADDD 400(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot16_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot16_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot16_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x0c, Y4, Y7 - VPSLLD $0x14, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x0c, Y5, Y7 - VPSLLD $0x14, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x0c, Y6, Y7 - VPSLLD $0x14, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x0c, Y0, Y7 - VPSLLD $0x14, Y0, Y0 - VPOR Y7, Y0, Y0 - VMOVDQA (R10), Y7 - VPADDD 496(SP), Y7, Y7 - VPADDD 560(SP), Y1, Y1 - VPADDD 304(SP), Y2, Y2 - VPADDD 592(SP), Y3, Y3 - VPADDD Y4, Y7, Y7 - VPXOR Y7, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y5, Y1, Y1 - VPXOR Y1, Y14, Y14 - VPSHUFB rot8_shuf<>+0(SB), Y14, Y14 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y15, Y15 - VPSHUFB rot8_shuf<>+0(SB), Y15, Y15 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y8, Y8 - VPSHUFB rot8_shuf<>+0(SB), Y8, Y8 - VPADDD Y9, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPADDD Y14, Y13, Y13 - VPXOR Y13, Y5, Y5 - VPADDD Y15, Y10, Y10 - VPXOR Y10, Y6, Y6 - VPADDD Y8, Y11, Y11 - VPXOR Y11, Y0, Y0 - VMOVDQA Y7, (R10) - VPSRLD $0x07, Y4, Y7 - VPSLLD $0x19, Y4, Y4 - VPOR Y7, Y4, Y4 - VPSRLD $0x07, Y5, Y7 - VPSLLD $0x19, Y5, Y5 - VPOR Y7, Y5, Y5 - VPSRLD $0x07, Y6, Y7 - VPSLLD $0x19, Y6, Y6 - VPOR Y7, Y6, Y6 - VPSRLD $0x07, Y0, Y7 - VPSLLD $0x19, Y0, Y0 - VPOR Y7, Y0, Y0 - - // Finalize rounds - VPXOR Y9, Y6, Y6 - VPXOR (R10), Y10, Y7 - VPXOR Y11, Y1, Y1 - VPXOR Y12, Y2, Y2 - VPXOR Y13, Y3, Y3 - VPXOR Y14, Y0, Y0 - VPXOR Y15, Y4, Y4 - VPXOR Y8, Y5, Y5 - - // Fix up registers for next iteration - VMOVDQU Y7, Y8 - VMOVDQU Y6, Y7 - VMOVDQU Y5, Y6 - VMOVDQU Y4, Y5 - VMOVDQU Y0, Y4 - VMOVDQU Y8, Y0 - - // If we have zero complete chunks, we're done - CMPQ R8, $0x00 - JNE loop_trailer - CMPQ R9, CX - JEQ finalize - -loop_trailer: - // Increment, reset flags, and loop - CMPQ CX, $0x000003c0 - JEQ finalize - ADDQ $0x40, CX - MOVL BX, 64(SP) - JMP loop - -finalize: - // Store result into out - VMOVDQU Y0, (SI) - VMOVDQU Y1, 32(SI) - VMOVDQU Y2, 64(SI) - VMOVDQU Y3, 96(SI) - VMOVDQU Y4, 128(SI) - VMOVDQU Y5, 160(SI) - VMOVDQU Y6, 192(SI) - VMOVDQU Y7, 224(SI) - VZEROUPPER - RET - -// func HashP(left *[32]uint32, right *[32]uint32, flags uint8, key *[8]uint32, out *[32]uint32, n int) -// Requires: AVX, AVX2 -TEXT ·HashP(SB), NOSPLIT, $72-48 - MOVQ left+0(FP), AX - MOVQ right+8(FP), CX - MOVBLZX flags+16(FP), DX - MOVQ key+24(FP), BX - MOVQ out+32(FP), BP - - // Allocate local space and align it - LEAQ 31(SP), SI - MOVQ $0x000000000000001f, DI - NOTQ DI - ANDQ DI, SI - - // Set up flags value - MOVL DX, 64(SP) - - // Perform the rounds - // Round 1 - VPBROADCASTD (BX), Y0 - VPADDD (AX), Y0, Y0 - VPBROADCASTD 4(BX), Y1 - VPADDD 64(AX), Y1, Y1 - VPBROADCASTD 8(BX), Y2 - VPADDD 128(AX), Y2, Y2 - VPBROADCASTD 12(BX), Y3 - VPADDD 192(AX), Y3, Y3 - VPBROADCASTD 16(BX), Y4 - VPADDD Y4, Y0, Y0 - VMOVDQU zero<>+0(SB), Y5 - VPXOR Y0, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPBROADCASTD 20(BX), Y6 - VPADDD Y6, Y1, Y1 - VMOVDQU zero<>+0(SB), Y7 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPBROADCASTD 24(BX), Y8 - VPADDD Y8, Y2, Y2 - VMOVDQU block_len<>+0(SB), Y9 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPBROADCASTD 28(BX), Y10 - VPADDD Y10, Y3, Y3 - VPBROADCASTD 64(SP), Y11 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPBROADCASTD iv<>+0(SB), Y12 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y4, Y4 - VPBROADCASTD iv<>+4(SB), Y13 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y6, Y6 - VPBROADCASTD iv<>+8(SB), Y14 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y8, Y8 - VPBROADCASTD iv<>+12(SB), Y15 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y10, Y10 - VMOVDQA Y0, (SI) - VPSRLD $0x0c, Y4, Y0 - VPSLLD $0x14, Y4, Y4 - VPOR Y0, Y4, Y0 - VPSRLD $0x0c, Y6, Y4 - VPSLLD $0x14, Y6, Y6 - VPOR Y4, Y6, Y4 - VPSRLD $0x0c, Y8, Y6 - VPSLLD $0x14, Y8, Y8 - VPOR Y6, Y8, Y6 - VPSRLD $0x0c, Y10, Y8 - VPSLLD $0x14, Y10, Y10 - VPOR Y8, Y10, Y8 - VMOVDQA (SI), Y10 - VPADDD 32(AX), Y10, Y10 - VPADDD 96(AX), Y1, Y1 - VPADDD 160(AX), Y2, Y2 - VPADDD 224(AX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD (CX), Y10, Y10 - VPADDD 64(CX), Y1, Y1 - VPADDD 128(CX), Y2, Y2 - VPADDD 192(CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 32(CX), Y10, Y10 - VPADDD 96(CX), Y1, Y1 - VPADDD 160(CX), Y2, Y2 - VPADDD 224(CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 2 - VMOVDQA (SI), Y10 - VPADDD 64(AX), Y10, Y10 - VPADDD 96(AX), Y1, Y1 - VPADDD 224(AX), Y2, Y2 - VPADDD 128(AX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 192(AX), Y10, Y10 - VPADDD 64(CX), Y1, Y1 - VPADDD (AX), Y2, Y2 - VPADDD 160(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 32(AX), Y10, Y10 - VPADDD 128(CX), Y1, Y1 - VPADDD 32(CX), Y2, Y2 - VPADDD 224(CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 96(CX), Y10, Y10 - VPADDD 160(AX), Y1, Y1 - VPADDD 192(CX), Y2, Y2 - VPADDD (CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 3 - VMOVDQA (SI), Y10 - VPADDD 96(AX), Y10, Y10 - VPADDD 64(CX), Y1, Y1 - VPADDD 160(CX), Y2, Y2 - VPADDD 224(AX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 128(AX), Y10, Y10 - VPADDD 128(CX), Y1, Y1 - VPADDD 64(AX), Y2, Y2 - VPADDD 192(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 192(AX), Y10, Y10 - VPADDD 32(CX), Y1, Y1 - VPADDD 96(CX), Y2, Y2 - VPADDD (CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 160(AX), Y10, Y10 - VPADDD (AX), Y1, Y1 - VPADDD 224(CX), Y2, Y2 - VPADDD 32(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 4 - VMOVDQA (SI), Y10 - VPADDD 64(CX), Y10, Y10 - VPADDD 128(CX), Y1, Y1 - VPADDD 192(CX), Y2, Y2 - VPADDD 160(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 224(AX), Y10, Y10 - VPADDD 32(CX), Y1, Y1 - VPADDD 96(AX), Y2, Y2 - VPADDD 224(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 128(AX), Y10, Y10 - VPADDD 96(CX), Y1, Y1 - VPADDD 160(AX), Y2, Y2 - VPADDD 32(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD (AX), Y10, Y10 - VPADDD 64(AX), Y1, Y1 - VPADDD (CX), Y2, Y2 - VPADDD 192(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 5 - VMOVDQA (SI), Y10 - VPADDD 128(CX), Y10, Y10 - VPADDD 32(CX), Y1, Y1 - VPADDD 224(CX), Y2, Y2 - VPADDD 192(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 160(CX), Y10, Y10 - VPADDD 96(CX), Y1, Y1 - VPADDD 64(CX), Y2, Y2 - VPADDD (CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 224(AX), Y10, Y10 - VPADDD 160(AX), Y1, Y1 - VPADDD (AX), Y2, Y2 - VPADDD 192(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 64(AX), Y10, Y10 - VPADDD 96(AX), Y1, Y1 - VPADDD 32(AX), Y2, Y2 - VPADDD 128(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 6 - VMOVDQA (SI), Y10 - VPADDD 32(CX), Y10, Y10 - VPADDD 96(CX), Y1, Y1 - VPADDD (CX), Y2, Y2 - VPADDD 224(CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 192(CX), Y10, Y10 - VPADDD 160(AX), Y1, Y1 - VPADDD 128(CX), Y2, Y2 - VPADDD 32(AX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 160(CX), Y10, Y10 - VPADDD (AX), Y1, Y1 - VPADDD 64(AX), Y2, Y2 - VPADDD 128(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 96(AX), Y10, Y10 - VPADDD 64(CX), Y1, Y1 - VPADDD 192(AX), Y2, Y2 - VPADDD 224(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Round 7 - VMOVDQA (SI), Y10 - VPADDD 96(CX), Y10, Y10 - VPADDD 160(AX), Y1, Y1 - VPADDD 32(AX), Y2, Y2 - VPADDD (CX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 224(CX), Y10, Y10 - VPADDD (AX), Y1, Y1 - VPADDD 32(CX), Y2, Y2 - VPADDD 192(AX), Y3, Y3 - VPADDD Y0, Y10, Y10 - VPXOR Y10, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y4, Y1, Y1 - VPXOR Y1, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y6, Y2, Y2 - VPXOR Y2, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y8, Y3, Y3 - VPXOR Y3, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y5, Y12, Y12 - VPXOR Y12, Y0, Y0 - VPADDD Y7, Y13, Y13 - VPXOR Y13, Y4, Y4 - VPADDD Y9, Y14, Y14 - VPXOR Y14, Y6, Y6 - VPADDD Y11, Y15, Y15 - VPXOR Y15, Y8, Y8 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VMOVDQA (SI), Y10 - VPADDD 192(CX), Y10, Y10 - VPADDD 64(AX), Y1, Y1 - VPADDD 96(AX), Y2, Y2 - VPADDD 224(AX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot16_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot16_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot16_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot16_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x0c, Y4, Y10 - VPSLLD $0x14, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x0c, Y6, Y10 - VPSLLD $0x14, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x0c, Y8, Y10 - VPSLLD $0x14, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x0c, Y0, Y10 - VPSLLD $0x14, Y0, Y0 - VPOR Y10, Y0, Y0 - VMOVDQA (SI), Y10 - VPADDD 64(CX), Y10, Y10 - VPADDD 128(CX), Y1, Y1 - VPADDD 128(AX), Y2, Y2 - VPADDD 160(CX), Y3, Y3 - VPADDD Y4, Y10, Y10 - VPXOR Y10, Y11, Y11 - VPSHUFB rot8_shuf<>+0(SB), Y11, Y11 - VPADDD Y6, Y1, Y1 - VPXOR Y1, Y5, Y5 - VPSHUFB rot8_shuf<>+0(SB), Y5, Y5 - VPADDD Y8, Y2, Y2 - VPXOR Y2, Y7, Y7 - VPSHUFB rot8_shuf<>+0(SB), Y7, Y7 - VPADDD Y0, Y3, Y3 - VPXOR Y3, Y9, Y9 - VPSHUFB rot8_shuf<>+0(SB), Y9, Y9 - VPADDD Y11, Y14, Y14 - VPXOR Y14, Y4, Y4 - VPADDD Y5, Y15, Y15 - VPXOR Y15, Y6, Y6 - VPADDD Y7, Y12, Y12 - VPXOR Y12, Y8, Y8 - VPADDD Y9, Y13, Y13 - VPXOR Y13, Y0, Y0 - VMOVDQA Y10, (SI) - VPSRLD $0x07, Y4, Y10 - VPSLLD $0x19, Y4, Y4 - VPOR Y10, Y4, Y4 - VPSRLD $0x07, Y6, Y10 - VPSLLD $0x19, Y6, Y6 - VPOR Y10, Y6, Y6 - VPSRLD $0x07, Y8, Y10 - VPSLLD $0x19, Y8, Y8 - VPOR Y10, Y8, Y8 - VPSRLD $0x07, Y0, Y10 - VPSLLD $0x19, Y0, Y0 - VPOR Y10, Y0, Y0 - - // Finalize - VPXOR (SI), Y12, Y10 - VPXOR Y13, Y1, Y1 - VPXOR Y14, Y2, Y2 - VPXOR Y15, Y3, Y3 - VPXOR Y5, Y0, Y0 - VPXOR Y7, Y4, Y4 - VPXOR Y9, Y6, Y5 - VPXOR Y11, Y8, Y6 - - // Store result into out - VMOVDQU Y10, (BP) - VMOVDQU Y1, 32(BP) - VMOVDQU Y2, 64(BP) - VMOVDQU Y3, 96(BP) - VMOVDQU Y0, 128(BP) - VMOVDQU Y4, 160(BP) - VMOVDQU Y5, 192(BP) - VMOVDQU Y6, 224(BP) - VZEROUPPER - RET diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go deleted file mode 100644 index 613972814..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/impl_other.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !amd64 - -package hash_avx2 - -import "github.com/zeebo/blake3/internal/alg/hash/hash_pure" - -func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { - hash_pure.HashF(input, length, counter, flags, key, out, chain) -} - -func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { - hash_pure.HashP(left, right, flags, key, out, n) -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go deleted file mode 100644 index 10e949550..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_avx2/stubs.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build amd64 - -package hash_avx2 - -//go:noescape -func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) - -//go:noescape -func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go deleted file mode 100644 index 0c6fd63cd..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashf.go +++ /dev/null @@ -1,56 +0,0 @@ -package hash_pure - -import ( - "unsafe" - - "github.com/zeebo/blake3/internal/alg/compress" - "github.com/zeebo/blake3/internal/consts" - "github.com/zeebo/blake3/internal/utils" -) - -func HashF(input *[8192]byte, length, counter uint64, flags uint32, key *[8]uint32, out *[64]uint32, chain *[8]uint32) { - var tmp [16]uint32 - - for i := uint64(0); consts.ChunkLen*i < length && i < 8; i++ { - bchain := *key - bflags := flags | consts.Flag_ChunkStart - start := consts.ChunkLen * i - - for n := uint64(0); n < 16; n++ { - if n == 15 { - bflags |= consts.Flag_ChunkEnd - } - if start+64*n >= length { - break - } - if start+64+64*n >= length { - *chain = bchain - } - - var blockPtr *[16]uint32 - if consts.IsLittleEndian { - blockPtr = (*[16]uint32)(unsafe.Pointer(&input[consts.ChunkLen*i+consts.BlockLen*n])) - } else { - var block [16]uint32 - utils.BytesToWords((*[64]uint8)(unsafe.Pointer(&input[consts.ChunkLen*i+consts.BlockLen*n])), &block) - blockPtr = &block - } - - compress.Compress(&bchain, blockPtr, counter, consts.BlockLen, bflags, &tmp) - - bchain = *(*[8]uint32)(unsafe.Pointer(&tmp[0])) - bflags = flags - } - - out[i+0] = bchain[0] - out[i+8] = bchain[1] - out[i+16] = bchain[2] - out[i+24] = bchain[3] - out[i+32] = bchain[4] - out[i+40] = bchain[5] - out[i+48] = bchain[6] - out[i+56] = bchain[7] - - counter++ - } -} diff --git a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go b/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go deleted file mode 100644 index bee5d8dd0..000000000 --- a/vendor/github.com/zeebo/blake3/internal/alg/hash/hash_pure/hashp.go +++ /dev/null @@ -1,38 +0,0 @@ -package hash_pure - -import "github.com/zeebo/blake3/internal/alg/compress" - -func HashP(left, right *[64]uint32, flags uint32, key *[8]uint32, out *[64]uint32, n int) { - var tmp [16]uint32 - var block [16]uint32 - - for i := 0; i < n && i < 8; i++ { - block[0] = left[i+0] - block[1] = left[i+8] - block[2] = left[i+16] - block[3] = left[i+24] - block[4] = left[i+32] - block[5] = left[i+40] - block[6] = left[i+48] - block[7] = left[i+56] - block[8] = right[i+0] - block[9] = right[i+8] - block[10] = right[i+16] - block[11] = right[i+24] - block[12] = right[i+32] - block[13] = right[i+40] - block[14] = right[i+48] - block[15] = right[i+56] - - compress.Compress(key, &block, 0, 64, flags, &tmp) - - out[i+0] = tmp[0] - out[i+8] = tmp[1] - out[i+16] = tmp[2] - out[i+24] = tmp[3] - out[i+32] = tmp[4] - out[i+40] = tmp[5] - out[i+48] = tmp[6] - out[i+56] = tmp[7] - } -} diff --git a/vendor/github.com/zeebo/blake3/internal/consts/consts.go b/vendor/github.com/zeebo/blake3/internal/consts/consts.go deleted file mode 100644 index 89f08fe10..000000000 --- a/vendor/github.com/zeebo/blake3/internal/consts/consts.go +++ /dev/null @@ -1,29 +0,0 @@ -package consts - -var IV = [...]uint32{IV0, IV1, IV2, IV3, IV4, IV5, IV6, IV7} - -const ( - IV0 = 0x6A09E667 - IV1 = 0xBB67AE85 - IV2 = 0x3C6EF372 - IV3 = 0xA54FF53A - IV4 = 0x510E527F - IV5 = 0x9B05688C - IV6 = 0x1F83D9AB - IV7 = 0x5BE0CD19 -) - -const ( - Flag_ChunkStart uint32 = 1 << 0 - Flag_ChunkEnd uint32 = 1 << 1 - Flag_Parent uint32 = 1 << 2 - Flag_Root uint32 = 1 << 3 - Flag_Keyed uint32 = 1 << 4 - Flag_DeriveKeyContext uint32 = 1 << 5 - Flag_DeriveKeyMaterial uint32 = 1 << 6 -) - -const ( - BlockLen = 64 - ChunkLen = 1024 -) diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu.go deleted file mode 100644 index 1eebff943..000000000 --- a/vendor/github.com/zeebo/blake3/internal/consts/cpu.go +++ /dev/null @@ -1,17 +0,0 @@ -package consts - -import ( - "os" - - "golang.org/x/sys/cpu" -) - -var ( - HasAVX2 = cpu.X86.HasAVX2 && - os.Getenv("BLAKE3_DISABLE_AVX2") == "" && - os.Getenv("BLAKE3_PUREGO") == "" - - HasSSE41 = cpu.X86.HasSSE41 && - os.Getenv("BLAKE3_DISABLE_SSE41") == "" && - os.Getenv("BLAKE3_PUREGO") == "" -) diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go deleted file mode 100644 index fb730464f..000000000 --- a/vendor/github.com/zeebo/blake3/internal/consts/cpu_big.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build mips mips64 ppc64 s390x - -package consts - -const IsLittleEndian = false diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go deleted file mode 100644 index 1bae02a74..000000000 --- a/vendor/github.com/zeebo/blake3/internal/consts/cpu_little.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build amd64 386 arm arm64 mipsle mips64le ppc64le riscv64 wasm - -package consts - -const IsLittleEndian = true diff --git a/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go b/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go deleted file mode 100644 index 5f7407a6a..000000000 --- a/vendor/github.com/zeebo/blake3/internal/consts/cpu_other.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !mips,!mips64,!ppc64,!s390x,!amd64,!386,!arm,!arm64,!mipsle,!mips64le,!ppc64le,!riscv64,!wasm - -package consts - -import "unsafe" - -var IsLittleEndian = *(*uint16)(unsafe.Pointer(&[2]byte{0, 1})) != 1 diff --git a/vendor/github.com/zeebo/blake3/internal/utils/utils.go b/vendor/github.com/zeebo/blake3/internal/utils/utils.go deleted file mode 100644 index 0b36f0f0f..000000000 --- a/vendor/github.com/zeebo/blake3/internal/utils/utils.go +++ /dev/null @@ -1,60 +0,0 @@ -package utils - -import ( - "encoding/binary" - "unsafe" -) - -func SliceToArray32(bytes []byte) *[32]uint8 { return (*[32]uint8)(unsafe.Pointer(&bytes[0])) } -func SliceToArray64(bytes []byte) *[64]uint8 { return (*[64]uint8)(unsafe.Pointer(&bytes[0])) } - -func BytesToWords(bytes *[64]uint8, words *[16]uint32) { - words[0] = binary.LittleEndian.Uint32(bytes[0*4:]) - words[1] = binary.LittleEndian.Uint32(bytes[1*4:]) - words[2] = binary.LittleEndian.Uint32(bytes[2*4:]) - words[3] = binary.LittleEndian.Uint32(bytes[3*4:]) - words[4] = binary.LittleEndian.Uint32(bytes[4*4:]) - words[5] = binary.LittleEndian.Uint32(bytes[5*4:]) - words[6] = binary.LittleEndian.Uint32(bytes[6*4:]) - words[7] = binary.LittleEndian.Uint32(bytes[7*4:]) - words[8] = binary.LittleEndian.Uint32(bytes[8*4:]) - words[9] = binary.LittleEndian.Uint32(bytes[9*4:]) - words[10] = binary.LittleEndian.Uint32(bytes[10*4:]) - words[11] = binary.LittleEndian.Uint32(bytes[11*4:]) - words[12] = binary.LittleEndian.Uint32(bytes[12*4:]) - words[13] = binary.LittleEndian.Uint32(bytes[13*4:]) - words[14] = binary.LittleEndian.Uint32(bytes[14*4:]) - words[15] = binary.LittleEndian.Uint32(bytes[15*4:]) -} - -func WordsToBytes(words *[16]uint32, bytes []byte) { - bytes = bytes[:64] - binary.LittleEndian.PutUint32(bytes[0*4:1*4], words[0]) - binary.LittleEndian.PutUint32(bytes[1*4:2*4], words[1]) - binary.LittleEndian.PutUint32(bytes[2*4:3*4], words[2]) - binary.LittleEndian.PutUint32(bytes[3*4:4*4], words[3]) - binary.LittleEndian.PutUint32(bytes[4*4:5*4], words[4]) - binary.LittleEndian.PutUint32(bytes[5*4:6*4], words[5]) - binary.LittleEndian.PutUint32(bytes[6*4:7*4], words[6]) - binary.LittleEndian.PutUint32(bytes[7*4:8*4], words[7]) - binary.LittleEndian.PutUint32(bytes[8*4:9*4], words[8]) - binary.LittleEndian.PutUint32(bytes[9*4:10*4], words[9]) - binary.LittleEndian.PutUint32(bytes[10*4:11*4], words[10]) - binary.LittleEndian.PutUint32(bytes[11*4:12*4], words[11]) - binary.LittleEndian.PutUint32(bytes[12*4:13*4], words[12]) - binary.LittleEndian.PutUint32(bytes[13*4:14*4], words[13]) - binary.LittleEndian.PutUint32(bytes[14*4:15*4], words[14]) - binary.LittleEndian.PutUint32(bytes[15*4:16*4], words[15]) -} - -func KeyFromBytes(key []byte, out *[8]uint32) { - key = key[:32] - out[0] = binary.LittleEndian.Uint32(key[0:]) - out[1] = binary.LittleEndian.Uint32(key[4:]) - out[2] = binary.LittleEndian.Uint32(key[8:]) - out[3] = binary.LittleEndian.Uint32(key[12:]) - out[4] = binary.LittleEndian.Uint32(key[16:]) - out[5] = binary.LittleEndian.Uint32(key[20:]) - out[6] = binary.LittleEndian.Uint32(key[24:]) - out[7] = binary.LittleEndian.Uint32(key[28:]) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index ff65f1ab7..f856e2f54 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -13,19 +13,18 @@ codeberg.org/gruf/go-format # codeberg.org/gruf/go-hashenc v1.0.1 ## explicit; go 1.16 codeberg.org/gruf/go-hashenc -# codeberg.org/gruf/go-mutexes v1.0.1 +# codeberg.org/gruf/go-mutexes v1.1.0 ## explicit; go 1.14 codeberg.org/gruf/go-mutexes # codeberg.org/gruf/go-nowish v1.1.0 ## explicit; go 1.14 -codeberg.org/gruf/go-nowish # codeberg.org/gruf/go-pools v1.0.2 ## explicit; go 1.16 codeberg.org/gruf/go-pools # codeberg.org/gruf/go-runners v1.2.0 ## explicit; go 1.14 codeberg.org/gruf/go-runners -# codeberg.org/gruf/go-store v1.2.2 +# codeberg.org/gruf/go-store v1.3.2 ## explicit; go 1.14 codeberg.org/gruf/go-store/kv codeberg.org/gruf/go-store/storage @@ -524,16 +523,6 @@ github.com/vmihailenco/tagparser/v2/internal/parser github.com/wagslane/go-password-validator # github.com/zeebo/blake3 v0.2.1 ## explicit; go 1.13 -github.com/zeebo/blake3 -github.com/zeebo/blake3/internal/alg -github.com/zeebo/blake3/internal/alg/compress -github.com/zeebo/blake3/internal/alg/compress/compress_pure -github.com/zeebo/blake3/internal/alg/compress/compress_sse41 -github.com/zeebo/blake3/internal/alg/hash -github.com/zeebo/blake3/internal/alg/hash/hash_avx2 -github.com/zeebo/blake3/internal/alg/hash/hash_pure -github.com/zeebo/blake3/internal/consts -github.com/zeebo/blake3/internal/utils # golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b ## explicit; go 1.17 golang.org/x/crypto/acme -- cgit v1.3 From 0e7f24ff268599d08a8f7122cfdfe495d9a560e4 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 18:11:27 +0100 Subject: test with disk storage as well --- internal/media/manager_test.go | 87 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index a3eb0360c..caba1dd36 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "codeberg.org/gruf/go-store/kv" + "codeberg.org/gruf/go-store/storage" "github.com/stretchr/testify/suite" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -268,6 +270,91 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { } } +func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { + ctx := context.Background() + + data := func(_ context.Context) (io.Reader, int, error) { + // load bytes from a test image + b, err := os.ReadFile("./test/test-jpeg.jpg") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), len(b), nil + } + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + temp := fmt.Sprintf("%s/store", os.TempDir()) + defer os.RemoveAll(temp) + + diskStorage, err := kv.OpenFile(temp, &storage.DiskConfig{}) + if err != nil { + panic(err) + } + + diskManager, err := media.NewManager(suite.db, diskStorage) + if err != nil { + panic(err) + } + suite.manager = diskManager + + // process the media with no additional info provided + processingMedia, err := diskManager.ProcessMedia(ctx, data, accountID, nil) + suite.NoError(err) + // fetch the attachment id from the processing media + attachmentID := processingMedia.AttachmentID() + + // do a blocking call to fetch the attachment + attachment, err := processingMedia.LoadAttachment(ctx) + suite.NoError(err) + suite.NotNil(attachment) + + // make sure it's got the stuff set on it that we expect + // the attachment ID and accountID we expect + suite.Equal(attachmentID, attachment.ID) + suite.Equal(accountID, attachment.AccountID) + + // file meta should be correctly derived from the image + suite.EqualValues(gtsmodel.Original{ + Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Original) + suite.EqualValues(gtsmodel.Small{ + Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777, + }, attachment.FileMeta.Small) + suite.Equal("image/jpeg", attachment.File.ContentType) + suite.Equal(269739, attachment.File.FileSize) + suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash) + + // now make sure the attachment is in the database + dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + // make sure the processed file is in storage + processedFullBytes, err := diskStorage.Get(attachment.File.Path) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) + + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedThumbnailBytes, err := diskStorage.Get(attachment.Thumbnail.Path) + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytes) + + processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg") + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytesExpected) + + suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) +} + func TestManagerTestSuite(t *testing.T) { suite.Run(t, &ManagerTestSuite{}) } -- cgit v1.3 From 8e6ba1de5228f999830416667124867983e4721b Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 18:11:38 +0100 Subject: start trying to figure out why this test is failing --- internal/api/client/account/accountverify_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/client/account/accountverify_test.go b/internal/api/client/account/accountverify_test.go index 702eb0836..1b5989704 100644 --- a/internal/api/client/account/accountverify_test.go +++ b/internal/api/client/account/accountverify_test.go @@ -42,7 +42,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { // set up the request recorder := httptest.NewRecorder() - ctx := suite.newContext(recorder, http.MethodPatch, nil, account.UpdateCredentialsPath, "") + ctx := suite.newContext(recorder, http.MethodGet, nil, account.VerifyPath, "") // call the handler suite.accountModule.AccountVerifyGETHandler(ctx) @@ -67,7 +67,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { suite.NoError(err) lastStatusAt, err := time.Parse(time.RFC3339, apimodelAccount.LastStatusAt) suite.NoError(err) - +aaaaaaaaaaaaaaaaaaa suite.Equal(testAccount.ID, apimodelAccount.ID) suite.Equal(testAccount.Username, apimodelAccount.Username) suite.Equal(testAccount.Username, apimodelAccount.Acct) -- cgit v1.3 From c12520167dc68284e2af4dd93a7723f86a413e42 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 18:12:04 +0100 Subject: use background context w/deadline --- internal/federation/dereferencing/account.go | 33 ++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 27591d857..9ff829579 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -27,6 +27,7 @@ import ( "net/url" "strings" "sync" + "time" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/activity/streams" @@ -54,7 +55,7 @@ func instanceAccount(account *gtsmodel.Account) bool { // the remote instance again. // // SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). -func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool, blocking bool) (*gtsmodel.Account, error) { +func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { new := true // check if we already have the account in our db, and just return it unless we'd doing a refresh @@ -65,8 +66,16 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc // make sure the account fields are populated before returning: // even if we're not doing a refresh, the caller might want to block // until everything is loaded - err = d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking) - return remoteAccount, err + if err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking); err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) + } + + updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) + } + + return updatedAccount, err } } @@ -193,7 +202,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem // populateAccountFields populates any fields on the given account that weren't populated by the initial // dereferencing. This includes things like header and avatar etc. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, refresh bool, blocking bool) error { +func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) error { // if we're dealing with an instance account, just bail, we don't need to do anything if instanceAccount(account) { return nil @@ -230,7 +239,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, refresh bool, blocking bool) error { +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) error { accountURI, err := url.Parse(targetAccount.URI) if err != nil { return fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err) @@ -270,7 +279,6 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm if err != nil { return err } - targetAccount.AvatarMediaAttachmentID = newProcessing.AttachmentID() // store it in our map to indicate it's in process d.dereferencingAvatarsLock.Lock() @@ -288,11 +296,15 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } else { // ...otherwise do it async go func() { - if err := lockAndLoad(ctx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil { + dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute)) + if err := lockAndLoad(dlCtx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil { logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of avatar: %s", err) } + done() }() } + + targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { @@ -325,7 +337,6 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm if err != nil { return err } - targetAccount.HeaderMediaAttachmentID = newProcessing.AttachmentID() // store it in our map to indicate it's in process d.dereferencingHeadersLock.Lock() @@ -343,11 +354,15 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } else { // ...otherwise do it async go func() { - if err := lockAndLoad(ctx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil { + dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute)) + if err := lockAndLoad(dlCtx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil { logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of header: %s", err) } + done() }() } + + targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() } return nil -- cgit v1.3 From a6fb93ae2ab14e43985fa45be479a0ed312e14df Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 18:12:28 +0100 Subject: ctx => innerctx --- internal/processing/account/update.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 758cc6600..4d10f1d0c 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -140,7 +140,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { f, err := avatar.Open() return f, int(avatar.Size), err } @@ -167,7 +167,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(ctx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { f, err := header.Open() return f, int(header.Size), err } -- cgit v1.3 From 926b37e271f993625183935a5ba29705b280b0d1 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 24 Jan 2022 18:12:42 +0100 Subject: change getaccount function signature --- internal/federation/dereferencing/dereferencer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index daf82b91e..855c4baf8 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -33,7 +33,7 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool, blocking bool) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) -- cgit v1.3 From eba66d3a88880d3a4023ecb57f0ad410426f3bc1 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 25 Jan 2022 11:21:22 +0100 Subject: only update account in db if changed --- internal/federation/dereferencing/account.go | 74 +++++++++++++++++----------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 9ff829579..0b7265723 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -66,16 +66,20 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc // make sure the account fields are populated before returning: // even if we're not doing a refresh, the caller might want to block // until everything is loaded - if err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking); err != nil { + changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking) + if err != nil { return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) } - updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) + if changed { + updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) + } + return updatedAccount, err } - return updatedAccount, err + return remoteAccount, nil } } @@ -97,7 +101,7 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc } newAccount.ID = ulid - if err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil { + if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil { return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) } @@ -120,16 +124,20 @@ func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAcc } refreshedAccount.ID = remoteAccount.ID - if err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking); err != nil { + changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking) + if err != nil { return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err) } - updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err) + if changed { + updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount) + if err != nil { + return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err) + } + return updatedAccount, nil } - return updatedAccount, nil + return refreshedAccount, nil } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -202,51 +210,57 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem // populateAccountFields populates any fields on the given account that weren't populated by the initial // dereferencing. This includes things like header and avatar etc. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) error { +func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) { // if we're dealing with an instance account, just bail, we don't need to do anything if instanceAccount(account) { - return nil + return false, nil } accountURI, err := url.Parse(account.URI) if err != nil { - return fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %s", account.URI, err) + return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %s", account.URI, err) } if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil { - return fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host) + return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host) } t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) if err != nil { - return fmt.Errorf("populateAccountFields: error getting transport for user: %s", err) + return false, fmt.Errorf("populateAccountFields: error getting transport for user: %s", err) } // fetch the header and avatar - if err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking); err != nil { - return fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err) + changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking) + if err != nil { + return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err) } - return nil + return changed, nil } // fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account, // using a transport on behalf of requestingUsername. // +// The returned boolean indicates whether anything changed -- in other words, whether the +// account should be updated in the database. +// // targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. // // If refresh is true, then the media will be fetched again even if it's already been fetched before. // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) error { +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) { + changed := false + accountURI, err := url.Parse(targetAccount.URI) if err != nil { - return fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err) + return changed, fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err) } if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil { - return fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host) + return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host) } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { @@ -264,7 +278,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // we're not already processing it so start now avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) if err != nil { - return err + return changed, err } data := func(innerCtx context.Context) (io.Reader, int, error) { @@ -277,7 +291,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm Avatar: &avatar, }) if err != nil { - return err + return changed, err } // store it in our map to indicate it's in process @@ -291,7 +305,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := lockAndLoad(ctx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil { - return err + return changed, err } } else { // ...otherwise do it async @@ -305,6 +319,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID() + changed = true } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { @@ -322,7 +337,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // we're not already processing it so start now headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) if err != nil { - return err + return changed, err } data := func(innerCtx context.Context) (io.Reader, int, error) { @@ -335,7 +350,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm Header: &header, }) if err != nil { - return err + return changed, err } // store it in our map to indicate it's in process @@ -349,7 +364,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm // block until loaded if required... if blocking { if err := lockAndLoad(ctx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil { - return err + return changed, err } } else { // ...otherwise do it async @@ -363,9 +378,10 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID() + changed = true } - return nil + return changed, nil } func lockAndLoad(ctx context.Context, lock *sync.Mutex, processing *media.ProcessingMedia, processingMap map[string]*media.ProcessingMedia, accountID string) error { -- cgit v1.3 From 5d9e9e0e7f7e026677e843a5897283faecfb2874 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 25 Jan 2022 12:03:25 +0100 Subject: fix up some account conversion logic --- internal/api/client/account/accountverify_test.go | 2 +- internal/typeutils/internaltoas.go | 74 ++++++++++++----------- internal/typeutils/internaltofrontend.go | 12 ++-- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/internal/api/client/account/accountverify_test.go b/internal/api/client/account/accountverify_test.go index 1b5989704..b5dfdd5e8 100644 --- a/internal/api/client/account/accountverify_test.go +++ b/internal/api/client/account/accountverify_test.go @@ -67,7 +67,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { suite.NoError(err) lastStatusAt, err := time.Parse(time.RFC3339, apimodelAccount.LastStatusAt) suite.NoError(err) -aaaaaaaaaaaaaaaaaaa + suite.Equal(testAccount.ID, apimodelAccount.ID) suite.Equal(testAccount.Username, apimodelAccount.Username) suite.Equal(testAccount.Username, apimodelAccount.Acct) diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index b3aaafff0..8e28bf0f0 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -215,62 +215,64 @@ func (c *converter) AccountToAS(ctx context.Context, a *gtsmodel.Account) (vocab // Used as profile avatar. if a.AvatarMediaAttachmentID != "" { if a.AvatarMediaAttachment == nil { - avatar := >smodel.MediaAttachment{} - if err := c.db.GetByID(ctx, a.AvatarMediaAttachmentID, avatar); err != nil { - return nil, err + avatar, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID) + if err == nil { + a.AvatarMediaAttachment = avatar } - a.AvatarMediaAttachment = avatar } - iconProperty := streams.NewActivityStreamsIconProperty() + if a.AvatarMediaAttachment != nil { + iconProperty := streams.NewActivityStreamsIconProperty() - iconImage := streams.NewActivityStreamsImage() + iconImage := streams.NewActivityStreamsImage() - mediaType := streams.NewActivityStreamsMediaTypeProperty() - mediaType.Set(a.AvatarMediaAttachment.File.ContentType) - iconImage.SetActivityStreamsMediaType(mediaType) + mediaType := streams.NewActivityStreamsMediaTypeProperty() + mediaType.Set(a.AvatarMediaAttachment.File.ContentType) + iconImage.SetActivityStreamsMediaType(mediaType) - avatarURLProperty := streams.NewActivityStreamsUrlProperty() - avatarURL, err := url.Parse(a.AvatarMediaAttachment.URL) - if err != nil { - return nil, err - } - avatarURLProperty.AppendIRI(avatarURL) - iconImage.SetActivityStreamsUrl(avatarURLProperty) + avatarURLProperty := streams.NewActivityStreamsUrlProperty() + avatarURL, err := url.Parse(a.AvatarMediaAttachment.URL) + if err != nil { + return nil, err + } + avatarURLProperty.AppendIRI(avatarURL) + iconImage.SetActivityStreamsUrl(avatarURLProperty) - iconProperty.AppendActivityStreamsImage(iconImage) - person.SetActivityStreamsIcon(iconProperty) + iconProperty.AppendActivityStreamsImage(iconImage) + person.SetActivityStreamsIcon(iconProperty) + } } // image // Used as profile header. if a.HeaderMediaAttachmentID != "" { if a.HeaderMediaAttachment == nil { - header := >smodel.MediaAttachment{} - if err := c.db.GetByID(ctx, a.HeaderMediaAttachmentID, header); err != nil { - return nil, err + header, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID) + if err == nil { + a.HeaderMediaAttachment = header } - a.HeaderMediaAttachment = header } - headerProperty := streams.NewActivityStreamsImageProperty() + if a.HeaderMediaAttachment != nil { + headerProperty := streams.NewActivityStreamsImageProperty() - headerImage := streams.NewActivityStreamsImage() + headerImage := streams.NewActivityStreamsImage() - mediaType := streams.NewActivityStreamsMediaTypeProperty() - mediaType.Set(a.HeaderMediaAttachment.File.ContentType) - headerImage.SetActivityStreamsMediaType(mediaType) + mediaType := streams.NewActivityStreamsMediaTypeProperty() + mediaType.Set(a.HeaderMediaAttachment.File.ContentType) + headerImage.SetActivityStreamsMediaType(mediaType) - headerURLProperty := streams.NewActivityStreamsUrlProperty() - headerURL, err := url.Parse(a.HeaderMediaAttachment.URL) - if err != nil { - return nil, err - } - headerURLProperty.AppendIRI(headerURL) - headerImage.SetActivityStreamsUrl(headerURLProperty) + headerURLProperty := streams.NewActivityStreamsUrlProperty() + headerURL, err := url.Parse(a.HeaderMediaAttachment.URL) + if err != nil { + return nil, err + } + headerURLProperty.AppendIRI(headerURL) + headerImage.SetActivityStreamsUrl(headerURLProperty) - headerProperty.AppendActivityStreamsImage(headerImage) - person.SetActivityStreamsImage(headerProperty) + headerProperty.AppendActivityStreamsImage(headerImage) + person.SetActivityStreamsImage(headerProperty) + } } return person, nil diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 52e89b7d2..989bf4a1e 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -104,10 +104,12 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A avi, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID) if err == nil { a.AvatarMediaAttachment = avi - aviURL = a.AvatarMediaAttachment.URL - aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL } } + if a.AvatarMediaAttachment != nil { + aviURL = a.AvatarMediaAttachment.URL + aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL + } } // set account header fields if available @@ -118,10 +120,12 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A avi, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID) if err == nil { a.HeaderMediaAttachment = avi - headerURL = a.HeaderMediaAttachment.URL - headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL } } + if a.HeaderMediaAttachment != nil { + headerURL = a.HeaderMediaAttachment.URL + headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL + } } // get the fields set on this account -- cgit v1.3 From c156602c6679224fe48b325674022f768aa74d27 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 25 Jan 2022 13:48:13 +0100 Subject: ensure blocking calls to getRemoteAccount before showing stuff to client --- internal/federation/dereferencing/status.go | 2 +- internal/processing/fromfederator.go | 101 +++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index 34310f4aa..cacca91b2 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -89,7 +89,7 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat } // do this so we know we have the remote account of the status in the db - _, err = d.GetRemoteAccount(ctx, username, accountURI, false, false) + _, err = d.GetRemoteAccount(ctx, username, accountURI, true, false) if err != nil { return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err) } diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 8b575dda8..3514614b5 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -115,6 +115,30 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa } } + // make sure the account is pinned + if status.Account == nil { + a, err := p.db.GetAccountByID(ctx, status.AccountID) + if err != nil { + return err + } + status.Account = a + } + + // do a BLOCKING get of the remote account to make sure the avi and header are cached + if status.Account.Domain != "" { + remoteAccountID, err := url.Parse(status.Account.URI) + if err != nil { + return err + } + + a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + if err != nil { + return err + } + + status.Account = a + } + if err := p.timelineStatus(ctx, status); err != nil { return err } @@ -133,6 +157,30 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato return errors.New("like was not parseable as *gtsmodel.StatusFave") } + // make sure the account is pinned + if incomingFave.Account == nil { + a, err := p.db.GetAccountByID(ctx, incomingFave.AccountID) + if err != nil { + return err + } + incomingFave.Account = a + } + + // do a BLOCKING get of the remote account to make sure the avi and header are cached + if incomingFave.Account.Domain != "" { + remoteAccountID, err := url.Parse(incomingFave.Account.URI) + if err != nil { + return err + } + + a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + if err != nil { + return err + } + + incomingFave.Account = a + } + if err := p.notifyFave(ctx, incomingFave); err != nil { return err } @@ -147,6 +195,30 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context, return errors.New("incomingFollowRequest was not parseable as *gtsmodel.FollowRequest") } + // make sure the account is pinned + if followRequest.Account == nil { + a, err := p.db.GetAccountByID(ctx, followRequest.AccountID) + if err != nil { + return err + } + followRequest.Account = a + } + + // do a BLOCKING get of the remote account to make sure the avi and header are cached + if followRequest.Account.Domain != "" { + remoteAccountID, err := url.Parse(followRequest.Account.URI) + if err != nil { + return err + } + + a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + if err != nil { + return err + } + + followRequest.Account = a + } + if followRequest.TargetAccount == nil { a, err := p.db.GetAccountByID(ctx, followRequest.TargetAccountID) if err != nil { @@ -154,9 +226,8 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context, } followRequest.TargetAccount = a } - targetAccount := followRequest.TargetAccount - if targetAccount.Locked { + if followRequest.TargetAccount.Locked { // if the account is locked just notify the follow request and nothing else return p.notifyFollowRequest(ctx, followRequest) } @@ -171,7 +242,7 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context, return err } - return p.notifyFollow(ctx, follow, targetAccount) + return p.notifyFollow(ctx, follow, followRequest.TargetAccount) } // processCreateAnnounceFromFederator handles Activity Create and Object Announce @@ -181,6 +252,30 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede return errors.New("announce was not parseable as *gtsmodel.Status") } + // make sure the account is pinned + if incomingAnnounce.Account == nil { + a, err := p.db.GetAccountByID(ctx, incomingAnnounce.AccountID) + if err != nil { + return err + } + incomingAnnounce.Account = a + } + + // do a BLOCKING get of the remote account to make sure the avi and header are cached + if incomingAnnounce.Account.Domain != "" { + remoteAccountID, err := url.Parse(incomingAnnounce.Account.URI) + if err != nil { + return err + } + + a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + if err != nil { + return err + } + + incomingAnnounce.Account = a + } + if err := p.federator.DereferenceAnnounce(ctx, incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil { return fmt.Errorf("error dereferencing announce from federator: %s", err) } -- cgit v1.3 From 9aa364f1eb686be51fc31c54209b41514cbd8f8b Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 25 Jan 2022 14:33:23 +0100 Subject: go mod tidy --- go.mod | 2 -- go.sum | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/go.mod b/go.mod index 9c33f7f36..b2b9cb5c7 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( codeberg.org/gruf/go-format v1.0.3 // indirect codeberg.org/gruf/go-hashenc v1.0.1 // indirect codeberg.org/gruf/go-mutexes v1.1.0 // indirect - codeberg.org/gruf/go-nowish v1.1.0 // indirect codeberg.org/gruf/go-pools v1.0.2 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -106,7 +105,6 @@ require ( github.com/ugorji/go/codec v1.2.6 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/zeebo/blake3 v0.2.1 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect diff --git a/go.sum b/go.sum index ccfb51a1c..8980a5551 100644 --- a/go.sum +++ b/go.sum @@ -60,20 +60,15 @@ codeberg.org/gruf/go-format v1.0.3 h1:WoUGzTwZe6SIhILNvtr0qNIA7BOOCgdBlk5bUrfeii codeberg.org/gruf/go-format v1.0.3/go.mod h1:k3TLXp1dqAXdDqxlon0yEM+3FFHdNn0D6BVJTwTy5As= codeberg.org/gruf/go-hashenc v1.0.1 h1:EBvNe2wW8IPMUqT1XihB6/IM6KMJDLMFBxIUvmsy1f8= codeberg.org/gruf/go-hashenc v1.0.1/go.mod h1:IfHhPCVScOiYmJLqdCQT9bYVS1nxNTV4ewMUvFWDPtc= -codeberg.org/gruf/go-mutexes v1.0.1 h1:X9bZW74YSEplWWdCrVXAvue5ztw3w5hh+INdXTENu88= -codeberg.org/gruf/go-mutexes v1.0.1/go.mod h1:y2hbGLkWVHhNyxBOIVsA3/y2QMm6RSrYsC3sLVZ4EXM= codeberg.org/gruf/go-mutexes v1.1.0 h1:kMVWHLxdfGEZTetNVRncdBMeqS4M8dSJxSGbRYXyvKk= codeberg.org/gruf/go-mutexes v1.1.0/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8= codeberg.org/gruf/go-nowish v1.0.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= -codeberg.org/gruf/go-nowish v1.1.0 h1:rj1z0AXDhLvnxs/DazWFxYAugs6rv5vhgWJkRCgrESg= codeberg.org/gruf/go-nowish v1.1.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s= codeberg.org/gruf/go-pools v1.0.2 h1:B0X6yoCL9FVmnvyoizb1SYRwMYPWwEJBjPnBMM5ILos= codeberg.org/gruf/go-pools v1.0.2/go.mod h1:MjUV3H6IASyBeBPCyCr7wjPpSNu8E2N87LG4r4TAyok= codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBsH4= codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= -codeberg.org/gruf/go-store v1.2.2 h1:YJPzJpZv/D3t9hQC00/u76eQDScQw4++OWjfobnjHAA= -codeberg.org/gruf/go-store v1.2.2/go.mod h1:Xjw1U098th0yXF2CCx6jThQ+9FIPWAX9OGjYslO+UtE= codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo= codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -728,12 +723,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= -github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/blake3 v0.2.1 h1:O+N0Y8Re2XAYjp0adlZDA2juyRguhMfPCgh8YIf7vyE= -github.com/zeebo/blake3 v0.2.1/go.mod h1:TSQ0KjMH+pht+bRyvVooJ1rBpvvngSGaPISafq9MxJk= -github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= -github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -949,7 +938,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -- cgit v1.3 From 4e74c84148cf3a1e19e5e957122dede5b403648a Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 29 Jan 2022 12:15:51 +0100 Subject: update go-store to latest --- go.mod | 2 +- go.sum | 2 + vendor/codeberg.org/gruf/go-store/storage/block.go | 2 +- vendor/codeberg.org/gruf/go-store/storage/disk.go | 56 +++++++++++++++++++--- vendor/codeberg.org/gruf/go-store/storage/lock.go | 9 +--- vendor/codeberg.org/gruf/go-store/util/fs.go | 31 ------------ vendor/codeberg.org/gruf/go-store/util/sys.go | 14 ++++++ vendor/modules.txt | 6 +-- 8 files changed, 70 insertions(+), 52 deletions(-) create mode 100644 vendor/codeberg.org/gruf/go-store/util/sys.go diff --git a/go.mod b/go.mod index b2b9cb5c7..e15720309 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( codeberg.org/gruf/go-runners v1.2.0 - codeberg.org/gruf/go-store v1.3.2 + codeberg.org/gruf/go-store v1.3.3 github.com/ReneKroon/ttlcache v1.7.0 github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.1.0 diff --git a/go.sum b/go.sum index 8980a5551..85d10b2a5 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBs codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo= codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= +codeberg.org/gruf/go-store v1.3.3 h1:fAP9FXy6HiLPxdD7cmpSzyfKXmVvZLjqn0m7HhxVT5M= +codeberg.org/gruf/go-store v1.3.3/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/vendor/codeberg.org/gruf/go-store/storage/block.go b/vendor/codeberg.org/gruf/go-store/storage/block.go index 5075c7d17..c50faa10b 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/block.go +++ b/vendor/codeberg.org/gruf/go-store/storage/block.go @@ -140,7 +140,7 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) { } // Open and acquire storage lock for path - lock, err := OpenLock(pb.Join(path, lockFile)) + lock, err := OpenLock(pb.Join(path, LockFile)) if err != nil { return nil, err } diff --git a/vendor/codeberg.org/gruf/go-store/storage/disk.go b/vendor/codeberg.org/gruf/go-store/storage/disk.go index 2ee00ddee..287042886 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/disk.go +++ b/vendor/codeberg.org/gruf/go-store/storage/disk.go @@ -5,6 +5,8 @@ import ( "io/fs" "os" "path" + _path "path" + "strings" "syscall" "codeberg.org/gruf/go-bytes" @@ -31,6 +33,11 @@ type DiskConfig struct { // Overwrite allows overwriting values of stored keys in the storage Overwrite bool + // LockFile allows specifying the filesystem path to use for the lockfile, + // providing only a filename it will store the lockfile within provided store + // path and nest the store under `path/store` to prevent access to lockfile + LockFile string + // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor } @@ -57,11 +64,17 @@ func getDiskConfig(cfg *DiskConfig) DiskConfig { cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize } + // Assume empty lockfile path == use default + if len(cfg.LockFile) < 1 { + cfg.LockFile = LockFile + } + // Return owned config copy return DiskConfig{ Transform: cfg.Transform, WriteBufSize: cfg.WriteBufSize, Overwrite: cfg.Overwrite, + LockFile: cfg.LockFile, Compression: cfg.Compression, } } @@ -76,16 +89,27 @@ type DiskStorage struct { // OpenFile opens a DiskStorage instance for given folder path and configuration func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { + // Get checked config + config := getDiskConfig(cfg) + // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) - // Clean provided path, ensure ends in '/' (should - // be dir, this helps with file path trimming later) - storePath := pb.Join(path, "store") + "/" + // Clean provided store path, ensure + // ends in '/' to help later path trimming + storePath := pb.Clean(path) + "/" - // Get checked config - config := getDiskConfig(cfg) + // Clean provided lockfile path + lockfile := pb.Clean(config.LockFile) + + // Check if lockfile is an *actual* path or just filename + if lockDir, _ := _path.Split(lockfile); len(lockDir) < 1 { + // Lockfile is a filename, store must be nested under + // $storePath/store to prevent access to the lockfile + storePath += "store/" + lockfile = pb.Join(path, lockfile) + } // Attempt to open dir path file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms) @@ -118,7 +142,7 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { } // Open and acquire storage lock for path - lock, err := OpenLock(pb.Join(path, lockFile)) + lock, err := OpenLock(lockfile) if err != nil { return nil, err } @@ -347,9 +371,27 @@ func (st *DiskStorage) filepath(key string) (string, error) { pb.AppendString(key) // Check for dir traversal outside of root - if util.IsDirTraversal(st.path, pb.StringPtr()) { + if isDirTraversal(st.path, pb.StringPtr()) { return "", ErrInvalidKey } return pb.String(), nil } + +// isDirTraversal will check if rootPlusPath is a dir traversal outside of root, +// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath) +func isDirTraversal(root, rootPlusPath string) bool { + switch { + // Root is $PWD, check for traversal out of + case root == ".": + return strings.HasPrefix(rootPlusPath, "../") + + // The path MUST be prefixed by root + case !strings.HasPrefix(rootPlusPath, root): + return true + + // In all other cases, check not equal + default: + return len(root) == len(rootPlusPath) + } +} diff --git a/vendor/codeberg.org/gruf/go-store/storage/lock.go b/vendor/codeberg.org/gruf/go-store/storage/lock.go index fae4351bf..8a6c4c5e8 100644 --- a/vendor/codeberg.org/gruf/go-store/storage/lock.go +++ b/vendor/codeberg.org/gruf/go-store/storage/lock.go @@ -8,13 +8,8 @@ import ( "codeberg.org/gruf/go-store/util" ) -// lockFile is our standard lockfile name. -var lockFile = "store.lock" - -// IsLockKey returns whether storage key is our lockfile. -func IsLockKey(key string) bool { - return key == lockFile -} +// LockFile is our standard lockfile name. +const LockFile = "store.lock" // Lock represents a filesystem lock to ensure only one storage instance open per path. type Lock struct { diff --git a/vendor/codeberg.org/gruf/go-store/util/fs.go b/vendor/codeberg.org/gruf/go-store/util/fs.go index 93b37a261..53fef7750 100644 --- a/vendor/codeberg.org/gruf/go-store/util/fs.go +++ b/vendor/codeberg.org/gruf/go-store/util/fs.go @@ -3,30 +3,10 @@ package util import ( "io/fs" "os" - "strings" - "syscall" "codeberg.org/gruf/go-fastpath" ) -// IsDirTraversal will check if rootPlusPath is a dir traversal outside of root, -// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath) -func IsDirTraversal(root string, rootPlusPath string) bool { - switch { - // Root is $PWD, check for traversal out of - case root == ".": - return strings.HasPrefix(rootPlusPath, "../") - - // The path MUST be prefixed by root - case !strings.HasPrefix(rootPlusPath, root): - return true - - // In all other cases, check not equal - default: - return len(root) == len(rootPlusPath) - } -} - // WalkDir traverses the dir tree of the supplied path, performing the supplied walkFn on each entry func WalkDir(pb *fastpath.Builder, path string, walkFn func(string, fs.DirEntry)) error { // Read supplied dir path @@ -100,14 +80,3 @@ func cleanDirs(pb *fastpath.Builder, path string) error { } return nil } - -// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received -func RetryOnEINTR(do func() error) error { - for { - err := do() - if err == syscall.EINTR { - continue - } - return err - } -} diff --git a/vendor/codeberg.org/gruf/go-store/util/sys.go b/vendor/codeberg.org/gruf/go-store/util/sys.go new file mode 100644 index 000000000..6661029e5 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-store/util/sys.go @@ -0,0 +1,14 @@ +package util + +import "syscall" + +// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received +func RetryOnEINTR(do func() error) error { + for { + err := do() + if err == syscall.EINTR { + continue + } + return err + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f856e2f54..2ef54775b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,15 +16,13 @@ codeberg.org/gruf/go-hashenc # codeberg.org/gruf/go-mutexes v1.1.0 ## explicit; go 1.14 codeberg.org/gruf/go-mutexes -# codeberg.org/gruf/go-nowish v1.1.0 -## explicit; go 1.14 # codeberg.org/gruf/go-pools v1.0.2 ## explicit; go 1.16 codeberg.org/gruf/go-pools # codeberg.org/gruf/go-runners v1.2.0 ## explicit; go 1.14 codeberg.org/gruf/go-runners -# codeberg.org/gruf/go-store v1.3.2 +# codeberg.org/gruf/go-store v1.3.3 ## explicit; go 1.14 codeberg.org/gruf/go-store/kv codeberg.org/gruf/go-store/storage @@ -521,8 +519,6 @@ github.com/vmihailenco/tagparser/v2/internal/parser # github.com/wagslane/go-password-validator v0.3.0 ## explicit; go 1.16 github.com/wagslane/go-password-validator -# github.com/zeebo/blake3 v0.2.1 -## explicit; go 1.13 # golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b ## explicit; go 1.17 golang.org/x/crypto/acme -- cgit v1.3 From d9a778a795fe8c235077bc0e8d904ba5695c5191 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 29 Jan 2022 12:16:12 +0100 Subject: put store lock file in base of storage --- cmd/gotosocial/action/server/server.go | 6 +++++- internal/media/manager_test.go | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 5c1192b56..ae43995e5 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -24,9 +24,11 @@ import ( "net/http" "os" "os/signal" + "path" "syscall" "codeberg.org/gruf/go-store/kv" + "codeberg.org/gruf/go-store/storage" "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action" @@ -99,7 +101,9 @@ var Start action.GTSAction = func(ctx context.Context) error { // Open the storage backend storageBasePath := viper.GetString(config.Keys.StorageLocalBasePath) - storage, err := kv.OpenFile(storageBasePath, nil) + storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{ + LockFile: path.Join(storageBasePath, "store.lock"), // put the store lockfile in the storage dir itself + }) if err != nil { return fmt.Errorf("error creating storage backend: %s", err) } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index caba1dd36..a9419754c 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "os" + "path" "testing" "time" @@ -284,10 +285,12 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { accountID := "01FS1X72SK9ZPW0J1QQ68BD264" - temp := fmt.Sprintf("%s/store", os.TempDir()) + temp := fmt.Sprintf("%s/gotosocial-test", os.TempDir()) defer os.RemoveAll(temp) - diskStorage, err := kv.OpenFile(temp, &storage.DiskConfig{}) + diskStorage, err := kv.OpenFile(temp, &storage.DiskConfig{ + LockFile: path.Join(temp, "store.lock"), + }) if err != nil { panic(err) } -- cgit v1.3 From 9db58dfd57784dee15c96b630bfbe74c991f518a Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 10:06:43 +0100 Subject: add comment explaining store.lock --- cmd/gotosocial/action/server/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index ae43995e5..d71ef5b4f 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -102,7 +102,12 @@ var Start action.GTSAction = func(ctx context.Context) error { // Open the storage backend storageBasePath := viper.GetString(config.Keys.StorageLocalBasePath) storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{ - LockFile: path.Join(storageBasePath, "store.lock"), // put the store lockfile in the storage dir itself + // Put the store lockfile in the storage dir itself. + // Normally this would not be safe, since we could end up + // overwriting the lockfile if we store a file called 'store.lock'. + // However, in this case it's OK because the keys are set by + // GtS and not the user, so we know we're never going to overwrite it. + LockFile: path.Join(storageBasePath, "store.lock"), }) if err != nil { return fmt.Errorf("error creating storage backend: %s", err) -- cgit v1.3 From e811d03cc94f42ea5aa0b4eb8d468d01fdf6d31c Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 10:08:27 +0100 Subject: update outdated comment --- internal/federation/dereferencing/account.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 0b7265723..d87192d3a 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -47,12 +47,11 @@ func instanceAccount(account *gtsmodel.Account) bool { } // GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, -// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new -// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before, -// it will be false. +// puts it in the database, and returns it to a caller. // // Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling -// the remote instance again. +// the remote instance again. Blocking indicates whether the function should block until processing of +// the fetched account is complete. // // SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { -- cgit v1.3 From 3301148bb73a0f5f32c49417daed6d914c9ec05e Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 12:17:53 +0100 Subject: merge fixup --- cmd/gotosocial/action/server/server.go | 2 +- internal/api/client/media/mediaupdate_test.go | 8 ++++---- internal/api/s2s/webfinger/webfingerget_test.go | 4 ++-- internal/processing/admin/emoji.go | 4 +++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index b8bf62274..8686f6a7f 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -138,7 +138,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } // create and start the message processor using the other services we've created so far - processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, dbService, emailSender) + processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender) if err := processor.Start(ctx); err != nil { return fmt.Errorf("error starting processor: %s", err) } diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index cac6c304e..b99c89c06 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -54,7 +54,7 @@ type MediaUpdateTestSuite struct { storage *kv.KVStore federator federation.Federator tc typeutils.TypeConverter - mediaHandler media.Handler + mediaManager media.Manager oauthServer oauth.Server emailSender email.Sender processor processing.Processor @@ -82,11 +82,11 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) // setup module being tested suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module) diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go index d3b0c32e8..3d271a260 100644 --- a/internal/api/s2s/webfinger/webfingerget_test.go +++ b/internal/api/s2s/webfinger/webfingerget_test.go @@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() { func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() @@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index bb9f4ecb5..6ef78aa65 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -20,6 +20,7 @@ package admin import ( "context" + "errors" "fmt" "io" @@ -55,7 +56,8 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, emoji, err := processingEmoji.LoadEmoji(ctx) if err != nil { - if err == db.ErrAlreadyExists { + var alreadyExistsError *db.ErrAlreadyExists + if errors.As(err, &alreadyExistsError) { return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode)) } return nil, gtserror.NewErrorInternalError(fmt.Errorf("error loading emoji: %s", err), "error loading emoji") -- cgit v1.3 From b42b0a667ef8c3287919a4b7f379e49683f13077 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 12:19:54 +0100 Subject: go fmt --- cmd/gotosocial/action/server/server.go | 2 +- internal/db/bundb/errors.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 8686f6a7f..8d447c59f 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,7 +105,7 @@ var Start action.GTSAction = func(ctx context.Context) error { // overwriting the lockfile if we store a file called 'store.lock'. // However, in this case it's OK because the keys are set by // GtS and not the user, so we know we're never going to overwrite it. - LockFile: path.Join(storageBasePath, "store.lock"), + LockFile: path.Join(storageBasePath, "store.lock"), }) if err != nil { return fmt.Errorf("error creating storage backend: %s", err) diff --git a/internal/db/bundb/errors.go b/internal/db/bundb/errors.go index 113679226..67a673e15 100644 --- a/internal/db/bundb/errors.go +++ b/internal/db/bundb/errors.go @@ -36,7 +36,7 @@ func processSQLiteError(err error) db.Error { // Handle supplied error code: switch sqliteErr.Code() { case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - return db.NewErrAlreadyExists(err.Error()) + return db.NewErrAlreadyExists(err.Error()) default: return err } -- cgit v1.3 From c4ece24654e195dbff06c1d7363c5fe92e1fd4c4 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 12:37:58 +0100 Subject: log error on header/avatar fetch fail --- internal/typeutils/internaltoas.go | 5 +++++ internal/typeutils/internaltofrontend.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index a22b926d6..f154398ce 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -25,6 +25,7 @@ import ( "fmt" "net/url" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams" @@ -218,6 +219,8 @@ func (c *converter) AccountToAS(ctx context.Context, a *gtsmodel.Account) (vocab avatar, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID) if err == nil { a.AvatarMediaAttachment = avatar + } else { + logrus.Errorf("AccountToAS: error getting Avatar with id %s: %s", a.AvatarMediaAttachmentID, err) } } @@ -250,6 +253,8 @@ func (c *converter) AccountToAS(ctx context.Context, a *gtsmodel.Account) (vocab header, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID) if err == nil { a.HeaderMediaAttachment = header + } else { + logrus.Errorf("AccountToAS: error getting Header with id %s: %s", a.HeaderMediaAttachmentID, err) } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 989bf4a1e..39a2861fb 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -104,6 +104,8 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A avi, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID) if err == nil { a.AvatarMediaAttachment = avi + } else { + logrus.Errorf("AccountToAPIAccountPublic: error getting Avatar with id %s: %s", a.AvatarMediaAttachmentID, err) } } if a.AvatarMediaAttachment != nil { @@ -120,6 +122,8 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A avi, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID) if err == nil { a.HeaderMediaAttachment = avi + } else { + logrus.Errorf("AccountToAPIAccountPublic: error getting Header with id %s: %s", a.HeaderMediaAttachmentID, err) } } if a.HeaderMediaAttachment != nil { -- cgit v1.3 From dba9ad434885603b85938386a4681350698decac Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 13:17:10 +0100 Subject: hopefully fix potential race condition --- internal/federation/dereferencing/account.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index d87192d3a..02afd9a9c 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -265,18 +265,18 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { var processingMedia *media.ProcessingMedia + d.dereferencingAvatarsLock.Lock() // LOCK HERE // first check if we're already processing this media - d.dereferencingAvatarsLock.Lock() if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } - d.dereferencingAvatarsLock.Unlock() if processingMedia == nil { // we're not already processing it so start now avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) if err != nil { + d.dereferencingAvatarsLock.Unlock() return changed, err } @@ -290,16 +290,15 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm Avatar: &avatar, }) if err != nil { + d.dereferencingAvatarsLock.Unlock() return changed, err } // store it in our map to indicate it's in process - d.dereferencingAvatarsLock.Lock() d.dereferencingAvatars[targetAccount.ID] = newProcessing - d.dereferencingAvatarsLock.Unlock() - processingMedia = newProcessing } + d.dereferencingAvatarsLock.Unlock() // UNLOCK HERE // block until loaded if required... if blocking { @@ -324,18 +323,18 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { var processingMedia *media.ProcessingMedia + d.dereferencingHeadersLock.Lock() // LOCK HERE // first check if we're already processing this media - d.dereferencingHeadersLock.Lock() if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok { // we're already on it, no worries processingMedia = alreadyProcessing } - d.dereferencingHeadersLock.Unlock() if processingMedia == nil { // we're not already processing it so start now headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) if err != nil { + d.dereferencingAvatarsLock.Unlock() return changed, err } @@ -349,16 +348,15 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm Header: &header, }) if err != nil { + d.dereferencingAvatarsLock.Unlock() return changed, err } // store it in our map to indicate it's in process - d.dereferencingHeadersLock.Lock() d.dereferencingHeaders[targetAccount.ID] = newProcessing - d.dereferencingHeadersLock.Unlock() - processingMedia = newProcessing } + d.dereferencingHeadersLock.Unlock() // UNLOCK HERE // block until loaded if required... if blocking { -- cgit v1.3 From 8c0141d103cb70fdbe74f1d5a936860707da973f Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 13:38:44 +0100 Subject: store and retrieve processState atomically --- internal/media/processingemoji.go | 23 +++++++++++------------ internal/media/processingmedia.go | 37 ++++++++++++++++++++----------------- internal/media/types.go | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 292712427..741854b9b 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -25,6 +25,7 @@ import ( "io" "strings" "sync" + "sync/atomic" "time" "codeberg.org/gruf/go-store/kv" @@ -53,9 +54,7 @@ type ProcessingEmoji struct { /* below fields represent the processing state of the static of the emoji */ - - staticState processState - fullSizeState processState + staticState int32 /* below pointers to database and storage are maintained so that @@ -104,17 +103,18 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingEmoji) Finished() bool { - return p.staticState == complete && p.fullSizeState == complete + return atomic.LoadInt32(&p.staticState) == int32(complete) } func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { - switch p.staticState { + staticState := atomic.LoadInt32(&p.staticState) + switch processState(staticState) { case received: // stream the original file out of storage... stored, err := p.storage.GetStream(p.emoji.ImagePath) if err != nil { p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } @@ -122,27 +122,27 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType) if err != nil { p.err = fmt.Errorf("loadStatic: error deriving static: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadStatic: error closing stored full size: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } // put the static in storage if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { p.err = fmt.Errorf("loadStatic: error storing static: %s", err) - p.staticState = errored + atomic.StoreInt32(&p.staticState, int32(errored)) return p.err } p.emoji.ImageStaticFileSize = len(static.small) // we're done processing the static version of the emoji! - p.staticState = complete + atomic.StoreInt32(&p.staticState, int32(complete)) fallthrough case complete: return nil @@ -281,8 +281,7 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode instanceAccountID: instanceAccount.ID, emoji: emoji, data: data, - staticState: received, - fullSizeState: received, + staticState: int32(received), database: m.db, storage: m.storage, } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 0bbe35aee..0f47ee4e6 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -25,6 +25,7 @@ import ( "io" "strings" "sync" + "sync/atomic" "time" "codeberg.org/gruf/go-store/kv" @@ -49,8 +50,8 @@ type ProcessingMedia struct { data DataFunc read bool // bool indicating that data function has been triggered already - thumbstate processState // the processing state of the media thumbnail - fullSizeState processState // the processing state of the full-sized media + thumbState int32 // the processing state of the media thumbnail + fullSizeState int32 // the processing state of the full-sized media /* below pointers to database and storage are maintained so that @@ -103,11 +104,12 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt // Finished returns true if processing has finished for both the thumbnail // and full fized version of this piece of media. func (p *ProcessingMedia) Finished() bool { - return p.thumbstate == complete && p.fullSizeState == complete + return atomic.LoadInt32(&p.thumbState) == int32(complete) && atomic.LoadInt32(&p.fullSizeState) == int32(complete) } func (p *ProcessingMedia) loadThumb(ctx context.Context) error { - switch p.thumbstate { + thumbState := atomic.LoadInt32(&p.thumbState) + switch processState(thumbState) { case received: // we haven't processed a thumbnail for this media yet so do it now @@ -122,7 +124,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { stored, err := p.storage.GetStream(p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } @@ -130,20 +132,20 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash) if err != nil { p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadThumb: error closing stored full size: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } // put the thumbnail in storage if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err } @@ -160,7 +162,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { p.attachment.Thumbnail.FileSize = len(thumb.small) // we're done processing the thumbnail! - p.thumbstate = complete + atomic.StoreInt32(&p.thumbState, int32(complete)) fallthrough case complete: return nil @@ -168,11 +170,12 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { return p.err } - return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbstate) + return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbState) } func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { - switch p.fullSizeState { + fullSizeState := atomic.LoadInt32(&p.fullSizeState) + switch processState(fullSizeState) { case received: var err error var decoded *imageMeta @@ -181,7 +184,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { stored, err := p.storage.GetStream(p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) - p.fullSizeState = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } @@ -198,13 +201,13 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { if err != nil { p.err = err - p.fullSizeState = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } if err := stored.Close(); err != nil { p.err = fmt.Errorf("loadFullSize: error closing stored full size: %s", err) - p.thumbstate = errored + atomic.StoreInt32(&p.fullSizeState, int32(errored)) return p.err } @@ -219,7 +222,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { p.attachment.Processing = gtsmodel.ProcessingStatusProcessed // we're done processing the full-size image - p.fullSizeState = complete + atomic.StoreInt32(&p.fullSizeState, int32(complete)) fallthrough case complete: return nil @@ -400,8 +403,8 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID processingMedia := &ProcessingMedia{ attachment: attachment, data: data, - thumbstate: received, - fullSizeState: received, + thumbState: int32(received), + fullSizeState: int32(received), database: m.db, storage: m.storage, } diff --git a/internal/media/types.go b/internal/media/types.go index b9c79d464..a6b38b467 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -45,7 +45,7 @@ const ( mimeImagePng = mimeImage + "/" + mimePng ) -type processState int +type processState int32 const ( received processState = iota // processing order has been received but not done yet -- cgit v1.3 From a766b9265beab5f1a4f50ae7ce0fa325ea2d90ec Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Tue, 8 Feb 2022 13:39:47 +0100 Subject: Go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 85d10b2a5..a878363c5 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,6 @@ codeberg.org/gruf/go-pools v1.0.2/go.mod h1:MjUV3H6IASyBeBPCyCr7wjPpSNu8E2N87LG4 codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBsH4= codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= -codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo= -codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= codeberg.org/gruf/go-store v1.3.3 h1:fAP9FXy6HiLPxdD7cmpSzyfKXmVvZLjqn0m7HhxVT5M= codeberg.org/gruf/go-store v1.3.3/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -- cgit v1.3