[feature] Media attachment placeholders (#2331)

* [feature] Use placeholders for unknown media types

* fix read of underreported small files

* switch to reduce nesting

* simplify cleanup

5 files changed, 110 insertions, 42 deletions

diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go
index 81485e165..5dc93fa1d 100644
--- a/internal/processing/account/update.go
+++ b/internal/processing/account/update.go
@@ -33,6 +33,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 	"github.com/superseriousbusiness/gotosocial/internal/validate"
 )
 
@@ -281,57 +282,76 @@ func (p *Processor) Update(ctx context.Context, account *gtsmodel.Account, form
 	return acctSensitive, nil
 }
 
-// UpdateAvatar does the dirty work of checking the avatar part of an account update form,
-// parsing and checking the image, and doing the necessary updates in the database for this to become
-// the account's new avatar image.
-func (p *Processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHeader, description *string, accountID string) (*gtsmodel.MediaAttachment, error) {
+// UpdateAvatar does the dirty work of checking the avatar
+// part of an account update form, parsing and checking the
+// media, and doing the necessary updates in the database
+// for this to become the account's new avatar.
+func (p *Processor) UpdateAvatar(
+	ctx context.Context,
+	avatar *multipart.FileHeader,
+	description *string,
+	accountID string,
+) (*gtsmodel.MediaAttachment, error) {
 	maxImageSize := config.GetMediaImageMaxSize()
 	if avatar.Size > int64(maxImageSize) {
-		return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
+		return nil, gtserror.Newf("size %d exceeded max media size of %d bytes", avatar.Size, maxImageSize)
 	}
 
-	dataFunc := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
+	data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
 		f, err := avatar.Open()
 		return f, avatar.Size, err
 	}
 
-	isAvatar := true
-	ai := &media.AdditionalMediaInfo{
-		Avatar:      &isAvatar,
+	// Process the media attachment and load it immediately.
+	media := p.mediaManager.PreProcessMedia(data, accountID, &media.AdditionalMediaInfo{
+		Avatar:      util.Ptr(true),
 		Description: description,
-	}
+	})
 
-	processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, accountID, ai)
+	attachment, err := media.LoadAttachment(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	} else if attachment.Type == gtsmodel.FileTypeUnknown {
+		err = gtserror.Newf("could not process uploaded file with extension %s", attachment.File.ContentType)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
 	}
 
-	return processingMedia.LoadAttachment(ctx)
+	return attachment, nil
 }
 
-// UpdateHeader does the dirty work of checking the header part of an account update form,
-// parsing and checking the image, and doing the necessary updates in the database for this to become
-// the account's new header image.
-func (p *Processor) UpdateHeader(ctx context.Context, header *multipart.FileHeader, description *string, accountID string) (*gtsmodel.MediaAttachment, error) {
+// UpdateHeader does the dirty work of checking the header
+// part of an account update form, parsing and checking the
+// media, and doing the necessary updates in the database
+// for this to become the account's new header.
+func (p *Processor) UpdateHeader(
+	ctx context.Context,
+	header *multipart.FileHeader,
+	description *string,
+	accountID string,
+) (*gtsmodel.MediaAttachment, error) {
 	maxImageSize := config.GetMediaImageMaxSize()
 	if header.Size > int64(maxImageSize) {
-		return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
+		return nil, gtserror.Newf("size %d exceeded max media size of %d bytes", header.Size, maxImageSize)
 	}
 
-	dataFunc := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
+	data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
 		f, err := header.Open()
 		return f, header.Size, err
 	}
 
-	isHeader := true
-	ai := &media.AdditionalMediaInfo{
-		Header: &isHeader,
-	}
+	// Process the media attachment and load it immediately.
+	media := p.mediaManager.PreProcessMedia(data, accountID, &media.AdditionalMediaInfo{
+		Header:      util.Ptr(true),
+		Description: description,
+	})
 
-	processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, accountID, ai)
+	attachment, err := media.LoadAttachment(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	} else if attachment.Type == gtsmodel.FileTypeUnknown {
+		err = gtserror.Newf("could not process uploaded file with extension %s", attachment.File.ContentType)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
 	}
 
-	return processingMedia.LoadAttachment(ctx)
+	return attachment, nil
 }
diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go
index 568c4350b..689aad9dc 100644
--- a/internal/processing/admin/emoji.go
+++ b/internal/processing/admin/emoji.go
@@ -55,7 +55,7 @@ func (p *Processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
 		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new emoji: %s", err), "error creating emoji ID")
 	}
 
-	emojiURI := uris.GenerateURIForEmoji(emojiID)
+	emojiURI := uris.URIForEmoji(emojiID)
 
 	data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
 		f, err := form.Image.Open()
@@ -335,7 +335,7 @@ func (p *Processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji,
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	newEmojiURI := uris.GenerateURIForEmoji(newEmojiID)
+	newEmojiURI := uris.URIForEmoji(newEmojiID)
 
 	data := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) {
 		rc, err := p.state.Storage.GetStream(ctx, emoji.ImagePath)
diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go
index b8c469dde..fe20457b4 100644
--- a/internal/processing/media/create.go
+++ b/internal/processing/media/create.go
@@ -42,18 +42,18 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form
 	}
 
 	// process the media attachment and load it immediately
-	media, err := p.mediaManager.PreProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{
+	media := p.mediaManager.PreProcessMedia(data, account.ID, &media.AdditionalMediaInfo{
 		Description: &form.Description,
 		FocusX:      &focusX,
 		FocusY:      &focusY,
 	})
-	if err != nil {
-		return nil, gtserror.NewErrorUnprocessableEntity(err)
-	}
 
 	attachment, err := media.LoadAttachment(ctx)
 	if err != nil {
-		return nil, gtserror.NewErrorUnprocessableEntity(err)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
+	} else if attachment.Type == gtsmodel.FileTypeUnknown {
+		err = gtserror.Newf("could not process uploaded file with extension %s", attachment.File.ContentType)
+		return nil, gtserror.NewErrorUnprocessableEntity(err, err.Error())
 	}
 
 	apiAttachment, err := p.converter.AttachmentToAPIAttachment(ctx, attachment)
diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go
index 386c3a9a2..28f5e6464 100644
--- a/internal/processing/media/getfile.go
+++ b/internal/processing/media/getfile.go
@@ -23,17 +23,24 @@ import (
 	"io"
 	"net/url"
 	"strings"
+	"time"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
-// GetFile retrieves a file from storage and streams it back to the caller via an io.reader embedded in *apimodel.Content.
-func (p *Processor) GetFile(ctx context.Context, requestingAccount *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
+// GetFile retrieves a file from storage and streams it back
+// to the caller via an io.reader embedded in *apimodel.Content.
+func (p *Processor) GetFile(
+	ctx context.Context,
+	requestingAccount *gtsmodel.Account,
+	form *apimodel.GetContentRequestForm,
+) (*apimodel.Content, gtserror.WithCode) {
 	// parse the form fields
 	mediaSize, err := parseSize(form.MediaSize)
 	if err != nil {
@@ -118,11 +125,35 @@ func (p *Processor) getAttachmentContent(ctx context.Context, requestingAccount
 	// retrieve attachment from the database and do basic checks on it
 	a, err := p.state.DB.GetAttachmentByID(ctx, wantedMediaID)
 	if err != nil {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %w", wantedMediaID, err))
+		err = gtserror.Newf("attachment %s could not be taken from the db: %w", wantedMediaID, err)
+		return nil, gtserror.NewErrorNotFound(err)
 	}
 
 	if a.AccountID != owningAccountID {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, owningAccountID))
+		err = gtserror.Newf("attachment %s is not owned by %s", wantedMediaID, owningAccountID)
+		return nil, gtserror.NewErrorNotFound(err)
+	}
+
+	// If this is an "Unknown" file type, ie., one we
+	// tried to process and couldn't, or one we refused
+	// to process because it wasn't supported, then we
+	// can skip a lot of steps here by simply forwarding
+	// the request to the remote URL.
+	if a.Type == gtsmodel.FileTypeUnknown {
+		remoteURL, err := url.Parse(a.RemoteURL)
+		if err != nil {
+			err = gtserror.Newf("error parsing remote URL of 'Unknown'-type attachment for redirection: %w", err)
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		url := &storage.PresignedURL{
+			URL: remoteURL,
+			// We might manage to cache the media
+			// at some point, so set a low-ish expiry.
+			Expiry: time.Now().Add(2 * time.Hour),
+		}
+
+		return &apimodel.Content{URL: url}, nil
 	}
 
 	if !*a.Cached {
@@ -205,7 +236,7 @@ func (p *Processor) getEmojiContent(ctx context.Context, fileName string, owning
 	// for using the static URL rather than full size url
 	// is that static emojis are always encoded as png,
 	// so this is more reliable than using full size url
-	imageStaticURL := uris.GenerateURIForAttachment(
+	imageStaticURL := uris.URIForAttachment(
 		owningAccountID,
 		string(media.TypeEmoji),
 		string(media.SizeStatic),
diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go
index 8c939a61e..170dd0e53 100644
--- a/internal/processing/status/get.go
+++ b/internal/processing/status/get.go
@@ -36,8 +36,12 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account
 	return p.c.GetAPIStatus(ctx, requestingAccount, targetStatus)
 }
 
-// ContextGet returns the context (previous and following posts) from the given status ID.
-func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode) {
+func (p *Processor) contextGet(
+	ctx context.Context,
+	requestingAccount *gtsmodel.Account,
+	targetStatusID string,
+	convert func(context.Context, *gtsmodel.Status, *gtsmodel.Account) (*apimodel.Status, error),
+) (*apimodel.Context, gtserror.WithCode) {
 	targetStatus, errWithCode := p.c.GetVisibleTargetStatus(ctx, requestingAccount, targetStatusID)
 	if errWithCode != nil {
 		return nil, errWithCode
@@ -55,7 +59,7 @@ func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.
 
 	for _, status := range parents {
 		if v, err := p.filter.StatusVisible(ctx, requestingAccount, status); err == nil && v {
-			apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount)
+			apiStatus, err := convert(ctx, status, requestingAccount)
 			if err == nil {
 				context.Ancestors = append(context.Ancestors, *apiStatus)
 			}
@@ -73,7 +77,7 @@ func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.
 
 	for _, status := range children {
 		if v, err := p.filter.StatusVisible(ctx, requestingAccount, status); err == nil && v {
-			apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount)
+			apiStatus, err := convert(ctx, status, requestingAccount)
 			if err == nil {
 				context.Descendants = append(context.Descendants, *apiStatus)
 			}
@@ -82,3 +86,16 @@ func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.
 
 	return context, nil
 }
+
+// ContextGet returns the context (previous and following posts) from the given status ID.
+func (p *Processor) ContextGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode) {
+	return p.contextGet(ctx, requestingAccount, targetStatusID, p.converter.StatusToAPIStatus)
+}
+
+// WebContextGet is like ContextGet, but is explicitly
+// for viewing statuses via the unauthenticated web UI.
+//
+// TODO: a more advanced threading model could be implemented here.
+func (p *Processor) WebContextGet(ctx context.Context, targetStatusID string) (*apimodel.Context, gtserror.WithCode) {
+	return p.contextGet(ctx, nil, targetStatusID, p.converter.StatusToWebStatus)
+}