summaryrefslogtreecommitdiff
path: root/internal/federation
diff options
context:
space:
mode:
Diffstat (limited to 'internal/federation')
-rw-r--r--internal/federation/dereferencing/account.go14
-rw-r--r--internal/federation/dereferencing/media.go54
-rw-r--r--internal/federation/dereferencing/media_test.go161
-rw-r--r--internal/federation/dereferencing/status.go31
4 files changed, 21 insertions, 239 deletions
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
index 562062c8d..d551c3f0b 100644
--- a/internal/federation/dereferencing/account.go
+++ b/internal/federation/dereferencing/account.go
@@ -622,21 +622,16 @@ func (d *Dereferencer) fetchRemoteAccountAvatar(ctx context.Context, tsport tran
processing, ok := d.derefAvatars[latestAcc.AvatarRemoteURL]
if !ok {
- var err error
-
// Set the media data function to dereference avatar from URI.
data := func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, avatarURI)
}
// Create new media processing request from the media manager instance.
- processing, err = d.mediaManager.PreProcessMedia(ctx, data, latestAcc.ID, &media.AdditionalMediaInfo{
+ processing = d.mediaManager.PreProcessMedia(data, latestAcc.ID, &media.AdditionalMediaInfo{
Avatar: func() *bool { v := true; return &v }(),
RemoteURL: &latestAcc.AvatarRemoteURL,
})
- if err != nil {
- return gtserror.Newf("error preprocessing media for attachment %s: %w", latestAcc.AvatarRemoteURL, err)
- }
// Store media in map to mark as processing.
d.derefAvatars[latestAcc.AvatarRemoteURL] = processing
@@ -713,21 +708,16 @@ func (d *Dereferencer) fetchRemoteAccountHeader(ctx context.Context, tsport tran
processing, ok := d.derefHeaders[latestAcc.HeaderRemoteURL]
if !ok {
- var err error
-
// Set the media data function to dereference avatar from URI.
data := func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, headerURI)
}
// Create new media processing request from the media manager instance.
- processing, err = d.mediaManager.PreProcessMedia(ctx, data, latestAcc.ID, &media.AdditionalMediaInfo{
+ processing = d.mediaManager.PreProcessMedia(data, latestAcc.ID, &media.AdditionalMediaInfo{
Header: func() *bool { v := true; return &v }(),
RemoteURL: &latestAcc.HeaderRemoteURL,
})
- if err != nil {
- return gtserror.Newf("error preprocessing media for attachment %s: %w", latestAcc.HeaderRemoteURL, err)
- }
// Store media in map to mark as processing.
d.derefHeaders[latestAcc.HeaderRemoteURL] = processing
diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go
deleted file mode 100644
index 15aa4bb08..000000000
--- a/internal/federation/dereferencing/media.go
+++ /dev/null
@@ -1,54 +0,0 @@
-// GoToSocial
-// Copyright (C) GoToSocial Authors admin@gotosocial.org
-// SPDX-License-Identifier: AGPL-3.0-or-later
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package dereferencing
-
-import (
- "context"
- "fmt"
- "io"
- "net/url"
-
- "github.com/superseriousbusiness/gotosocial/internal/media"
-)
-
-func (d *Dereferencer) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) {
- if accountID == "" {
- return nil, fmt.Errorf("GetRemoteMedia: account ID was empty")
- }
-
- t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
- if err != nil {
- return nil, fmt.Errorf("GetRemoteMedia: error creating transport: %s", err)
- }
-
- derefURI, err := url.Parse(remoteURL)
- if err != nil {
- return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err)
- }
-
- dataFunc := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
- return t.DereferenceMedia(innerCtx, derefURI)
- }
-
- processingMedia, err := d.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai)
- if err != nil {
- return nil, fmt.Errorf("GetRemoteMedia: error processing attachment: %s", err)
- }
-
- return processingMedia, nil
-}
diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go
deleted file mode 100644
index 0de3f294f..000000000
--- a/internal/federation/dereferencing/media_test.go
+++ /dev/null
@@ -1,161 +0,0 @@
-// GoToSocial
-// Copyright (C) GoToSocial Authors admin@gotosocial.org
-// SPDX-License-Identifier: AGPL-3.0-or-later
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package dereferencing_test
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/stretchr/testify/suite"
- "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/media"
-)
-
-type AttachmentTestSuite struct {
- DereferencerStandardTestSuite
-}
-
-func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
- ctx := context.Background()
-
- fetchingAccount := suite.testAccounts["local_account_1"]
-
- attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
- attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8"
- attachmentContentType := "image/jpeg"
- attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
- attachmentDescription := "It's a cute plushie."
- attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
-
- media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
- StatusID: &attachmentStatus,
- RemoteURL: &attachmentURL,
- Description: &attachmentDescription,
- Blurhash: &attachmentBlurhash,
- })
- suite.NoError(err)
-
- // make a blocking call to load the attachment from the in-process media
- attachment, err := media.LoadAttachment(ctx)
- suite.NoError(err)
-
- suite.NotNil(attachment)
-
- suite.Equal(attachmentOwner, attachment.AccountID)
- suite.Equal(attachmentStatus, attachment.StatusID)
- suite.Equal(attachmentURL, attachment.RemoteURL)
- suite.NotEmpty(attachment.URL)
- suite.NotEmpty(attachment.Blurhash)
- suite.NotEmpty(attachment.ID)
- suite.NotEmpty(attachment.CreatedAt)
- suite.NotEmpty(attachment.UpdatedAt)
- suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
- suite.Equal(2071680, attachment.FileMeta.Original.Size)
- suite.Equal(1245, attachment.FileMeta.Original.Height)
- suite.Equal(1664, attachment.FileMeta.Original.Width)
- suite.Equal(attachmentBlurhash, attachment.Blurhash)
- suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
- suite.NotEmpty(attachment.File.Path)
- suite.Equal(attachmentContentType, attachment.File.ContentType)
- suite.Equal(attachmentDescription, attachment.Description)
-
- suite.NotEmpty(attachment.Thumbnail.Path)
- suite.NotEmpty(attachment.Type)
-
- // attachment should also now be in the database
- dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID)
- suite.NoError(err)
- suite.NotNil(dbAttachment)
-
- suite.Equal(attachmentOwner, dbAttachment.AccountID)
- suite.Equal(attachmentStatus, dbAttachment.StatusID)
- suite.Equal(attachmentURL, dbAttachment.RemoteURL)
- suite.NotEmpty(dbAttachment.URL)
- suite.NotEmpty(dbAttachment.Blurhash)
- suite.NotEmpty(dbAttachment.ID)
- suite.NotEmpty(dbAttachment.CreatedAt)
- suite.NotEmpty(dbAttachment.UpdatedAt)
- suite.EqualValues(1.3365462, dbAttachment.FileMeta.Original.Aspect)
- suite.Equal(2071680, dbAttachment.FileMeta.Original.Size)
- suite.Equal(1245, dbAttachment.FileMeta.Original.Height)
- suite.Equal(1664, dbAttachment.FileMeta.Original.Width)
- suite.Equal(attachmentBlurhash, dbAttachment.Blurhash)
- suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing)
- suite.NotEmpty(dbAttachment.File.Path)
- suite.Equal(attachmentContentType, dbAttachment.File.ContentType)
- suite.Equal(attachmentDescription, dbAttachment.Description)
-
- suite.NotEmpty(dbAttachment.Thumbnail.Path)
- suite.NotEmpty(dbAttachment.Type)
-}
-
-func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
- ctx := context.Background()
-
- fetchingAccount := suite.testAccounts["local_account_1"]
-
- attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
- attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8"
- attachmentContentType := "image/jpeg"
- attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
- attachmentDescription := "It's a cute plushie."
- attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
-
- processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
- StatusID: &attachmentStatus,
- RemoteURL: &attachmentURL,
- Description: &attachmentDescription,
- Blurhash: &attachmentBlurhash,
- })
- suite.NoError(err)
- attachmentID := processingMedia.AttachmentID()
-
- time.Sleep(time.Second * 3)
-
- // now get the attachment from the database
- attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
- suite.NoError(err)
-
- suite.NotNil(attachment)
-
- suite.Equal(attachmentOwner, attachment.AccountID)
- suite.Equal(attachmentStatus, attachment.StatusID)
- suite.Equal(attachmentURL, attachment.RemoteURL)
- suite.NotEmpty(attachment.URL)
- suite.NotEmpty(attachment.Blurhash)
- suite.NotEmpty(attachment.ID)
- suite.NotEmpty(attachment.CreatedAt)
- suite.NotEmpty(attachment.UpdatedAt)
- suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
- suite.Equal(2071680, attachment.FileMeta.Original.Size)
- suite.Equal(1245, attachment.FileMeta.Original.Height)
- suite.Equal(1664, attachment.FileMeta.Original.Width)
- suite.Equal(attachmentBlurhash, attachment.Blurhash)
- suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
- suite.NotEmpty(attachment.File.Path)
- suite.Equal(attachmentContentType, attachment.File.ContentType)
- suite.Equal(attachmentDescription, attachment.Description)
-
- suite.NotEmpty(attachment.Thumbnail.Path)
- suite.NotEmpty(attachment.Type)
-}
-
-func TestAttachmentTestSuite(t *testing.T) {
- suite.Run(t, new(AttachmentTestSuite))
-}
diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
index 1f09dedc0..032238bd5 100644
--- a/internal/federation/dereferencing/status.go
+++ b/internal/federation/dereferencing/status.go
@@ -789,7 +789,7 @@ func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transp
for i := range status.Attachments {
attachment := status.Attachments[i]
- // Look for existing media attachment with remoet URL first.
+ // Look for existing media attachment with remote URL first.
existing, ok := existing.GetAttachmentByRemoteURL(attachment.RemoteURL)
if ok && existing.ID != "" && *existing.Cached {
status.Attachments[i] = existing
@@ -804,25 +804,33 @@ func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transp
continue
}
- // Start pre-processing remote media at remote URL.
- processing, err := d.mediaManager.PreProcessMedia(ctx, func(ctx context.Context) (io.ReadCloser, int64, error) {
+ data := func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, remoteURL)
- }, status.AccountID, &media.AdditionalMediaInfo{
+ }
+
+ ai := &media.AdditionalMediaInfo{
StatusID: &status.ID,
RemoteURL: &attachment.RemoteURL,
Description: &attachment.Description,
Blurhash: &attachment.Blurhash,
- })
- if err != nil {
- log.Errorf(ctx, "error processing attachment: %v", err)
- continue
}
+ // Start pre-processing remote media at remote URL.
+ processing := d.mediaManager.PreProcessMedia(data, status.AccountID, ai)
+
// Force attachment loading *right now*.
attachment, err = processing.LoadAttachment(ctx)
if err != nil {
- log.Errorf(ctx, "error loading attachment: %v", err)
- continue
+ if attachment == nil {
+ // Totally failed to load;
+ // bail on this attachment.
+ log.Errorf(ctx, "error loading attachment: %v", err)
+ continue
+ }
+
+ // Partially loaded. Keep as
+ // placeholder and try again later.
+ log.Warnf(ctx, "partially loaded attachment: %v", err)
}
// Set the *new* attachment and ID.
@@ -832,8 +840,7 @@ func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transp
for i := 0; i < len(status.AttachmentIDs); {
if status.AttachmentIDs[i] == "" {
- // This is a failed attachment population, this may
- // be due to us not currently supporting a media type.
+ // Remove totally failed attachment populations
copy(status.Attachments[i:], status.Attachments[i+1:])
copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:])
status.Attachments = status.Attachments[:len(status.Attachments)-1]
generated by cgit v1.2.3 (git 2.46.0) at 2025-08-01 07:36:54 +0000