From f61c3ddcf72ff689b9d253546c58d499b6fe6ac8 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 8 Jan 2022 17:17:01 +0100 Subject: compiling now --- internal/federation/dereferencing/account.go | 48 +++++++--- internal/federation/dereferencing/attachment.go | 102 -------------------- .../federation/dereferencing/attachment_test.go | 106 --------------------- internal/federation/dereferencing/dereferencer.go | 29 +----- internal/federation/dereferencing/media.go | 55 +++++++++++ internal/federation/dereferencing/media_test.go | 102 ++++++++++++++++++++ internal/federation/dereferencing/status.go | 10 +- 7 files changed, 202 insertions(+), 250 deletions(-) delete mode 100644 internal/federation/dereferencing/attachment.go delete mode 100644 internal/federation/dereferencing/attachment_test.go create mode 100644 internal/federation/dereferencing/media.go create mode 100644 internal/federation/dereferencing/media_test.go (limited to 'internal/federation') diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 19c98e203..5912ff29a 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -246,25 +246,49 @@ func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount * } if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.AvatarRemoteURL, - Avatar: true, - }, targetAccount.ID) + avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL) if err != nil { - return fmt.Errorf("error processing avatar for user: %s", err) + return err } - targetAccount.AvatarMediaAttachmentID = a.ID + + data, err := t.DereferenceMedia(ctx, avatarIRI) + if err != nil { + return err + } + + media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.AvatarRemoteURL) + if err != nil { + return err + } + + if err := media.SetAsAvatar(ctx); err != nil { + return err + } + + targetAccount.AvatarMediaAttachmentID = media.AttachmentID() } if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := d.mediaManager.ProcessRemoteHeaderOrAvatar(ctx, t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.HeaderRemoteURL, - Header: true, - }, targetAccount.ID) + headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) if err != nil { - return fmt.Errorf("error processing header for user: %s", err) + return err } - targetAccount.HeaderMediaAttachmentID = a.ID + + data, err := t.DereferenceMedia(ctx, headerIRI) + if err != nil { + return err + } + + media, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, targetAccount.HeaderRemoteURL) + if err != nil { + return err + } + + if err := media.SetAsHeader(ctx); err != nil { + return err + } + + targetAccount.HeaderMediaAttachmentID = media.AttachmentID() } return nil } diff --git a/internal/federation/dereferencing/attachment.go b/internal/federation/dereferencing/attachment.go deleted file mode 100644 index 30ab6da10..000000000 --- a/internal/federation/dereferencing/attachment.go +++ /dev/null @@ -1,102 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package dereferencing - -import ( - "context" - "fmt" - "net/url" - - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - if minAttachment.RemoteURL == "" { - return nil, fmt.Errorf("GetRemoteAttachment: minAttachment remote URL was empty") - } - remoteAttachmentURL := minAttachment.RemoteURL - - l := logrus.WithFields(logrus.Fields{ - "username": requestingUsername, - "remoteAttachmentURL": remoteAttachmentURL, - }) - - // return early if we already have the attachment somewhere - maybeAttachment := >smodel.MediaAttachment{} - where := []db.Where{ - { - Key: "remote_url", - Value: remoteAttachmentURL, - }, - } - - if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil { - // we already the attachment in the database - l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID) - return maybeAttachment, nil - } - - a, err := d.RefreshAttachment(ctx, requestingUsername, minAttachment) - if err != nil { - return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err) - } - - if err := d.db.Put(ctx, a); err != nil { - if err != db.ErrAlreadyExists { - return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err) - } - } - - return a, nil -} - -func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) { - // it just doesn't exist or we have to refresh - if minAttachment.AccountID == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") - } - - if minAttachment.File.ContentType == "" { - return nil, fmt.Errorf("RefreshAttachment: minAttachment.file.contentType was empty") - } - - t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) - } - - derefURI, err := url.Parse(minAttachment.RemoteURL) - if err != nil { - return nil, err - } - - attachmentBytes, err := t.DereferenceMedia(ctx, derefURI, minAttachment.File.ContentType) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) - } - - a, err := d.mediaManager.ProcessAttachment(ctx, attachmentBytes, minAttachment) - if err != nil { - return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) - } - - return a, nil -} diff --git a/internal/federation/dereferencing/attachment_test.go b/internal/federation/dereferencing/attachment_test.go deleted file mode 100644 index d07cf1c6a..000000000 --- a/internal/federation/dereferencing/attachment_test.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package dereferencing_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type AttachmentTestSuite struct { - DereferencerStandardTestSuite -} - -func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { - fetchingAccount := suite.testAccounts["local_account_1"] - - attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" - attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8" - attachmentContentType := "image/jpeg" - attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" - attachmentDescription := "It's a cute plushie." - - minAttachment := >smodel.MediaAttachment{ - RemoteURL: attachmentURL, - AccountID: attachmentOwner, - StatusID: attachmentStatus, - File: gtsmodel.File{ - ContentType: attachmentContentType, - }, - Description: attachmentDescription, - } - - attachment, err := suite.dereferencer.GetRemoteAttachment(context.Background(), fetchingAccount.Username, minAttachment) - suite.NoError(err) - suite.NotNil(attachment) - - suite.Equal(attachmentOwner, attachment.AccountID) - suite.Equal(attachmentStatus, attachment.StatusID) - suite.Equal(attachmentURL, attachment.RemoteURL) - suite.NotEmpty(attachment.URL) - suite.NotEmpty(attachment.Blurhash) - suite.NotEmpty(attachment.ID) - suite.NotEmpty(attachment.CreatedAt) - suite.NotEmpty(attachment.UpdatedAt) - suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) - suite.Equal(2071680, attachment.FileMeta.Original.Size) - suite.Equal(1245, attachment.FileMeta.Original.Height) - suite.Equal(1664, attachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) - suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) - suite.NotEmpty(attachment.File.Path) - suite.Equal(attachmentContentType, attachment.File.ContentType) - suite.Equal(attachmentDescription, attachment.Description) - - suite.NotEmpty(attachment.Thumbnail.Path) - suite.NotEmpty(attachment.Type) - - // attachment should also now be in the database - dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID) - suite.NoError(err) - suite.NotNil(dbAttachment) - - suite.Equal(attachmentOwner, dbAttachment.AccountID) - suite.Equal(attachmentStatus, dbAttachment.StatusID) - suite.Equal(attachmentURL, dbAttachment.RemoteURL) - suite.NotEmpty(dbAttachment.URL) - suite.NotEmpty(dbAttachment.Blurhash) - suite.NotEmpty(dbAttachment.ID) - suite.NotEmpty(dbAttachment.CreatedAt) - suite.NotEmpty(dbAttachment.UpdatedAt) - suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect) - suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) - suite.Equal(1245, dbAttachment.FileMeta.Original.Height) - suite.Equal(1664, dbAttachment.FileMeta.Original.Width) - suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) - suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) - suite.NotEmpty(dbAttachment.File.Path) - suite.Equal(attachmentContentType, dbAttachment.File.ContentType) - suite.Equal(attachmentDescription, dbAttachment.Description) - - suite.NotEmpty(dbAttachment.Thumbnail.Path) - suite.NotEmpty(dbAttachment.Type) -} - -func TestAttachmentTestSuite(t *testing.T) { - suite.Run(t, new(AttachmentTestSuite)) -} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 4f977b8c8..d4786f62d 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,34 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - // GetRemoteAttachment takes a minimal attachment struct and converts it into a fully fleshed out attachment, stored in the database and instance storage. - // - // The parameter minAttachment must have at least the following fields defined: - // * minAttachment.RemoteURL - // * minAttachment.AccountID - // * minAttachment.File.ContentType - // - // The returned attachment will have an ID generated for it, so no need to generate one beforehand. - // A blurhash will also be generated for the attachment. - // - // Most other fields will be preserved on the passed attachment, including: - // * minAttachment.StatusID - // * minAttachment.CreatedAt - // * minAttachment.UpdatedAt - // * minAttachment.FileMeta - // * minAttachment.AccountID - // * minAttachment.Description - // * minAttachment.ScheduledStatusID - // * minAttachment.Thumbnail.RemoteURL - // * minAttachment.Avatar - // * minAttachment.Header - // - // GetRemoteAttachment will return early if an attachment with the same value as minAttachment.RemoteURL - // is found in the database -- then that attachment will be returned and nothing else will be changed or stored. - GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) - // RefreshAttachment is like GetRemoteAttachment, but the attachment will always be dereferenced again, - // whether or not it was already stored in the database. - RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) + GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go new file mode 100644 index 000000000..4d62fe0a6 --- /dev/null +++ b/internal/federation/dereferencing/media.go @@ -0,0 +1,55 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/media" +) + +func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string) (*media.Media, error) { + if accountID == "" { + return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty") + } + + t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err) + } + + derefURI, err := url.Parse(remoteURL) + if err != nil { + return nil, err + } + + data, err := t.DereferenceMedia(ctx, derefURI) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err) + } + + m, err := d.mediaManager.ProcessMedia(ctx, data, accountID, remoteURL) + if err != nil { + return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err) + } + + return m, nil +} diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go new file mode 100644 index 000000000..cc158c9a9 --- /dev/null +++ b/internal/federation/dereferencing/media_test.go @@ -0,0 +1,102 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type AttachmentTestSuite struct { + DereferencerStandardTestSuite +} + +func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() { + ctx := context.Background() + + fetchingAccount := suite.testAccounts["local_account_1"] + + attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM" + attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8" + attachmentContentType := "image/jpeg" + attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" + attachmentDescription := "It's a cute plushie." + + media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL) + suite.NoError(err) + + attachment, err := media.LoadAttachment(ctx) + suite.NoError(err) + + suite.NotNil(attachment) + + suite.Equal(attachmentOwner, attachment.AccountID) + suite.Equal(attachmentStatus, attachment.StatusID) + suite.Equal(attachmentURL, attachment.RemoteURL) + suite.NotEmpty(attachment.URL) + suite.NotEmpty(attachment.Blurhash) + suite.NotEmpty(attachment.ID) + suite.NotEmpty(attachment.CreatedAt) + suite.NotEmpty(attachment.UpdatedAt) + suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) + suite.Equal(2071680, attachment.FileMeta.Original.Size) + suite.Equal(1245, attachment.FileMeta.Original.Height) + suite.Equal(1664, attachment.FileMeta.Original.Width) + suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash) + suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing) + suite.NotEmpty(attachment.File.Path) + suite.Equal(attachmentContentType, attachment.File.ContentType) + suite.Equal(attachmentDescription, attachment.Description) + + suite.NotEmpty(attachment.Thumbnail.Path) + suite.NotEmpty(attachment.Type) + + // attachment should also now be in the database + dbAttachment, err := suite.db.GetAttachmentByID(context.Background(), attachment.ID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + suite.Equal(attachmentOwner, dbAttachment.AccountID) + suite.Equal(attachmentStatus, dbAttachment.StatusID) + suite.Equal(attachmentURL, dbAttachment.RemoteURL) + suite.NotEmpty(dbAttachment.URL) + suite.NotEmpty(dbAttachment.Blurhash) + suite.NotEmpty(dbAttachment.ID) + suite.NotEmpty(dbAttachment.CreatedAt) + suite.NotEmpty(dbAttachment.UpdatedAt) + suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect) + suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) + suite.Equal(1245, dbAttachment.FileMeta.Original.Height) + suite.Equal(1664, dbAttachment.FileMeta.Original.Width) + suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash) + suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing) + suite.NotEmpty(dbAttachment.File.Path) + suite.Equal(attachmentContentType, dbAttachment.File.ContentType) + suite.Equal(attachmentDescription, dbAttachment.Description) + + suite.NotEmpty(dbAttachment.Thumbnail.Path) + suite.NotEmpty(dbAttachment.Type) +} + +func TestAttachmentTestSuite(t *testing.T) { + suite.Run(t, new(AttachmentTestSuite)) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index d7de5936a..e184b585f 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -393,9 +393,15 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. a.AccountID = status.AccountID a.StatusID = status.ID - attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, a) + media, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL) if err != nil { - logrus.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err) + logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err) + continue + } + + attachment, err := media.LoadAttachment(ctx) + if err != nil { + logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err) continue } -- cgit v1.2.3