diff options
| author | 2022-10-13 15:16:24 +0200 | |
|---|---|---|
| committer | 2022-10-13 15:16:24 +0200 | |
| commit | 70d65b683fa963d2a8761182a2ddd2f4f9a86bb4 (patch) | |
| tree | 9cbd8f6870569b2514683c0e8ff6ea32e6e81780 | |
| parent | [frontend] Use new GET custom_emoji admin api (#908) (diff) | |
| download | gotosocial-70d65b683fa963d2a8761182a2ddd2f4f9a86bb4.tar.xz | |
[feature] Refetch emojis when they change on remote instances (#905)
* select emoji using image_static_url
* use updated on AP emojis
* allow refetch of updated emojis
* cheeky workaround for test
* clean up old files for refreshed emoji
* check error for originalPostData
* shorten GetEmojiByStaticImageURL
* delete kirby (sorry nintendo)
22 files changed, 414 insertions, 75 deletions
diff --git a/internal/ap/extract.go b/internal/ap/extract.go index 8bd8aa3f4..a7a46e51c 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -535,6 +535,11 @@ func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {  	emoji.Disabled = new(bool)  	emoji.VisibleInPicker = new(bool) +	updatedProp := i.GetActivityStreamsUpdated() +	if updatedProp != nil && updatedProp.IsXMLSchemaDateTime() { +		emoji.UpdatedAt = updatedProp.Get() +	} +  	return emoji, nil  } diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go index a408b3773..364ad336f 100644 --- a/internal/api/s2s/user/inboxpost_test.go +++ b/internal/api/s2s/user/inboxpost_test.go @@ -239,7 +239,13 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {  func (suite *InboxPostTestSuite) TestPostUpdate() {  	updatedAccount := *suite.testAccounts["remote_account_1"]  	updatedAccount.DisplayName = "updated display name!" -	testEmoji := testrig.NewTestEmojis()["rainbow"] + +	// ad an emoji to the account; because we're serializing this remote +	// account from our own instance, we need to cheat a bit to get the emoji +	// to work properly, just for this test +	testEmoji := >smodel.Emoji{} +	*testEmoji = *testrig.NewTestEmojis()["yell"] +	testEmoji.ImageURL = testEmoji.ImageRemoteURL // <- here's the cheat  	updatedAccount.Emojis = []*gtsmodel.Emoji{testEmoji}  	asAccount, err := suite.tc.AccountToAS(context.Background(), &updatedAccount) diff --git a/internal/cache/emoji.go b/internal/cache/emoji.go index eda7583ea..117f5475e 100644 --- a/internal/cache/emoji.go +++ b/internal/cache/emoji.go @@ -37,19 +37,26 @@ func NewEmojiCache() *EmojiCache {  		RegisterLookups: func(lm *cache.LookupMap[string, string]) {  			lm.RegisterLookup("uri")  			lm.RegisterLookup("shortcodedomain") +			lm.RegisterLookup("imagestaticurl")  		},  		AddLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) { +			lm.Set("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain), emoji.ID)  			if uri := emoji.URI; uri != "" { -				lm.Set("uri", uri, emoji.URI) -				lm.Set("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain), emoji.ID) +				lm.Set("uri", uri, emoji.ID) +			} +			if imageStaticURL := emoji.ImageStaticURL; imageStaticURL != "" { +				lm.Set("imagestaticurl", imageStaticURL, emoji.ID)  			}  		},  		DeleteLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) { +			lm.Delete("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain))  			if uri := emoji.URI; uri != "" {  				lm.Delete("uri", uri) -				lm.Delete("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain)) +			} +			if imageStaticURL := emoji.ImageStaticURL; imageStaticURL != "" { +				lm.Delete("imagestaticurl", imageStaticURL)  			}  		},  	}) @@ -72,6 +79,10 @@ func (c *EmojiCache) GetByShortcodeDomain(shortcode string, domain string) (*gts  	return c.cache.GetBy("shortcodedomain", shortcodeDomainKey(shortcode, domain))  } +func (c *EmojiCache) GetByImageStaticURL(imageStaticURL string) (*gtsmodel.Emoji, bool) { +	return c.cache.GetBy("imagestaticurl", imageStaticURL) +} +  // Put places an emoji in the cache, ensuring that the object place is a copy for thread-safety  func (c *EmojiCache) Put(emoji *gtsmodel.Emoji) {  	if emoji == nil || emoji.ID == "" { @@ -80,6 +91,10 @@ func (c *EmojiCache) Put(emoji *gtsmodel.Emoji) {  	c.cache.Set(emoji.ID, copyEmoji(emoji))  } +func (c *EmojiCache) Invalidate(emojiID string) { +	c.cache.Invalidate(emojiID) +} +  // copyEmoji performs a surface-level copy of emoji, only keeping attached IDs intact, not the objects.  // due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr)  // this should be a relatively cheap process diff --git a/internal/db/bundb/emoji.go b/internal/db/bundb/emoji.go index 640e354c4..4fb4f0ce6 100644 --- a/internal/db/bundb/emoji.go +++ b/internal/db/bundb/emoji.go @@ -21,6 +21,7 @@ package bundb  import (  	"context"  	"strings" +	"time"  	"github.com/superseriousbusiness/gotosocial/internal/cache"  	"github.com/superseriousbusiness/gotosocial/internal/db" @@ -50,6 +51,23 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error  	return nil  } +func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) { +	// Update the emoji's last-updated +	emoji.UpdatedAt = time.Now() + +	if _, err := e.conn. +		NewUpdate(). +		Model(emoji). +		Where("? = ?", bun.Ident("emoji.id"), emoji.ID). +		Column(columns...). +		Exec(ctx); err != nil { +		return nil, e.conn.ProcessError(err) +	} + +	e.cache.Invalidate(emoji.ID) +	return emoji, nil +} +  func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) {  	emojiIDs := []string{} @@ -232,6 +250,21 @@ func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode strin  	)  } +func (e *emojiDB) GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, db.Error) { +	return e.getEmoji( +		ctx, +		func() (*gtsmodel.Emoji, bool) { +			return e.cache.GetByImageStaticURL(imageStaticURL) +		}, +		func(emoji *gtsmodel.Emoji) error { +			return e. +				newEmojiQ(emoji). +				Where("? = ?", bun.Ident("emoji.image_static_url"), imageStaticURL). +				Scan(ctx) +		}, +	) +} +  func (e *emojiDB) getEmoji(ctx context.Context, cacheGet func() (*gtsmodel.Emoji, bool), dbQuery func(*gtsmodel.Emoji) error) (*gtsmodel.Emoji, db.Error) {  	// Attempt to fetch cached emoji  	emoji, cached := cacheGet() diff --git a/internal/db/bundb/emoji_test.go b/internal/db/bundb/emoji_test.go index 3c61fb620..c6577a721 100644 --- a/internal/db/bundb/emoji_test.go +++ b/internal/db/bundb/emoji_test.go @@ -38,6 +38,13 @@ func (suite *EmojiTestSuite) TestGetUseableEmojis() {  	suite.Equal("rainbow", emojis[0].Shortcode)  } +func (suite *EmojiTestSuite) TestGetEmojiByStaticURL() { +	emoji, err := suite.db.GetEmojiByStaticURL(context.Background(), "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png") +	suite.NoError(err) +	suite.NotNil(emoji) +	suite.Equal("rainbow", emoji.Shortcode) +} +  func (suite *EmojiTestSuite) TestGetAllEmojis() {  	emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0) diff --git a/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go b/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go new file mode 100644 index 000000000..28ba41a3a --- /dev/null +++ b/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go @@ -0,0 +1,48 @@ +/* +   GoToSocial +   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + +   This program is free software: you can redistribute it and/or modify +   it under the terms of the GNU Affero General Public License as published by +   the Free Software Foundation, either version 3 of the License, or +   (at your option) any later version. + +   This program is distributed in the hope that it will be useful, +   but WITHOUT ANY WARRANTY; without even the implied warranty of +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +   GNU Affero General Public License for more details. + +   You should have received a copy of the GNU Affero General Public License +   along with this program.  If not, see <http://www.gnu.org/licenses/>. +*/ + +package migrations + +import ( +	"context" + +	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +	"github.com/uptrace/bun" +) + +func init() { +	up := func(ctx context.Context, db *bun.DB) error { +		_, err := db. +			NewCreateIndex(). +			Model(>smodel.Emoji{}). +			Index("emojis_image_static_url_idx"). +			Column("image_static_url"). +			Exec(ctx) +		return err +	} + +	down := func(ctx context.Context, db *bun.DB) error { +		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { +			return nil +		}) +	} + +	if err := Migrations.Register(up, down); err != nil { +		panic(err) +	} +} diff --git a/internal/db/emoji.go b/internal/db/emoji.go index 4316a43ef..831629232 100644 --- a/internal/db/emoji.go +++ b/internal/db/emoji.go @@ -32,6 +32,9 @@ const EmojiAllDomains string = "all"  type Emoji interface {  	// PutEmoji puts one emoji in the database.  	PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) Error +	// UpdateEmoji updates the given columns of one emoji. +	// If no columns are specified, every column is updated. +	UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, Error)  	// GetUseableEmojis gets all emojis which are useable by accounts on this instance.  	GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, Error)  	// GetEmojis gets emojis based on given parameters. Useful for admin actions. @@ -43,4 +46,6 @@ type Emoji interface {  	GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error)  	// GetEmojiByURI returns one emoji based on its ActivityPub URI.  	GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, Error) +	// GetEmojiByStaticURL gets an emoji using the URL of the static version of the emoji image. +	GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, Error)  } diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index aec612ac8..ddd9456e8 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -224,6 +224,7 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() {  				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",  				Shortcode:       "kip_van_den_bos",  				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), +				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),  				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",  				Disabled:        testrig.FalseBool(),  				VisibleInPicker: testrig.FalseBool(), @@ -275,10 +276,12 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() {  			{  				URI:             knownEmoji.URI,  				Shortcode:       knownEmoji.Shortcode, -				UpdatedAt:       knownEmoji.CreatedAt, +				UpdatedAt:       knownEmoji.UpdatedAt, +				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,  				ImageRemoteURL:  knownEmoji.ImageRemoteURL,  				Disabled:        knownEmoji.Disabled,  				VisibleInPicker: knownEmoji.VisibleInPicker, +				Domain:          knownEmoji.Domain,  			},  		},  	} @@ -326,10 +329,12 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {  			{  				URI:             knownEmoji.URI,  				Shortcode:       knownEmoji.Shortcode, -				UpdatedAt:       knownEmoji.CreatedAt, +				UpdatedAt:       knownEmoji.UpdatedAt, +				ImageUpdatedAt:  knownEmoji.ImageUpdatedAt,  				ImageRemoteURL:  knownEmoji.ImageRemoteURL,  				Disabled:        knownEmoji.Disabled,  				VisibleInPicker: knownEmoji.VisibleInPicker, +				Domain:          knownEmoji.Domain,  			},  		},  	} @@ -372,6 +377,7 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {  				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",  				Shortcode:       "kip_van_den_bos",  				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), +				ImageUpdatedAt:  testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),  				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",  				Disabled:        testrig.FalseBool(),  				VisibleInPicker: testrig.FalseBool(), diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 331df3215..a6cb9b15f 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface {  	GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)  	GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) -	GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) +	GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error)  	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error  	DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go index 622b131c9..3cdb1d52d 100644 --- a/internal/federation/dereferencing/emoji.go +++ b/internal/federation/dereferencing/emoji.go @@ -31,7 +31,7 @@ import (  	"github.com/superseriousbusiness/gotosocial/internal/media"  ) -func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) { +func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) {  	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)  	if err != nil {  		return nil, fmt.Errorf("GetRemoteEmoji: error creating transport: %s", err) @@ -46,7 +46,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r  		return t.DereferenceMedia(innerCtx, derefURI)  	} -	processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai) +	processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh)  	if err != nil {  		return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji: %s", err)  	} @@ -69,12 +69,34 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,  		var err error  		// check if we've already got this emoji in the db -		if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries { +		if gotEmoji, err = d.db.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {  			log.Errorf("populateEmojis: error checking database for emoji %s: %s", e.URI, err)  			continue  		} -		if gotEmoji == nil { +		if gotEmoji != nil { +			// we had the emoji in our database already; make sure the one we have is up to date +			if (e.UpdatedAt.After(gotEmoji.ImageUpdatedAt)) || (e.URI != gotEmoji.URI) || (e.ImageRemoteURL != gotEmoji.ImageRemoteURL) { +				emojiID := gotEmoji.ID // use existing ID +				processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, emojiID, e.URI, &media.AdditionalEmojiInfo{ +					Domain:               &e.Domain, +					ImageRemoteURL:       &e.ImageRemoteURL, +					ImageStaticRemoteURL: &e.ImageRemoteURL, +					Disabled:             gotEmoji.Disabled, +					VisibleInPicker:      gotEmoji.VisibleInPicker, +				}, true) + +				if err != nil { +					log.Errorf("populateEmojis: couldn't refresh remote emoji %s: %s", e.URI, err) +					continue +				} + +				if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil { +					log.Errorf("populateEmojis: couldn't load refreshed remote emoji %s: %s", e.URI, err) +					continue +				} +			} +		} else {  			// it's new! go get it!  			newEmojiID, err := id.NewRandomULID()  			if err != nil { @@ -88,7 +110,7 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,  				ImageStaticRemoteURL: &e.ImageRemoteURL,  				Disabled:             e.Disabled,  				VisibleInPicker:      e.VisibleInPicker, -			}) +			}, false)  			if err != nil {  				log.Errorf("populateEmojis: couldn't get remote emoji %s: %s", e.URI, err) diff --git a/internal/federation/dereferencing/emoji_test.go b/internal/federation/dereferencing/emoji_test.go index b03d839ce..3093a1e7f 100644 --- a/internal/federation/dereferencing/emoji_test.go +++ b/internal/federation/dereferencing/emoji_test.go @@ -51,7 +51,7 @@ func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() {  		VisibleInPicker:      &emojiVisibleInPicker,  	} -	processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai) +	processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai, false)  	suite.NoError(err)  	// make a blocking call to load the emoji from the in-process media diff --git a/internal/media/manager.go b/internal/media/manager.go index 828aa033b..62998156e 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -69,7 +69,9 @@ type Manager interface {  	// uri is the ActivityPub URI/ID of the emoji.  	//  	// ai is optional and can be nil. Any additional information about the emoji provided will be put in the database. -	ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) +	// +	// If refresh is true, this indicates that the emoji image has changed and should be updated. +	ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error)  	// RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote.  	RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) @@ -164,8 +166,8 @@ func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, postData Post  	return processingMedia, nil  } -func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { -	processingEmoji, err := m.preProcessEmoji(ctx, data, postData, shortcode, id, uri, ai) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { +	processingEmoji, err := m.preProcessEmoji(ctx, data, postData, shortcode, id, uri, ai, refresh)  	if err != nil {  		return nil, err  	} diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index f9c96259d..e00cdd98d 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -55,7 +55,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() {  	emojiID := "01GDQ9G782X42BAMFASKP64343"  	emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" -	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil) +	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil, false)  	suite.NoError(err)  	// do a blocking call to fetch the emoji @@ -101,6 +101,99 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() {  	suite.Equal(processedStaticBytesExpected, processedStaticBytes)  } +func (suite *ManagerTestSuite) TestEmojiProcessBlockingRefresh() { +	ctx := context.Background() + +	// we're going to 'refresh' the remote 'yell' emoji by changing the image url to the pixellated gts logo +	originalEmoji := suite.testEmojis["yell"] + +	emojiToUpdate := >smodel.Emoji{} +	*emojiToUpdate = *originalEmoji +	newImageRemoteURL := "http://fossbros-anonymous.io/some/image/path.png" + +	oldEmojiImagePath := emojiToUpdate.ImagePath +	oldEmojiImageStaticPath := emojiToUpdate.ImageStaticPath + +	data := func(_ context.Context) (io.Reader, int64, error) { +		b, err := os.ReadFile("./test/gts_pixellated-original.png") +		if err != nil { +			panic(err) +		} +		return bytes.NewBuffer(b), int64(len(b)), nil +	} + +	emojiID := emojiToUpdate.ID +	emojiURI := emojiToUpdate.URI + +	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "yell", emojiID, emojiURI, &media.AdditionalEmojiInfo{ +		CreatedAt:      &emojiToUpdate.CreatedAt, +		Domain:         &emojiToUpdate.Domain, +		ImageRemoteURL: &newImageRemoteURL, +	}, true) +	suite.NoError(err) + +	// do a blocking call to fetch the emoji +	emoji, err := processingEmoji.LoadEmoji(ctx) +	suite.NoError(err) +	suite.NotNil(emoji) + +	// make sure it's got the stuff set on it that we expect +	suite.Equal(emojiID, emoji.ID) + +	// file meta should be correctly derived from the image +	suite.Equal("image/png", emoji.ImageContentType) +	suite.Equal("image/png", emoji.ImageStaticContentType) +	suite.Equal(10296, emoji.ImageFileSize) + +	// now make sure the emoji is in the database +	dbEmoji, err := suite.db.GetEmojiByID(ctx, emojiID) +	suite.NoError(err) +	suite.NotNil(dbEmoji) + +	// make sure the processed emoji file is in storage +	processedFullBytes, err := suite.storage.Get(ctx, emoji.ImagePath) +	suite.NoError(err) +	suite.NotEmpty(processedFullBytes) + +	// load the processed bytes from our test folder, to compare +	processedFullBytesExpected, err := os.ReadFile("./test/gts_pixellated-original.png") +	suite.NoError(err) +	suite.NotEmpty(processedFullBytesExpected) + +	// the bytes in storage should be what we expected +	suite.Equal(processedFullBytesExpected, processedFullBytes) + +	// now do the same for the thumbnail and make sure it's what we expected +	processedStaticBytes, err := suite.storage.Get(ctx, emoji.ImageStaticPath) +	suite.NoError(err) +	suite.NotEmpty(processedStaticBytes) + +	processedStaticBytesExpected, err := os.ReadFile("./test/gts_pixellated-static.png") +	suite.NoError(err) +	suite.NotEmpty(processedStaticBytesExpected) + +	suite.Equal(processedStaticBytesExpected, processedStaticBytes) + +	// most fields should be different on the emoji now from what they were before +	suite.Equal(originalEmoji.ID, dbEmoji.ID) +	suite.NotEqual(originalEmoji.ImageRemoteURL, dbEmoji.ImageRemoteURL) +	suite.NotEqual(originalEmoji.ImageURL, dbEmoji.ImageURL) +	suite.NotEqual(originalEmoji.ImageStaticURL, dbEmoji.ImageStaticURL) +	suite.NotEqual(originalEmoji.ImageFileSize, dbEmoji.ImageFileSize) +	suite.NotEqual(originalEmoji.ImageStaticFileSize, dbEmoji.ImageStaticFileSize) +	suite.NotEqual(originalEmoji.ImagePath, dbEmoji.ImagePath) +	suite.NotEqual(originalEmoji.ImageStaticPath, dbEmoji.ImageStaticPath) +	suite.NotEqual(originalEmoji.ImageStaticPath, dbEmoji.ImageStaticPath) +	suite.NotEqual(originalEmoji.UpdatedAt, dbEmoji.UpdatedAt) +	suite.NotEqual(originalEmoji.ImageUpdatedAt, dbEmoji.ImageUpdatedAt) + +	// the old image files should no longer be in storage +	_, err = suite.storage.Get(ctx, oldEmojiImagePath) +	suite.ErrorIs(err, storage.ErrNotFound) +	_, err = suite.storage.Get(ctx, oldEmojiImageStaticPath) +	suite.ErrorIs(err, storage.ErrNotFound) +} +  func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {  	ctx := context.Background() @@ -116,7 +209,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {  	emojiID := "01GDQ9G782X42BAMFASKP64343"  	emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" -	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil) +	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil, false)  	suite.NoError(err)  	// do a blocking call to fetch the emoji @@ -140,7 +233,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() {  	emojiID := "01GDQ9G782X42BAMFASKP64343"  	emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" -	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil) +	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil, false)  	suite.NoError(err)  	// do a blocking call to fetch the emoji @@ -165,7 +258,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() {  	emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343"  	// process the media with no additional info provided -	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil) +	processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil, false)  	suite.NoError(err)  	// do a blocking call to fetch the emoji diff --git a/internal/media/media_test.go b/internal/media/media_test.go index fda1963a7..e2c3914a3 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -35,6 +35,7 @@ type MediaStandardTestSuite struct {  	manager         media.Manager  	testAttachments map[string]*gtsmodel.MediaAttachment  	testAccounts    map[string]*gtsmodel.Account +	testEmojis      map[string]*gtsmodel.Emoji  }  func (suite *MediaStandardTestSuite) SetupSuite() { @@ -50,6 +51,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {  	testrig.StandardDBSetup(suite.db, nil)  	suite.testAttachments = testrig.NewTestAttachments()  	suite.testAccounts = testrig.NewTestAccounts() +	suite.testEmojis = testrig.NewTestEmojis()  	suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)  } diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 6495b991e..e1c6f2efb 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -21,6 +21,7 @@ package media  import (  	"bytes"  	"context" +	"errors"  	"fmt"  	"io"  	"strings" @@ -28,9 +29,11 @@ import (  	"sync/atomic"  	"time" +	gostore "codeberg.org/gruf/go-store/storage"  	"github.com/superseriousbusiness/gotosocial/internal/config"  	"github.com/superseriousbusiness/gotosocial/internal/db"  	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +	"github.com/superseriousbusiness/gotosocial/internal/id"  	"github.com/superseriousbusiness/gotosocial/internal/log"  	"github.com/superseriousbusiness/gotosocial/internal/storage"  	"github.com/superseriousbusiness/gotosocial/internal/uris" @@ -71,6 +74,11 @@ type ProcessingEmoji struct {  	// track whether this emoji has already been put in the databse  	insertedInDB bool + +	// is this a refresh of an existing emoji? +	refresh bool +	// if it is a refresh, which alternate ID should we use in the storage and URL paths? +	newPathID string  }  // EmojiID returns the ID of the underlying emoji without blocking processing. @@ -94,8 +102,28 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error  	// store the result in the database before returning it  	if !p.insertedInDB { -		if err := p.database.PutEmoji(ctx, p.emoji); err != nil { -			return nil, err +		if p.refresh { +			columns := []string{ +				"updated_at", +				"image_remote_url", +				"image_static_remote_url", +				"image_url", +				"image_static_url", +				"image_path", +				"image_static_path", +				"image_content_type", +				"image_file_size", +				"image_static_file_size", +				"image_updated_at", +				"uri", +			} +			if _, err := p.database.UpdateEmoji(ctx, p.emoji, columns...); err != nil { +				return nil, err +			} +		} else { +			if err := p.database.PutEmoji(ctx, p.emoji); err != nil { +				return nil, err +			}  		}  		p.insertedInDB = true  	} @@ -203,8 +231,14 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {  	// set some additional fields on the emoji now that  	// we know more about what the underlying image actually is -	p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) -	p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) +	var pathID string +	if p.refresh { +		pathID = p.newPathID +	} else { +		pathID = p.emoji.ID +	} +	p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), pathID, extension) +	p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, pathID, extension)  	p.emoji.ImageContentType = contentType  	// concatenate the first bytes with the existing bytes still in the reader (thanks Mara) @@ -251,39 +285,87 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {  	return nil  } -func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, emojiID string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) {  	instanceAccount, err := m.db.GetInstanceAccount(ctx, "")  	if err != nil {  		return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err)  	} -	disabled := false -	visibleInPicker := true - -	// populate initial fields on the emoji -- some of these will be overwritten as we proceed -	emoji := >smodel.Emoji{ -		ID:                     id, -		CreatedAt:              time.Now(), -		UpdatedAt:              time.Now(), -		Shortcode:              shortcode, -		Domain:                 "", // assume our own domain unless told otherwise -		ImageRemoteURL:         "", -		ImageStaticRemoteURL:   "", -		ImageURL:               "",                                                                                                    // we don't know yet -		ImageStaticURL:         uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), id, mimePng), // all static emojis are encoded as png -		ImagePath:              "",                                                                                                    // we don't know yet -		ImageStaticPath:        fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, id, mimePng),                 // all static emojis are encoded as png -		ImageContentType:       "",                                                                                                    // we don't know yet -		ImageStaticContentType: mimeImagePng,                                                                                          // all static emojis are encoded as png -		ImageFileSize:          0, -		ImageStaticFileSize:    0, -		ImageUpdatedAt:         time.Now(), -		Disabled:               &disabled, -		URI:                    uri, -		VisibleInPicker:        &visibleInPicker, -		CategoryID:             "", +	var newPathID string +	var emoji *gtsmodel.Emoji +	if refresh { +		emoji, err = m.db.GetEmojiByID(ctx, emojiID) +		if err != nil { +			return nil, fmt.Errorf("preProcessEmoji: error fetching emoji to refresh from the db: %s", err) +		} + +		// if this is a refresh, we will end up with new images +		// stored for this emoji, so we can use the postData function +		// to perform clean up of the old images from storage +		originalPostData := postData +		originalImagePath := emoji.ImagePath +		originalImageStaticPath := emoji.ImageStaticPath +		postData = func(ctx context.Context) error { +			// trigger the original postData function if it was provided +			if originalPostData != nil { +				if err := originalPostData(ctx); err != nil { +					return err +				} +			} + +			l := log.WithField("shortcode@domain", emoji.Shortcode+"@"+emoji.Domain) +			l.Debug("postData: cleaning up old emoji files for refreshed emoji") +			if err := m.storage.Delete(ctx, originalImagePath); err != nil && !errors.Is(err, gostore.ErrNotFound) { +				l.Errorf("postData: error cleaning up old emoji image at %s for refreshed emoji: %s", originalImagePath, err) +			} +			if err := m.storage.Delete(ctx, originalImageStaticPath); err != nil && !errors.Is(err, gostore.ErrNotFound) { +				l.Errorf("postData: error cleaning up old emoji static image at %s for refreshed emoji: %s", originalImageStaticPath, err) +			} + +			return nil +		} + +		newPathID, err = id.NewRandomULID() +		if err != nil { +			return nil, fmt.Errorf("preProcessEmoji: error generating alternateID for emoji refresh: %s", err) +		} + +		// store + serve static image at new path ID +		emoji.ImageStaticURL = uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newPathID, mimePng) +		emoji.ImageStaticPath = fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, newPathID, mimePng) + +		// update these fields as we go +		emoji.URI = uri +	} else { +		disabled := false +		visibleInPicker := true + +		// populate initial fields on the emoji -- some of these will be overwritten as we proceed +		emoji = >smodel.Emoji{ +			ID:                     emojiID, +			CreatedAt:              time.Now(), +			Shortcode:              shortcode, +			Domain:                 "", // assume our own domain unless told otherwise +			ImageRemoteURL:         "", +			ImageStaticRemoteURL:   "", +			ImageURL:               "",                                                                                                         // we don't know yet +			ImageStaticURL:         uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), emojiID, mimePng), // all static emojis are encoded as png +			ImagePath:              "",                                                                                                         // we don't know yet +			ImageStaticPath:        fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, emojiID, mimePng),                 // all static emojis are encoded as png +			ImageContentType:       "",                                                                                                         // we don't know yet +			ImageStaticContentType: mimeImagePng,                                                                                               // all static emojis are encoded as png +			ImageFileSize:          0, +			ImageStaticFileSize:    0, +			Disabled:               &disabled, +			URI:                    uri, +			VisibleInPicker:        &visibleInPicker, +			CategoryID:             "", +		}  	} +	emoji.ImageUpdatedAt = time.Now() +	emoji.UpdatedAt = time.Now() +  	// check if we have additional info to add to the emoji,  	// and overwrite some of the emoji fields if so  	if ai != nil { @@ -324,6 +406,8 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData P  		staticState:       int32(received),  		database:          m.db,  		storage:           m.storage, +		refresh:           refresh, +		newPathID:         newPathID,  	}  	return processingEmoji, nil diff --git a/internal/media/test/gts_pixellated-original.png b/internal/media/test/gts_pixellated-original.png Binary files differnew file mode 100644 index 000000000..564ee76d8 --- /dev/null +++ b/internal/media/test/gts_pixellated-original.png diff --git a/internal/media/test/gts_pixellated-static.png b/internal/media/test/gts_pixellated-static.png Binary files differnew file mode 100644 index 000000000..c6dcb0f4a --- /dev/null +++ b/internal/media/test/gts_pixellated-static.png diff --git a/internal/media/types.go b/internal/media/types.go index d71080658..3238916b8 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -98,8 +98,8 @@ type AdditionalMediaInfo struct {  	FocusY *float32  } -// AdditionalMediaInfo represents additional information -// that should be added to an emoji when processing it. +// AdditionalEmojiInfo represents additional information +// that should be taken into account when processing an emoji.  type AdditionalEmojiInfo struct {  	// Time that this emoji was created; defaults to time.Now().  	CreatedAt *time.Time diff --git a/internal/processing/admin/createemoji.go b/internal/processing/admin/createemoji.go index 50399279c..93ae17496 100644 --- a/internal/processing/admin/createemoji.go +++ b/internal/processing/admin/createemoji.go @@ -57,7 +57,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,  		return f, form.Image.Size, err  	} -	processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil) +	processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil, false)  	if err != nil {  		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji")  	} diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 104bca770..693b8685b 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -30,9 +30,10 @@ import (  	"github.com/superseriousbusiness/gotosocial/internal/gtserror"  	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"  	"github.com/superseriousbusiness/gotosocial/internal/media" +	"github.com/superseriousbusiness/gotosocial/internal/uris"  ) -func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) GetFile(ctx context.Context, requestingAccount *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {  	// parse the form fields  	mediaSize, err := media.ParseMediaSize(form.MediaSize)  	if err != nil { @@ -49,25 +50,25 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form  		return nil, gtserror.NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName))  	}  	wantedMediaID := spl[0] -	expectedAccountID := form.AccountID +	owningAccountID := form.AccountID  	// get the account that owns the media and make sure it's not suspended -	acct, err := p.db.GetAccountByID(ctx, expectedAccountID) +	owningAccount, err := p.db.GetAccountByID(ctx, owningAccountID)  	if err != nil { -		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", expectedAccountID, err)) +		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", owningAccountID, err))  	} -	if !acct.SuspendedAt.IsZero() { -		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", expectedAccountID)) +	if !owningAccount.SuspendedAt.IsZero() { +		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", owningAccountID))  	}  	// make sure the requesting account and the media account don't block each other -	if account != nil { -		blocked, err := p.db.IsBlocked(ctx, account.ID, expectedAccountID, true) +	if requestingAccount != nil { +		blocked, err := p.db.IsBlocked(ctx, requestingAccount.ID, owningAccountID, true)  		if err != nil { -			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", expectedAccountID, account.ID, err)) +			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", owningAccountID, requestingAccount.ID, err))  		}  		if blocked { -			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", expectedAccountID, account.ID)) +			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", owningAccountID, requestingAccount.ID))  		}  	} @@ -75,15 +76,15 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form  	// so we need to take different steps depending on the media type being requested  	switch mediaType {  	case media.TypeEmoji: -		return p.getEmojiContent(ctx, wantedMediaID, mediaSize) +		return p.getEmojiContent(ctx, wantedMediaID, owningAccountID, mediaSize)  	case media.TypeAttachment, media.TypeHeader, media.TypeAvatar: -		return p.getAttachmentContent(ctx, account, wantedMediaID, expectedAccountID, mediaSize) +		return p.getAttachmentContent(ctx, requestingAccount, wantedMediaID, owningAccountID, mediaSize)  	default:  		return nil, gtserror.NewErrorNotFound(fmt.Errorf("media type %s not recognized", mediaType))  	}  } -func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, expectedAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, owningAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) {  	attachmentContent := &apimodel.Content{}  	var storagePath string @@ -93,8 +94,8 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount  		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))  	} -	if a.AccountID != expectedAccountID { -		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, expectedAccountID)) +	if a.AccountID != owningAccountID { +		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, owningAccountID))  	}  	// get file information from the attachment depending on the requested media size @@ -227,17 +228,23 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount  	return attachmentContent, nil  } -func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) getEmojiContent(ctx context.Context, fileName string, owningAccountID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) {  	emojiContent := &apimodel.Content{}  	var storagePath string -	e, err := p.db.GetEmojiByID(ctx, wantedEmojiID) +	// reconstruct the static emoji image url -- reason +	// for using the static URL rather than full size url +	// is that static emojis are always encoded as png, +	// so this is more reliable than using full size url +	imageStaticURL := uris.GenerateURIForAttachment(owningAccountID, string(media.TypeEmoji), string(media.SizeStatic), fileName, "png") + +	e, err := p.db.GetEmojiByStaticURL(ctx, imageStaticURL)  	if err != nil { -		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err)) +		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", fileName, err))  	}  	if *e.Disabled { -		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedEmojiID)) +		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", fileName))  	}  	switch emojiSize { diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index 6194dba82..c84dd09f4 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -760,6 +760,10 @@ func (c *converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too  	iconProperty.AppendActivityStreamsImage(iconImage)  	emoji.SetActivityStreamsIcon(iconProperty) +	updatedProp := streams.NewActivityStreamsUpdatedProperty() +	updatedProp.Set(e.ImageUpdatedAt) +	emoji.SetActivityStreamsUpdated(updatedProp) +  	return emoji, nil  } diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index f2845be02..b63fa7aea 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -72,7 +72,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() {  	// this is necessary because the order of multiple 'context' entries is not determinate  	trimmed := strings.Split(string(bytes), "\"discoverable\"")[1] -	suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) +	suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed)  }  func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() { @@ -157,7 +157,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {  	// http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams --  	// will appear, so trim them out of the string for consistency  	trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] -	suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) +	suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)  }  func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { @@ -179,7 +179,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {  	// http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams --  	// will appear, so trim them out of the string for consistency  	trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] -	suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) +	suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed)  }  func (suite *InternalToASTestSuite) TestStatusToASWithMentions() {  | 
