diff options
Diffstat (limited to 'internal/db')
| -rw-r--r-- | internal/db/bundb/basic_test.go | 2 | ||||
| -rw-r--r-- | internal/db/bundb/bundb_test.go | 3 | ||||
| -rw-r--r-- | internal/db/bundb/media.go | 24 | ||||
| -rw-r--r-- | internal/db/bundb/media_test.go | 48 | ||||
| -rw-r--r-- | internal/db/bundb/migrations/20220214175650_media_cleanup.go | 172 | ||||
| -rw-r--r-- | internal/db/bundb/migrations/20220214175650_media_cleanup/account.go | 117 | ||||
| -rw-r--r-- | internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go | 116 | ||||
| -rw-r--r-- | internal/db/bundb/util.go | 14 | ||||
| -rw-r--r-- | internal/db/media.go | 7 |
9 files changed, 500 insertions, 3 deletions
diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index cb93be877..2719f1bb8 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -43,7 +43,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 14) + suite.Len(s, 15) } func (suite *BasicTestSuite) TestGetAllNotNull() { diff --git a/internal/db/bundb/bundb_test.go b/internal/db/bundb/bundb_test.go index 62610237e..581573056 100644 --- a/internal/db/bundb/bundb_test.go +++ b/internal/db/bundb/bundb_test.go @@ -55,9 +55,8 @@ func (suite *BunDBStandardTestSuite) SetupSuite() { } func (suite *BunDBStandardTestSuite) SetupTest() { - testrig.InitTestLog() testrig.InitTestConfig() - + testrig.InitTestLog() suite.db = testrig.NewTestDB() testrig.StandardDBSetup(suite.db, suite.testAccounts) } diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index f24ea0407..4da80e757 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -20,6 +20,7 @@ package bundb import ( "context" + "time" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -48,3 +49,26 @@ func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.M } return attachment, nil } + +func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) { + attachments := []*gtsmodel.MediaAttachment{} + + q := m.conn. + NewSelect(). + Model(&attachments). + Where("media_attachment.cached = true"). + Where("media_attachment.avatar = false"). + Where("media_attachment.header = false"). + Where("media_attachment.created_at < ?", olderThan). + WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")). + Order("media_attachment.created_at DESC") + + if limit != 0 { + q = q.Limit(limit) + } + + if err := q.Scan(ctx); err != nil { + return nil, m.conn.ProcessError(err) + } + return attachments, nil +} diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go new file mode 100644 index 000000000..1d0b86275 --- /dev/null +++ b/internal/db/bundb/media_test.go @@ -0,0 +1,48 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package bundb_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type MediaTestSuite struct { + BunDBStandardTestSuite +} + +func (suite *MediaTestSuite) TestGetAttachmentByID() { + testAttachment := suite.testAttachments["admin_account_status_1_attachment_1"] + attachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.NoError(err) + suite.NotNil(attachment) +} + +func (suite *MediaTestSuite) TestGetOlder() { + attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20) + suite.NoError(err) + suite.Len(attachments, 1) +} + +func TestMediaTestSuite(t *testing.T) { + suite.Run(t, new(MediaTestSuite)) +} diff --git a/internal/db/bundb/migrations/20220214175650_media_cleanup.go b/internal/db/bundb/migrations/20220214175650_media_cleanup.go new file mode 100644 index 000000000..427f3bb9e --- /dev/null +++ b/internal/db/bundb/migrations/20220214175650_media_cleanup.go @@ -0,0 +1,172 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package migrations + +import ( + "context" + "database/sql" + "time" + + previousgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init" + newgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20220214175650_media_cleanup" + "github.com/uptrace/bun" +) + +func init() { + const batchSize = 100 + up := func(ctx context.Context, db *bun.DB) error { + // we need to migrate media attachments into a new table + // see section 6 here: https://www.sqlite.org/lang_altertable.html + + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // create the new media attachments table + if _, err := tx. + NewCreateTable(). + ModelTableExpr("new_media_attachments"). + Model(&newgtsmodel.MediaAttachment{}). + IfNotExists(). + Exec(ctx); err != nil { + return err + } + + offset := time.Now() + // migrate existing media attachments into new table + migrateLoop: + for { + oldAttachments := []*previousgtsmodel.MediaAttachment{} + err := tx. + NewSelect(). + Model(&oldAttachments). + // subtract a millisecond from the offset just to make sure we're not getting double entries (this happens sometimes) + Where("media_attachment.created_at < ?", offset.Add(-1*time.Millisecond)). + Order("media_attachment.created_at DESC"). + Limit(batchSize). + Scan(ctx) + if err != nil && err != sql.ErrNoRows { + // there's been a real error + return err + } + + if err == sql.ErrNoRows || len(oldAttachments) == 0 { + // we're finished migrating + break migrateLoop + } + + // update the offset to the createdAt time of the oldest media attachment in the slice + offset = oldAttachments[len(oldAttachments)-1].CreatedAt + + // for every old attachment, we need to make a new attachment out of it by taking the same values + newAttachments := []*newgtsmodel.MediaAttachment{} + for _, old := range oldAttachments { + new := &newgtsmodel.MediaAttachment{ + ID: old.ID, + CreatedAt: old.CreatedAt, + UpdatedAt: old.UpdatedAt, + StatusID: old.StatusID, + URL: old.URL, + RemoteURL: old.RemoteURL, + Type: newgtsmodel.FileType(old.Type), + FileMeta: newgtsmodel.FileMeta{ + Original: newgtsmodel.Original{ + Width: old.FileMeta.Original.Width, + Height: old.FileMeta.Original.Height, + Size: old.FileMeta.Original.Size, + Aspect: old.FileMeta.Original.Aspect, + }, + Small: newgtsmodel.Small{ + Width: old.FileMeta.Small.Width, + Height: old.FileMeta.Small.Height, + Size: old.FileMeta.Small.Size, + Aspect: old.FileMeta.Small.Aspect, + }, + Focus: newgtsmodel.Focus{ + X: old.FileMeta.Focus.X, + Y: old.FileMeta.Focus.Y, + }, + }, + AccountID: old.AccountID, + Description: old.Description, + ScheduledStatusID: old.ScheduledStatusID, + Blurhash: old.Blurhash, + Processing: newgtsmodel.ProcessingStatus(old.Processing), + File: newgtsmodel.File{ + Path: old.File.Path, + ContentType: old.File.ContentType, + FileSize: old.File.FileSize, + UpdatedAt: old.File.UpdatedAt, + }, + Thumbnail: newgtsmodel.Thumbnail{ + Path: old.Thumbnail.Path, + ContentType: old.Thumbnail.ContentType, + FileSize: old.Thumbnail.FileSize, + UpdatedAt: old.Thumbnail.UpdatedAt, + URL: old.Thumbnail.URL, + RemoteURL: old.Thumbnail.RemoteURL, + }, + Avatar: old.Avatar, + Header: old.Header, + Cached: true, + } + newAttachments = append(newAttachments, new) + } + + // insert this batch of new attachments, and then continue the loop + if _, err := tx. + NewInsert(). + Model(&newAttachments). + ModelTableExpr("new_media_attachments"). + Exec(ctx); err != nil { + return err + } + } + + // we have all the data we need from the old table, so we can safely drop it now + if _, err := tx.NewDropTable().Model(&previousgtsmodel.MediaAttachment{}).Exec(ctx); err != nil { + return err + } + + // rename the new table to the same name as the old table was + if _, err := tx.QueryContext(ctx, "ALTER TABLE new_media_attachments RENAME TO media_attachments;"); err != nil { + return err + } + + // add an index to the new table + if _, err := tx. + NewCreateIndex(). + Model(&newgtsmodel.MediaAttachment{}). + Index("media_attachments_id_idx"). + Column("id"). + Exec(ctx); err != nil { + return err + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20220214175650_media_cleanup/account.go b/internal/db/bundb/migrations/20220214175650_media_cleanup/account.go new file mode 100644 index 000000000..0b456339a --- /dev/null +++ b/internal/db/bundb/migrations/20220214175650_media_cleanup/account.go @@ -0,0 +1,117 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +// Package gtsmodel contains types used *internally* by GoToSocial and added/removed/selected from the database. +// These types should never be serialized and/or sent out via public APIs, as they contain sensitive information. +// The annotation used on these structs is for handling them via the bun-db ORM. +// See here for more info on bun model annotations: https://bun.uptrace.dev/guide/models.html +package gtsmodel + +import ( + "crypto/rsa" + "time" +) + +// Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc). +type Account struct { + ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + Username string `validate:"required" bun:",nullzero,notnull,unique:userdomain"` // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other + Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"` // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username. + AvatarMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present + AvatarMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to avatarMediaAttachmentID + AvatarRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched? + HeaderMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present + HeaderMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to headerMediaAttachmentID + HeaderRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched? + DisplayName string `validate:"-" bun:""` // DisplayName for this account. Can be empty, then just the Username will be used for display purposes. + Fields []Field `validate:"-"` // a key/value map of fields that this account has added to their profile + Note string `validate:"-" bun:""` // A note that this account has on their profile (ie., the account's bio/description of themselves) + Memorial bool `validate:"-" bun:",default:false"` // Is this a memorial account, ie., has the user passed away? + AlsoKnownAs string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account is associated with x account id (TODO: migrate to be AlsoKnownAsID) + MovedToAccountID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account has moved this account id in the database + Bot bool `validate:"-" bun:",default:false"` // Does this account identify itself as a bot? + Reason string `validate:"-" bun:""` // What reason was given for signing up when this account was created? + Locked bool `validate:"-" bun:",default:true"` // Does this account need an approval for new followers? + Discoverable bool `validate:"-" bun:",default:false"` // Should this account be shown in the instance's profile directory? + Privacy Visibility `validate:"required_without=Domain,omitempty,oneof=public unlocked followers_only mutuals_only direct" bun:",nullzero"` // Default post privacy for this account + Sensitive bool `validate:"-" bun:",default:false"` // Set posts from this account to sensitive by default? + Language string `validate:"omitempty,bcp47_language_tag" bun:",nullzero,notnull,default:'en'"` // What language does this account post in? + URI string `validate:"required,url" bun:",nullzero,notnull,unique"` // ActivityPub URI for this account. + URL string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Web URL for this account's profile + LastWebfingeredAt time.Time `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"` // Last time this account was refreshed/located with webfinger. + InboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's ActivityPub inbox, for sending activity to + OutboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's activitypub outbox + FollowingURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the following list of this account + FollowersURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the followers list of this account + FeaturedCollectionURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URL for getting the featured collection list of this account + ActorType string `validate:"oneof=Application Group Organization Person Service" bun:",nullzero,notnull"` // What type of activitypub actor is this account? + PrivateKey *rsa.PrivateKey `validate:"required_without=Domain"` // Privatekey for validating activitypub requests, will only be defined for local accounts + PublicKey *rsa.PublicKey `validate:"required"` // Publickey for encoding activitypub requests, will be defined for both local and remote accounts + PublicKeyURI string `validate:"required,url" bun:",nullzero,notnull,unique"` // Web-reachable location of this account's public key + SensitizedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account set to have all its media shown as sensitive? + SilencedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account silenced (eg., statuses only visible to followers, not public)? + SuspendedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account suspended (eg., don't allow it to log in/post, don't accept media/posts from this account) + HideCollections bool `validate:"-" bun:",default:false"` // Hide this account's collections + SuspensionOrigin string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // id of the database entry that caused this account to become suspended -- can be an account ID or a domain block ID +} + +// Field represents a key value field on an account, for things like pronouns, website, etc. +// VerifiedAt is optional, to be used only if Value is a URL to a webpage that contains the +// username of the user. +type Field struct { + Name string `validate:"required"` // Name of this field. + Value string `validate:"required"` // Value of this field. + VerifiedAt time.Time `validate:"-" bun:",nullzero"` // This field was verified at (optional). +} + +// Relationship describes a requester's relationship with another account. +type Relationship struct { + ID string // The account id. + Following bool // Are you following this user? + ShowingReblogs bool // Are you receiving this user's boosts in your home timeline? + Notifying bool // Have you enabled notifications for this user? + FollowedBy bool // Are you followed by this user? + Blocking bool // Are you blocking this user? + BlockedBy bool // Is this user blocking you? + Muting bool // Are you muting this user? + MutingNotifications bool // Are you muting notifications from this user? + Requested bool // Do you have a pending follow request for this user? + DomainBlocking bool // Are you blocking this user's domain? + Endorsed bool // Are you featuring this user on your profile? + Note string // Your note on this account. +} + +// Visibility represents the visibility granularity of a status. +type Visibility string + +const ( + // VisibilityPublic means this status will be visible to everyone on all timelines. + VisibilityPublic Visibility = "public" + // VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists. + VisibilityUnlocked Visibility = "unlocked" + // VisibilityFollowersOnly means this status is viewable to followers only. + VisibilityFollowersOnly Visibility = "followers_only" + // VisibilityMutualsOnly means this status is visible to mutual followers only. + VisibilityMutualsOnly Visibility = "mutuals_only" + // VisibilityDirect means this status is visible only to mentioned recipients. + VisibilityDirect Visibility = "direct" + // VisibilityDefault is used when no other setting can be found. + VisibilityDefault Visibility = VisibilityUnlocked +) diff --git a/internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go b/internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go new file mode 100644 index 000000000..a2f0b5f29 --- /dev/null +++ b/internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go @@ -0,0 +1,116 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package gtsmodel + +import ( + "time" +) + +type MediaAttachment struct { + ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + StatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // ID of the status to which this is attached + URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on *this* server + RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media) + Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video) + FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file + AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong + Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID + Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders) + ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong + Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment + Processing ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"` // What is the processing status of this attachment + File File `validate:"required" bun:",embed:file_,notnull,nullzero"` // metadata for the whole file + Thumbnail Thumbnail `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file. + Avatar bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as an avatar? + Header bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as a header? + Cached bool `validate:"-" bun:",notnull"` // Is this attachment currently cached by our instance? +} + +// File refers to the metadata for the whole file +type File struct { + Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage. + ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file. + FileSize int `validate:"required" bun:",notnull"` // File size in bytes + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated. +} + +// Thumbnail refers to a small image thumbnail derived from a larger image, video, or audio file. +type Thumbnail struct { + Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage. + ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file. + FileSize int `validate:"required" bun:",notnull"` // File size in bytes + UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated. + URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // What is the URL of the thumbnail on the local server + RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // What is the remote URL of the thumbnail (empty for local media) +} + +// ProcessingStatus refers to how far along in the processing stage the attachment is. +type ProcessingStatus int + +// MediaAttachment processing states. +const ( + ProcessingStatusReceived ProcessingStatus = 0 // ProcessingStatusReceived indicates the attachment has been received and is awaiting processing. No thumbnail available yet. + ProcessingStatusProcessing ProcessingStatus = 1 // ProcessingStatusProcessing indicates the attachment is currently being processed. Thumbnail is available but full media is not. + ProcessingStatusProcessed ProcessingStatus = 2 // ProcessingStatusProcessed indicates the attachment has been fully processed and is ready to be served. + ProcessingStatusError ProcessingStatus = 666 // ProcessingStatusError indicates something went wrong processing the attachment and it won't be tried again--these can be deleted. +) + +// FileType refers to the file type of the media attaachment. +type FileType string + +// MediaAttachment file types. +const ( + FileTypeImage FileType = "Image" // FileTypeImage is for jpegs and pngs + FileTypeGif FileType = "Gif" // FileTypeGif is for native gifs and soundless videos that have been converted to gifs + FileTypeAudio FileType = "Audio" // FileTypeAudio is for audio-only files (no video) + FileTypeVideo FileType = "Video" // FileTypeVideo is for files with audio + visual + FileTypeUnknown FileType = "Unknown" // FileTypeUnknown is for unknown file types (surprise surprise!) +) + +// FileMeta describes metadata about the actual contents of the file. +type FileMeta struct { + Original Original `validate:"required" bun:"embed:original_"` + Small Small `bun:"embed:small_"` + Focus Focus `bun:"embed:focus_"` +} + +// Small can be used for a thumbnail of any media type +type Small struct { + Width int `validate:"required_with=Height Size Aspect"` // width in pixels + Height int `validate:"required_with=Width Size Aspect"` // height in pixels + Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height) + Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height) +} + +// Original can be used for original metadata for any media type +type Original struct { + Width int `validate:"required_with=Height Size Aspect"` // width in pixels + Height int `validate:"required_with=Width Size Aspect"` // height in pixels + Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height) + Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height) +} + +// Focus describes the 'center' of the image for display purposes. +// X and Y should each be between -1 and 1 +type Focus struct { + X float32 `validate:"omitempty,max=1,min=-1"` + Y float32 `validate:"omitempty,max=1,min=-1"` +} diff --git a/internal/db/bundb/util.go b/internal/db/bundb/util.go index 58d06d605..34a988472 100644 --- a/internal/db/bundb/util.go +++ b/internal/db/bundb/util.go @@ -37,6 +37,20 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery { } } +// whereNotEmptyAndNotNull is a convenience function to return a bun WhereGroup that specifies +// that the given column should be NEITHER an empty string NOR null. +// +// Use it as follows: +// +// q = q.WhereGroup(" AND ", whereNotEmptyAndNotNull("whatever_column")) +func whereNotEmptyAndNotNull(column string) func(*bun.SelectQuery) *bun.SelectQuery { + return func(q *bun.SelectQuery) *bun.SelectQuery { + return q. + Where("? IS NOT NULL", bun.Ident(column)). + Where("? != ''", bun.Ident(column)) + } +} + // updateWhere parses []db.Where and adds it to the given update query. func updateWhere(q *bun.UpdateQuery, where []db.Where) { for _, w := range where { diff --git a/internal/db/media.go b/internal/db/media.go index b8c84e133..c734502a1 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -20,6 +20,7 @@ package db import ( "context" + "time" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -28,4 +29,10 @@ import ( type Media interface { // GetAttachmentByID gets a single attachment by its ID GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error) + // GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time. + // These will be returned in order of attachment.created_at descending (newest to oldest in other words). + // + // The selected media attachments will be those with both a URL and a RemoteURL filled in. + // In other words, media attachments that originated remotely, and that we currently have cached locally. + GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error) } |