diff options
| author | 2025-04-06 14:39:40 +0200 | |
|---|---|---|
| committer | 2025-04-06 14:39:40 +0200 | |
| commit | 8ae2440da3a9b66c379c5a9444b50e758deef61b (patch) | |
| tree | 06b3ba58208434f6ddbd198a396159ac2c4d9c80 /internal/db/bundb/migrations | |
| parent | [feature] Allow deleting avatar + header via settings panel (#3970) (diff) | |
| download | gotosocial-8ae2440da3a9b66c379c5a9444b50e758deef61b.tar.xz | |
[chore] Migrate accounts to new table, relax uniqueness constraint of actor `url` and collections (#3928)
* [chore] Migrate accounts to new table, relax uniqueness constraint of actor url and collections
* fiddle with it! (that's what she said)
* remove unused cache fields
* sillyness
* fix tiny whoopsie
Diffstat (limited to 'internal/db/bundb/migrations')
4 files changed, 592 insertions, 0 deletions
diff --git a/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness.go b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness.go new file mode 100644 index 000000000..acc37529d --- /dev/null +++ b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness.go @@ -0,0 +1,398 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package migrations + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + new_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/new" + old_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/old" + "github.com/superseriousbusiness/gotosocial/internal/log" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func init() { + up := func(ctx context.Context, bdb *bun.DB) error { + log.Info(ctx, "converting accounts to new model; this may take a while, please don't interrupt!") + + return bdb.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + var ( + // We have to use different + // syntax for this query + // depending on dialect. + dbDialect = tx.Dialect().Name() + + // ID for paging. + maxID string + + // Batch size for + // selecting + updating. + batchsz = 100 + + // Number of accounts + // updated so far. + updated int + + // We need to know our own host + // for updating instance account. + host = config.GetHost() + ) + + // Create the new accounts table. + if _, err := tx. + NewCreateTable(). + ModelTableExpr("new_accounts"). + Model(&new_gtsmodel.Account{}). + Exec(ctx); err != nil { + return err + } + + // Count number of accounts + // we need to update. + total, err := tx. + NewSelect(). + Table("accounts"). + Count(ctx) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return err + } + + // Create a subquery for + // Postgres to reuse. + var orderQPG *bun.RawQuery + if dbDialect == dialect.PG { + orderQPG = tx.NewRaw( + "(COALESCE(?, ?) || ? || ?) COLLATE ?", + bun.Ident("domain"), "", + "/@", + bun.Ident("username"), + bun.Ident("C"), + ) + } + + var orderQSqlite *bun.RawQuery + if dbDialect == dialect.SQLite { + orderQSqlite = tx.NewRaw( + "(COALESCE(?, ?) || ? || ?)", + bun.Ident("domain"), "", + "/@", + bun.Ident("username"), + ) + } + + for { + // Batch of old model account IDs to select. + oldAccountIDs := make([]string, 0, batchsz) + + // Start building IDs query. + idsQ := tx. + NewSelect(). + Table("accounts"). + Column("id"). + Limit(batchsz) + + if dbDialect == dialect.SQLite { + // For SQLite we can just select + // our indexed expression once + // as a column alias. + idsQ = idsQ. + ColumnExpr( + "(COALESCE(?, ?) || ? || ?) AS ?", + bun.Ident("domain"), "", + "/@", + bun.Ident("username"), + bun.Ident("domain_username"), + ) + } + + // Return only accounts with `[domain]/@[username]` + // later in the alphabet (a-z) than provided maxID. + if maxID != "" { + if dbDialect == dialect.SQLite { + idsQ = idsQ.Where("? > ?", bun.Ident("domain_username"), maxID) + } else { + idsQ = idsQ.Where("? > ?", orderQPG, maxID) + } + } + + // Page down. + // It's counterintuitive because it + // says ASC in the query, but we're + // going forwards in the alphabet, + // and z > a in a string comparison. + if dbDialect == dialect.SQLite { + idsQ = idsQ.OrderExpr("? ASC", bun.Ident("domain_username")) + } else { + idsQ = idsQ.OrderExpr("? ASC", orderQPG) + } + + // Select this batch, providing a + // slice to throw away username_domain. + err := idsQ.Scan(ctx, &oldAccountIDs, new([]string)) + if err != nil { + return err + } + + l := len(oldAccountIDs) + if len(oldAccountIDs) == 0 { + // Nothing left + // to update. + break + } + + // Get ready to select old accounts by their IDs. + oldAccounts := make([]*old_gtsmodel.Account, 0, l) + batchQ := tx. + NewSelect(). + Model(&oldAccounts). + Where("? IN (?)", bun.Ident("id"), bun.In(oldAccountIDs)) + + // Order batch by usernameDomain + // to ensure paging consistent. + if dbDialect == dialect.SQLite { + batchQ = batchQ.OrderExpr("? ASC", orderQSqlite) + } else { + batchQ = batchQ.OrderExpr("? ASC", orderQPG) + } + + // Select old accounts. + if err := batchQ.Scan(ctx); err != nil { + return err + } + + // Convert old accounts into new accounts. + newAccounts := make([]*new_gtsmodel.Account, 0, l) + for _, oldAccount := range oldAccounts { + + var actorType new_gtsmodel.AccountActorType + if oldAccount.Domain == "" && oldAccount.Username == host { + // This is our instance account, override actor + // type to Service, as previously it was just person. + actorType = new_gtsmodel.AccountActorTypeService + } else { + // Not our instance account, just parse new actor type. + actorType = new_gtsmodel.ParseAccountActorType(oldAccount.ActorType) + } + + if actorType == new_gtsmodel.AccountActorTypeUnknown { + // This should not really happen, but it if does + // just warn + set to person rather than failing. + log.Warnf(ctx, + "account %s actor type %s was not a recognized actor type, falling back to Person", + oldAccount.ID, oldAccount.ActorType, + ) + actorType = new_gtsmodel.AccountActorTypePerson + } + + newAccount := &new_gtsmodel.Account{ + ID: oldAccount.ID, + CreatedAt: oldAccount.CreatedAt, + UpdatedAt: oldAccount.UpdatedAt, + FetchedAt: oldAccount.FetchedAt, + Username: oldAccount.Username, + Domain: oldAccount.Domain, + AvatarMediaAttachmentID: oldAccount.AvatarMediaAttachmentID, + AvatarRemoteURL: oldAccount.AvatarRemoteURL, + HeaderMediaAttachmentID: oldAccount.HeaderMediaAttachmentID, + HeaderRemoteURL: oldAccount.HeaderRemoteURL, + DisplayName: oldAccount.DisplayName, + EmojiIDs: oldAccount.EmojiIDs, + Fields: oldAccount.Fields, + FieldsRaw: oldAccount.FieldsRaw, + Note: oldAccount.Note, + NoteRaw: oldAccount.NoteRaw, + AlsoKnownAsURIs: oldAccount.AlsoKnownAsURIs, + MovedToURI: oldAccount.MovedToURI, + MoveID: oldAccount.MoveID, + Locked: oldAccount.Locked, + Discoverable: oldAccount.Discoverable, + URI: oldAccount.URI, + URL: oldAccount.URL, + InboxURI: oldAccount.InboxURI, + SharedInboxURI: oldAccount.SharedInboxURI, + OutboxURI: oldAccount.OutboxURI, + FollowingURI: oldAccount.FollowingURI, + FollowersURI: oldAccount.FollowersURI, + FeaturedCollectionURI: oldAccount.FeaturedCollectionURI, + ActorType: actorType, + PrivateKey: oldAccount.PrivateKey, + PublicKey: oldAccount.PublicKey, + PublicKeyURI: oldAccount.PublicKeyURI, + PublicKeyExpiresAt: oldAccount.PublicKeyExpiresAt, + SensitizedAt: oldAccount.SensitizedAt, + SilencedAt: oldAccount.SilencedAt, + SuspendedAt: oldAccount.SuspendedAt, + SuspensionOrigin: oldAccount.SuspensionOrigin, + } + + newAccounts = append(newAccounts, newAccount) + } + + // Insert this batch of accounts. + res, err := tx. + NewInsert(). + Model(&newAccounts). + Returning(""). + Exec(ctx) + if err != nil { + return err + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + + // Add to updated count. + updated += int(rowsAffected) + if updated == total { + // Done. + break + } + + // Set next page. + fromAcct := oldAccounts[l-1] + maxID = fromAcct.Domain + "/@" + fromAcct.Username + + // Log helpful message to admin. + log.Infof(ctx, + "migrated %d of %d accounts (next page will be from %s)", + updated, total, maxID, + ) + } + + if total != int(updated) { + // Return error here in order to rollback the whole transaction. + return fmt.Errorf("total=%d does not match updated=%d", total, updated) + } + + log.Infof(ctx, "finished migrating %d accounts", total) + + // Drop the old table. + log.Info(ctx, "dropping old accounts table") + if _, err := tx. + NewDropTable(). + Table("accounts"). + Exec(ctx); err != nil { + return err + } + + // Rename new table to old table. + log.Info(ctx, "renaming new accounts table") + if _, err := tx. + ExecContext( + ctx, + "ALTER TABLE ? RENAME TO ?", + bun.Ident("new_accounts"), + bun.Ident("accounts"), + ); err != nil { + return err + } + + // Add all account indexes to the new table. + log.Info(ctx, "recreating indexes on new accounts table") + for index, columns := range map[string][]string{ + "accounts_domain_idx": {"domain"}, + "accounts_uri_idx": {"uri"}, + "accounts_url_idx": {"url"}, + "accounts_inbox_uri_idx": {"inbox_uri"}, + "accounts_outbox_uri_idx": {"outbox_uri"}, + "accounts_followers_uri_idx": {"followers_uri"}, + "accounts_following_uri_idx": {"following_uri"}, + } { + if _, err := tx. + NewCreateIndex(). + Table("accounts"). + Index(index). + Column(columns...). + Exec(ctx); err != nil { + return err + } + } + + if dbDialect == dialect.PG { + log.Info(ctx, "moving postgres constraints from old table to new table") + + type spec struct { + old string + new string + columns []string + } + + // Rename uniqueness constraints from + // "new_accounts_*" to "accounts_*". + for _, spec := range []spec{ + { + old: "new_accounts_pkey", + new: "accounts_pkey", + columns: []string{"id"}, + }, + { + old: "new_accounts_uri_key", + new: "accounts_uri_key", + columns: []string{"uri"}, + }, + { + old: "new_accounts_public_key_uri_key", + new: "accounts_public_key_uri_key", + columns: []string{"public_key_uri"}, + }, + } { + if _, err := tx.ExecContext( + ctx, + "ALTER TABLE ? DROP CONSTRAINT IF EXISTS ?", + bun.Ident("public.accounts"), + bun.Safe(spec.old), + ); err != nil { + return err + } + + if _, err := tx.ExecContext( + ctx, + "ALTER TABLE ? ADD CONSTRAINT ? UNIQUE(?)", + bun.Ident("public.accounts"), + bun.Safe(spec.new), + bun.Safe(strings.Join(spec.columns, ",")), + ); err != nil { + return err + } + } + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/common/common.go b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/common/common.go new file mode 100644 index 000000000..8db9f1c31 --- /dev/null +++ b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/common/common.go @@ -0,0 +1,26 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package common + +import "time" + +type Field struct { + Name string + Value string + VerifiedAt time.Time `bun:",nullzero"` +} diff --git a/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/new/account.go b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/new/account.go new file mode 100644 index 000000000..55fa7f1b4 --- /dev/null +++ b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/new/account.go @@ -0,0 +1,98 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import ( + "crypto/rsa" + "strings" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/common" + "github.com/uptrace/bun" +) + +type Account struct { + bun.BaseModel `bun:"table:new_accounts"` + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` + Username string `bun:",nullzero,notnull,unique:accounts_username_domain_uniq"` + Domain string `bun:",nullzero,unique:accounts_username_domain_uniq"` + AvatarMediaAttachmentID string `bun:"type:CHAR(26),nullzero"` + AvatarRemoteURL string `bun:",nullzero"` + HeaderMediaAttachmentID string `bun:"type:CHAR(26),nullzero"` + HeaderRemoteURL string `bun:",nullzero"` + DisplayName string `bun:",nullzero"` + EmojiIDs []string `bun:"emojis,array"` + Fields []*common.Field `bun:",nullzero"` + FieldsRaw []*common.Field `bun:",nullzero"` + Note string `bun:",nullzero"` + NoteRaw string `bun:",nullzero"` + MemorializedAt time.Time `bun:"type:timestamptz,nullzero"` + AlsoKnownAsURIs []string `bun:"also_known_as_uris,array"` + MovedToURI string `bun:",nullzero"` + MoveID string `bun:"type:CHAR(26),nullzero"` + Locked *bool `bun:",nullzero,notnull,default:true"` + Discoverable *bool `bun:",nullzero,notnull,default:false"` + URI string `bun:",nullzero,notnull,unique"` + URL string `bun:",nullzero"` + InboxURI string `bun:",nullzero"` + SharedInboxURI *string `bun:""` + OutboxURI string `bun:",nullzero"` + FollowingURI string `bun:",nullzero"` + FollowersURI string `bun:",nullzero"` + FeaturedCollectionURI string `bun:",nullzero"` + ActorType AccountActorType `bun:",nullzero,notnull"` + PrivateKey *rsa.PrivateKey `bun:""` + PublicKey *rsa.PublicKey `bun:",notnull"` + PublicKeyURI string `bun:",nullzero,notnull,unique"` + PublicKeyExpiresAt time.Time `bun:"type:timestamptz,nullzero"` + SensitizedAt time.Time `bun:"type:timestamptz,nullzero"` + SilencedAt time.Time `bun:"type:timestamptz,nullzero"` + SuspendedAt time.Time `bun:"type:timestamptz,nullzero"` + SuspensionOrigin string `bun:"type:CHAR(26),nullzero"` +} + +type AccountActorType int16 + +const ( + AccountActorTypeUnknown AccountActorType = 0 + AccountActorTypeApplication AccountActorType = 1 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-application + AccountActorTypeGroup AccountActorType = 2 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-group + AccountActorTypeOrganization AccountActorType = 3 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-organization + AccountActorTypePerson AccountActorType = 4 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-person + AccountActorTypeService AccountActorType = 5 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-service +) + +func ParseAccountActorType(in string) AccountActorType { + switch strings.ToLower(in) { + case "application": + return AccountActorTypeApplication + case "group": + return AccountActorTypeGroup + case "organization": + return AccountActorTypeOrganization + case "person": + return AccountActorTypePerson + case "service": + return AccountActorTypeService + default: + return AccountActorTypeUnknown + } +} diff --git a/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/old/account.go b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/old/account.go new file mode 100644 index 000000000..a630805d4 --- /dev/null +++ b/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/old/account.go @@ -0,0 +1,70 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import ( + "crypto/rsa" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20250321131230_relax_account_uri_uniqueness/common" + "github.com/uptrace/bun" +) + +type Account struct { + bun.BaseModel `bun:"table:accounts"` + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` + Username string `bun:",nullzero,notnull,unique:usernamedomain"` + Domain string `bun:",nullzero,unique:usernamedomain"` + AvatarMediaAttachmentID string `bun:"type:CHAR(26),nullzero"` + AvatarRemoteURL string `bun:",nullzero"` + HeaderMediaAttachmentID string `bun:"type:CHAR(26),nullzero"` + HeaderRemoteURL string `bun:",nullzero"` + DisplayName string `bun:""` + EmojiIDs []string `bun:"emojis,array"` + Fields []*common.Field `bun:""` + FieldsRaw []*common.Field `bun:""` + Note string `bun:""` + NoteRaw string `bun:""` + Memorial *bool `bun:",default:false"` + AlsoKnownAsURIs []string `bun:"also_known_as_uris,array"` + MovedToURI string `bun:",nullzero"` + MoveID string `bun:"type:CHAR(26),nullzero"` + Bot *bool `bun:",default:false"` + Locked *bool `bun:",default:true"` + Discoverable *bool `bun:",default:false"` + URI string `bun:",nullzero,notnull,unique"` + URL string `bun:",nullzero,unique"` + InboxURI string `bun:",nullzero,unique"` + SharedInboxURI *string `bun:""` + OutboxURI string `bun:",nullzero,unique"` + FollowingURI string `bun:",nullzero,unique"` + FollowersURI string `bun:",nullzero,unique"` + FeaturedCollectionURI string `bun:",nullzero,unique"` + ActorType string `bun:",nullzero,notnull"` + PrivateKey *rsa.PrivateKey `bun:""` + PublicKey *rsa.PublicKey `bun:",notnull"` + PublicKeyURI string `bun:",nullzero,notnull,unique"` + PublicKeyExpiresAt time.Time `bun:"type:timestamptz,nullzero"` + SensitizedAt time.Time `bun:"type:timestamptz,nullzero"` + SilencedAt time.Time `bun:"type:timestamptz,nullzero"` + SuspendedAt time.Time `bun:"type:timestamptz,nullzero"` + SuspensionOrigin string `bun:"type:CHAR(26),nullzero"` +} |
