diff options
author | 2021-09-09 16:15:25 +0200 | |
---|---|---|
committer | 2021-09-09 16:15:25 +0200 | |
commit | 555ea8edfb2c30d149b3ca6cb0fbe53f2798c7bc (patch) | |
tree | 24567c4c365a007fcd2d6603e696b363129abb77 /internal | |
parent | Merge pull request #198 from NyaaaWhatsUpDoc/update/sqlite-library (diff) | |
download | gotosocial-555ea8edfb2c30d149b3ca6cb0fbe53f2798c7bc.tar.xz |
Import export (#194)
* start with export/import code
* messing about with decoding/encoding
* some more fiddling
* stuff is WORKING
* working pretty alright!
* go fmt
* fix up tests, add docs
* start backup/restore doc
* tweaks
* credits
* update advancedVisibility settings
* update bun library -> v1.0.4
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
* update oauth library -> v4.3.1-SSB
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
* handle oauth token scope, fix user.SigninCount + token.UserID
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
* update oauth library --> v4.3.2-SSB
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
* update sqlite library -> v1.13.0
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
* review changes
* start with export/import code
* messing about with decoding/encoding
* some more fiddling
* stuff is WORKING
* working pretty alright!
* go fmt
* fix up tests, add docs
* start backup/restore doc
* tweaks
* credits
* update advancedVisibility settings
* review changes
Co-authored-by: kim (grufwub) <grufwub@gmail.com>
Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
Diffstat (limited to 'internal')
45 files changed, 1728 insertions, 139 deletions
diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go index 75edbc882..a785b2cff 100644 --- a/internal/api/s2s/user/repliesget_test.go +++ b/internal/api/s2s/user/repliesget_test.go @@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { b, err := ioutil.ReadAll(result.Body) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) @@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) ctx.Request.Header.Set("Date", signedRequest.DateHeader) @@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { assert.NoError(suite.T(), err) fmt.Println(string(b)) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) diff --git a/internal/cache/status.go b/internal/cache/status.go index 028abc8f7..f6fe45d99 100644 --- a/internal/cache/status.go +++ b/internal/cache/status.go @@ -144,7 +144,10 @@ func copyStatus(status *gtsmodel.Status) *gtsmodel.Status { Sensitive: status.Sensitive, Language: status.Language, CreatedWithApplicationID: status.CreatedWithApplicationID, - VisibilityAdvanced: status.VisibilityAdvanced, + Federated: status.Federated, + Boostable: status.Boostable, + Replyable: status.Replyable, + Likeable: status.Likeable, ActivityStreamsType: status.ActivityStreamsType, Text: status.Text, Pinned: status.Pinned, diff --git a/internal/cliactions/admin/trans/export.go b/internal/cliactions/admin/trans/export.go new file mode 100644 index 000000000..3d9607ea6 --- /dev/null +++ b/internal/cliactions/admin/trans/export.go @@ -0,0 +1,52 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Export exports info from the database into a file +var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + exporter := trans.NewExporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := exporter.ExportMinimal(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/cliactions/admin/trans/import.go b/internal/cliactions/admin/trans/import.go new file mode 100644 index 000000000..7b137eccc --- /dev/null +++ b/internal/cliactions/admin/trans/import.go @@ -0,0 +1,56 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Import imports info from a file into the database +var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + importer := trans.NewImporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := dbConn.CreateAllTables(ctx); err != nil { + return err + } + + if err := importer.Import(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 0769ade82..3ef714fb0 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/oidc" @@ -51,32 +50,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/web" ) -var models []interface{} = []interface{}{ - >smodel.Account{}, - >smodel.Application{}, - >smodel.Block{}, - >smodel.DomainBlock{}, - >smodel.EmailDomainBlock{}, - >smodel.Follow{}, - >smodel.FollowRequest{}, - >smodel.MediaAttachment{}, - >smodel.Mention{}, - >smodel.Status{}, - >smodel.StatusToEmoji{}, - >smodel.StatusToTag{}, - >smodel.StatusFave{}, - >smodel.StatusBookmark{}, - >smodel.StatusMute{}, - >smodel.Tag{}, - >smodel.User{}, - >smodel.Emoji{}, - >smodel.Instance{}, - >smodel.Notification{}, - >smodel.RouterSession{}, - >smodel.Token{}, - >smodel.Client{}, -} - // Start creates and starts a gotosocial server var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { dbService, err := bundb.NewBunDBService(ctx, c, log) @@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log return fmt.Errorf("error creating dbservice: %s", err) } - for _, m := range models { - if err := dbService.CreateTable(ctx, m); err != nil { - return fmt.Errorf("table creation error: %s", err) - } + if err := dbService.CreateAllTables(ctx); err != nil { + return fmt.Errorf("error creating database tables: %s", err) } if err := dbService.CreateInstanceAccount(ctx); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 68e958995..bb789b7d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,9 @@ const ( PasswordFlag = "password" PasswordUsage = "the password to set for this account" + + TransPathFlag = "path" + TransPathUsage = "the path of the file to import from/export to" ) // Config pulls together all the configuration needed to run gotosocial @@ -65,6 +68,7 @@ type Config struct { Not parsed from .yaml configuration file. */ AccountCLIFlags map[string]string + ExportCLIFlags map[string]string SoftwareVersion string } @@ -92,6 +96,7 @@ func Empty() *Config { LetsEncryptConfig: &LetsEncryptConfig{}, OIDCConfig: &OIDCConfig{}, AccountCLIFlags: make(map[string]string), + ExportCLIFlags: make(map[string]string), } } @@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error { c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag) c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag) + // export CLI flags + c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag) + c.SoftwareVersion = version return nil } diff --git a/internal/db/basic.go b/internal/db/basic.go index cf65ddc09..2a1141c8d 100644 --- a/internal/db/basic.go +++ b/internal/db/basic.go @@ -26,6 +26,10 @@ type Basic interface { // For implementations that don't use tables, this can just return nil. CreateTable(ctx context.Context, i interface{}) Error + // CreateAllTables creates *all* tables necessary for the running of GoToSocial. + // Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized. + CreateAllTables(ctx context.Context) Error + // DropTable drops the table for the given interface. // For implementations that don't use tables, this can just return nil. DropTable(ctx context.Context, i interface{}) Error diff --git a/internal/db/bundb/basic.go b/internal/db/bundb/basic.go index a3a8d0ae9..d4de5bb0b 100644 --- a/internal/db/bundb/basic.go +++ b/internal/db/bundb/basic.go @@ -24,6 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/uptrace/bun" ) @@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{}) } q := b.conn.NewSelect().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + + selectWhere(q, where) err := q.Scan(ctx) return b.conn.ProcessError(err) @@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface NewDelete(). Model(i) - for _, w := range where { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } + deleteWhere(q, where) _, err := q.Exec(ctx) return b.conn.ProcessError(err) @@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error { q := b.conn.NewUpdate().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + updateWhere(q, where) q = q.Set("? = ?", bun.Safe(key), value) @@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error { return err } +func (b *basicDB) CreateAllTables(ctx context.Context) db.Error { + models := []interface{}{ + >smodel.Account{}, + >smodel.Application{}, + >smodel.Block{}, + >smodel.DomainBlock{}, + >smodel.EmailDomainBlock{}, + >smodel.Follow{}, + >smodel.FollowRequest{}, + >smodel.MediaAttachment{}, + >smodel.Mention{}, + >smodel.Status{}, + >smodel.StatusToEmoji{}, + >smodel.StatusToTag{}, + >smodel.StatusFave{}, + >smodel.StatusBookmark{}, + >smodel.StatusMute{}, + >smodel.Tag{}, + >smodel.User{}, + >smodel.Emoji{}, + >smodel.Instance{}, + >smodel.Notification{}, + >smodel.RouterSession{}, + >smodel.Token{}, + >smodel.Client{}, + } + for _, i := range models { + if err := b.CreateTable(ctx, i); err != nil { + return err + } + } + return nil +} + func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error { _, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx) return b.conn.ProcessError(err) diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index d8067fb9d..e5f7e159a 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 12) + suite.Len(s, 13) +} + +func (suite *BasicTestSuite) TestGetAllNotNull() { + where := []db.Where{{ + Key: "domain", + Value: nil, + Not: true, + }} + + a := []*gtsmodel.Account{} + + err := suite.db.GetWhere(context.Background(), where, &a) + suite.NoError(err) + suite.NotEmpty(a) + + for _, acct := range a { + suite.NotEmpty(acct.Domain) + } } func TestBasicTestSuite(t *testing.T) { diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 9464cfadf..2c26a7df9 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -240,11 +240,11 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, } } - // only do one loop if we only want direct children - if onlyDirect { - return + // if we're not only looking for direct children of status, then do the same children-finding + // operation for the found child status too. + if !onlyDirect { + s.statusChildren(ctx, child, foundStatuses, false, minID) } - s.statusChildren(ctx, child, foundStatuses, false, minID) } } diff --git a/internal/db/bundb/status_test.go b/internal/db/bundb/status_test.go index 4b4a5aca4..64079c78f 100644 --- a/internal/db/bundb/status_test.go +++ b/internal/db/bundb/status_test.go @@ -43,10 +43,14 @@ func (suite *StatusTestSuite) TestGetStatusByID() { suite.Nil(status.BoostOfAccount) suite.Nil(status.InReplyTo) suite.Nil(status.InReplyToAccount) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusByURI() { - status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_1_status_1"].URI) + status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_2_status_3"].URI) if err != nil { suite.FailNow(err.Error()) } @@ -57,6 +61,10 @@ func (suite *StatusTestSuite) TestGetStatusByURI() { suite.Nil(status.BoostOfAccount) suite.Nil(status.InReplyTo) suite.Nil(status.InReplyToAccount) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.False(status.Replyable) + suite.False(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusWithExtras() { @@ -70,6 +78,10 @@ func (suite *StatusTestSuite) TestGetStatusWithExtras() { suite.NotEmpty(status.Tags) suite.NotEmpty(status.Attachments) suite.NotEmpty(status.Emojis) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusWithMention() { @@ -83,6 +95,10 @@ func (suite *StatusTestSuite) TestGetStatusWithMention() { suite.NotEmpty(status.MentionIDs) suite.NotEmpty(status.InReplyToID) suite.NotEmpty(status.InReplyToAccountID) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusTwice() { @@ -104,6 +120,18 @@ func (suite *StatusTestSuite) TestGetStatusTwice() { suite.Less(duration2, duration1) } +func (suite *StatusTestSuite) TestGetStatusChildren() { + targetStatus := suite.testStatuses["local_account_1_status_1"] + children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "") + suite.NoError(err) + suite.Len(children, 2) + for _, c := range children { + suite.Equal(targetStatus.URI, c.InReplyToURI) + suite.Equal(targetStatus.AccountID, c.InReplyToAccountID) + suite.Equal(targetStatus.ID, c.InReplyToID) + } +} + func TestStatusTestSuite(t *testing.T) { suite.Run(t, new(StatusTestSuite)) } diff --git a/internal/db/bundb/util.go b/internal/db/bundb/util.go index 9e1afb87e..459f65d8c 100644 --- a/internal/db/bundb/util.go +++ b/internal/db/bundb/util.go @@ -19,6 +19,7 @@ package bundb import ( + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/uptrace/bun" ) @@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery { WhereOr("? = ''", bun.Ident(column)) } } + +// updateWhere parses []db.Where and adds it to the given update query. +func updateWhere(q *bun.UpdateQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// selectWhere parses []db.Where and adds it to the given select query. +func selectWhere(q *bun.SelectQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// deleteWhere parses []db.Where and adds it to the given where query. +func deleteWhere(q *bun.DeleteQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// parseWhere looks through the options on a single db.Where entry, and +// returns the appropriate query string and arguments. +func parseWhere(w db.Where) (query string, args []interface{}) { + if w.Not { + if w.Value == nil { + query = "? IS NOT NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) != LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? != ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + if w.Value == nil { + query = "? IS NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) = LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? = ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return +} diff --git a/internal/db/params.go b/internal/db/params.go index f0c384435..dbbf734a1 100644 --- a/internal/db/params.go +++ b/internal/db/params.go @@ -22,9 +22,13 @@ package db type Where struct { // The table to search on. Key string - // The value that must be set. + // The value to match. Value interface{} // Whether the value (if a string) should be case sensitive or not. // Defaults to false. CaseInsensitive bool + // If set, reverse the where. + // `WHERE k = v` becomes `WHERE k != v`. + // `WHERE k IS NULL` becomes `WHERE k IS NOT NULL` + Not bool } diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go index d5cc5ad0c..f0b4c9e9a 100644 --- a/internal/federation/dereferencing/announce.go +++ b/internal/federation/dereferencing/announce.go @@ -60,7 +60,10 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat announce.BoostOfID = boostedStatus.ID announce.BoostOfAccountID = boostedStatus.AccountID announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced + announce.Federated = boostedStatus.Federated + announce.Boostable = boostedStatus.Boostable + announce.Replyable = boostedStatus.Replyable + announce.Likeable = boostedStatus.Likeable announce.BoostOf = boostedStatus return nil } diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go index aef83f689..636870232 100644 --- a/internal/federation/dereferencing/status_test.go +++ b/internal/federation/dereferencing/status_test.go @@ -58,6 +58,10 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() { dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) suite.NoError(err) suite.Equal(status.ID, dbStatus.ID) + suite.True(dbStatus.Federated) + suite.True(dbStatus.Boostable) + suite.True(dbStatus.Replyable) + suite.True(dbStatus.Likeable) // account should be in the database now too account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) @@ -96,6 +100,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() { dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) suite.NoError(err) suite.Equal(status.ID, dbStatus.ID) + suite.True(dbStatus.Federated) + suite.True(dbStatus.Boostable) + suite.True(dbStatus.Replyable) + suite.True(dbStatus.Likeable) // account should be in the database now too account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index f298e71cd..38cb6e9c1 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -57,10 +57,13 @@ type Status struct { Language string `validate:"-" bun:",nullzero"` // what language is this status written in? CreatedWithApplicationID string `validate:"required_if=Local true,omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Which application was used to create this status? CreatedWithApplication *Application `validate:"-" bun:"rel:belongs-to"` // application corresponding to createdWithApplicationID - VisibilityAdvanced VisibilityAdvanced `validate:"required" bun:",nullzero,notnull" ` // advanced visibility for this status ActivityStreamsType string `validate:"required" bun:",nullzero,notnull"` // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!. Text string `validate:"-" bun:",nullzero"` // Original text of the status without formatting - Pinned bool `validate:"-" bun:",notnull,default:false" ` // Has this status been pinned by its owner? + Pinned bool `validate:"-" bun:",notnull,default:false"` // Has this status been pinned by its owner? + Federated bool `validate:"-" bun:",notnull"` // This status will be federated beyond the local timeline(s) + Boostable bool `validate:"-" bun:",notnull"` // This status can be boosted/reblogged + Replyable bool `validate:"-" bun:",notnull"` // This status can be replied to + Likeable bool `validate:"-" bun:",notnull"` // This status can be liked/faved } // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags. @@ -96,21 +99,3 @@ const ( // VisibilityDefault is used when no other setting can be found. VisibilityDefault Visibility = VisibilityUnlocked ) - -// VisibilityAdvanced models flags for fine-tuning visibility and interactivity of a status. -// -// All flags default to true. -// -// If PUBLIC is selected, flags will all be overwritten to TRUE regardless of what is selected. -// -// If UNLOCKED is selected, any flags can be turned on or off in any combination. -// -// If FOLLOWERS-ONLY or MUTUALS-ONLY are selected, boostable will always be FALSE. Other flags can be turned on or off as desired. -// -// If DIRECT is selected, boostable will be FALSE, and all other flags will be TRUE. -type VisibilityAdvanced struct { - Federated bool `validate:"-" bun:",notnull,default:true"` // This status will be federated beyond the local timeline(s) - Boostable bool `validate:"-" bun:",notnull,default:true"` // This status can be boosted/reblogged - Replyable bool `validate:"-" bun:",notnull,default:true"` // This status can be replied to - Likeable bool `validate:"-" bun:",notnull,default:true"` // This status can be liked/faved -} diff --git a/internal/processing/fromclientapi.go b/internal/processing/fromclientapi.go index 5f1ed325b..5fd88d8e4 100644 --- a/internal/processing/fromclientapi.go +++ b/internal/processing/fromclientapi.go @@ -51,7 +51,7 @@ func (p *processor) ProcessFromClientAPI(ctx context.Context, clientMsg messages return err } - if status.VisibilityAdvanced.Federated { + if status.Federated { return p.federateStatus(ctx, status) } case ap.ActivityFollow: diff --git a/internal/processing/status/boost.go b/internal/processing/status/boost.go index d6c4ada41..4276ca9fa 100644 --- a/internal/processing/status/boost.go +++ b/internal/processing/status/boost.go @@ -46,7 +46,7 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou if !visible { return nil, gtserror.NewErrorNotFound(errors.New("status is not visible")) } - if !targetStatus.VisibilityAdvanced.Boostable { + if !targetStatus.Boostable { return nil, gtserror.NewErrorForbidden(errors.New("status is not boostable")) } diff --git a/internal/processing/status/fave.go b/internal/processing/status/fave.go index 195bfa56a..f3f10c43c 100644 --- a/internal/processing/status/fave.go +++ b/internal/processing/status/fave.go @@ -49,7 +49,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun if !visible { return nil, gtserror.NewErrorNotFound(errors.New("status is not visible")) } - if !targetStatus.VisibilityAdvanced.Likeable { + if !targetStatus.Likeable { return nil, gtserror.NewErrorForbidden(errors.New("status is not faveable")) } diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go index 8861a532b..5ed63d919 100644 --- a/internal/processing/status/util.go +++ b/internal/processing/status/util.go @@ -33,12 +33,10 @@ import ( func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error { // by default all flags are set to true - gtsAdvancedVis := gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - } + federated := true + boostable := true + replyable := true + likeable := true var vis gtsmodel.Visibility // If visibility isn't set on the form, then just take the account default. @@ -58,47 +56,50 @@ func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.Advanc case gtsmodel.VisibilityUnlocked: // for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated + federated = *form.Federated } if form.Boostable != nil { - gtsAdvancedVis.Boostable = *form.Boostable + boostable = *form.Boostable } if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable + replyable = *form.Replyable } if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable + likeable = *form.Likeable } case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly: // for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them - gtsAdvancedVis.Boostable = false + boostable = false if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated + federated = *form.Federated } if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable + replyable = *form.Replyable } if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable + likeable = *form.Likeable } case gtsmodel.VisibilityDirect: // direct is pretty easy: there's only one possible setting so return it - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Boostable = false - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Likeable = true + federated = true + boostable = false + replyable = true + likeable = true } status.Visibility = vis - status.VisibilityAdvanced = gtsAdvancedVis + status.Federated = federated + status.Boostable = boostable + status.Replyable = replyable + status.Likeable = likeable return nil } @@ -123,7 +124,7 @@ func (p *processor) ProcessReplyToID(ctx context.Context, form *apimodel.Advance } return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err) } - if !repliedStatus.VisibilityAdvanced.Replyable { + if !repliedStatus.Replyable { return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID) } diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index 96c333c5f..6c4a58c76 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() { suite.FailNow(err.Error()) } - // we only have 12 statuses in the test suite - suite.Len(statuses, 12) + // we only have 13 statuses in the test suite + suite.Len(statuses, 13) // statuses should be sorted highest to lowest ID var highest string @@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go index 25565a1de..2a6429b3e 100644 --- a/internal/timeline/index_test.go +++ b/internal/timeline/index_test.go @@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() { // the oldest indexed post should be the lowest one we have in our testrig postID, err := suite.timeline.OldestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID) + suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID) indexLength := suite.timeline.PostIndexLength(context.Background()) suite.Equal(10, indexLength) @@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() { // the newest indexed post should be the highest one we have in our testrig postID, err := suite.timeline.NewestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID) + suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID) // indexLength should be 10 because that's all this user has hometimelineable indexLength := suite.timeline.PostIndexLength(context.Background()) diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go index ea4dc4c12..a67a8ae5a 100644 --- a/internal/timeline/manager_test.go +++ b/internal/timeline/manager_test.go @@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20) suite.NoError(err) - // local_account_1 can see 12 statuses out of the testrig statuses in its home timeline + // local_account_1 can see 13 statuses out of the testrig statuses in its home timeline indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(12, indexedLen) + suite.Equal(13, indexedLen) // oldest should now be set oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // get hometimeline statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false) suite.NoError(err) - suite.Len(statuses, 12) + suite.Len(statuses, 13) // now wipe the last status from all timelines, as though it had been deleted by the owner err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R") @@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(11, indexedLen) + suite.Equal(12, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(10, indexedLen) + suite.Equal(11, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID) suite.NoError(err) - // timeline should be empty now + // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(5, indexedLen) + suite.Equal(6, indexedLen) // ingest 1 into the timeline status1 := suite.testStatuses["admin_account_status_1"] @@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be longer now indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(7, indexedLen) + suite.Equal(8, indexedLen) // try to ingest status 2 again ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID) diff --git a/internal/trans/decoders.go b/internal/trans/decoders.go new file mode 100644 index 000000000..b4f146023 --- /dev/null +++ b/internal/trans/decoders.go @@ -0,0 +1,138 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "time" + + "github.com/mitchellh/mapstructure" + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func newDecoder(target interface{}) (*mapstructure.Decoder, error) { + decoderConfig := &mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string + Result: target, + } + return mapstructure.NewDecoder(decoderConfig) +} + +func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) { + a := &transmodel.Account{} + if err := i.simpleDecode(e, a); err != nil { + return nil, err + } + + // extract public key + publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString)) + if publicKeyBlock == nil { + return nil, errors.New("accountDecode: error decoding account public key") + } + publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err) + } + a.PublicKey = publicKey + + if a.Domain == "" { + // extract private key (local account) + privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString)) + if privateKeyBlock == nil { + return nil, errors.New("accountDecode: error decoding account private key") + } + privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err) + } + a.PrivateKey = privateKey + } + + return a, nil +} + +func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) { + b := &transmodel.Block{} + if err := i.simpleDecode(e, b); err != nil { + return nil, err + } + + return b, nil +} + +func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) { + b := &transmodel.DomainBlock{} + if err := i.simpleDecode(e, b); err != nil { + return nil, err + } + + return b, nil +} + +func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) { + f := &transmodel.Follow{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) { + f := &transmodel.FollowRequest{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) { + inst := &transmodel.Instance{} + if err := i.simpleDecode(e, inst); err != nil { + return nil, err + } + + return inst, nil +} + +func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) { + u := &transmodel.User{} + if err := i.simpleDecode(e, u); err != nil { + return nil, err + } + + return u, nil +} + +func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error { + decoder, err := newDecoder(target) + if err != nil { + return fmt.Errorf("simpleDecode: error creating decoder: %s", err) + } + + if err := decoder.Decode(&entry); err != nil { + return fmt.Errorf("simpleDecode: error decoding: %s", err) + } + + return nil +} diff --git a/internal/trans/encoders.go b/internal/trans/encoders.go new file mode 100644 index 000000000..76c2acadc --- /dev/null +++ b/internal/trans/encoders.go @@ -0,0 +1,83 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +// accountEncode handles special fields like private + public keys on accounts +func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error { + a.Type = transmodel.TransAccount + + // marshal public key + encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey) + if encodedPublicKey == nil { + return errors.New("could not MarshalPKCS1PublicKey") + } + publicKeyBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PUBLIC KEY", + Bytes: encodedPublicKey, + }) + a.PublicKeyString = string(publicKeyBytes) + + if a.Domain == "" { + // marshal private key for local account + encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey) + if encodedPrivateKey == nil { + return errors.New("could not MarshalPKCS1PrivateKey") + } + privateKeyBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: encodedPrivateKey, + }) + a.PrivateKeyString = string(privateKeyBytes) + } + + return e.simpleEncode(ctx, f, a, a.ID) +} + +// simpleEncode can be used for any type that doesn't have special keys which need handling differently, +// or for types where special keys have already been handled. +// +// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know +// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time. +func (e *exporter) simpleEncode(ctx context.Context, file *os.File, i interface{}, id string) error { + _, alreadyWritten := e.writtenIDs[id] + if alreadyWritten { + // this exporter has already exported an entry with this ID, no need to do it twice + return nil + } + + err := json.NewEncoder(file).Encode(i) + if err != nil { + return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err) + } + + e.writtenIDs[id] = true + return nil +} diff --git a/internal/trans/export.go b/internal/trans/export.go new file mode 100644 index 000000000..a432975be --- /dev/null +++ b/internal/trans/export.go @@ -0,0 +1,223 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/package trans + +import ( + "context" + "fmt" + "os" + + "github.com/superseriousbusiness/gotosocial/internal/db" + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, file *os.File) ([]*transmodel.Account, error) { + // select using the 'where' we've been provided + accounts := []*transmodel.Account{} + if err := e.db.GetWhere(ctx, where, &accounts); err != nil { + return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err) + } + + // write any accounts found to file + for _, a := range accounts { + if err := e.accountEncode(ctx, file, a); err != nil { + return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err) + } + } + + return accounts, nil +} + +func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Block, error) { + blocksUnique := make(map[string]*transmodel.Block) + + // for each account we want to export both where it's blocking and where it's blocked + for _, a := range accounts { + // 1. export blocks owned by given account + whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}} + blocking := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err) + } + for _, b := range blocking { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + + // 2. export blocks that target given account + whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}} + blocked := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err) + } + for _, b := range blocked { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + } + + // now return all the blocks we found + blocks := []*transmodel.Block{} + for _, b := range blocksUnique { + blocks = append(blocks, b) + } + + return blocks, nil +} + +func (e *exporter) exportDomainBlocks(ctx context.Context, file *os.File) ([]*transmodel.DomainBlock, error) { + domainBlocks := []*transmodel.DomainBlock{} + + if err := e.db.GetAll(ctx, &domainBlocks); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err) + } + + for _, b := range domainBlocks { + b.Type = transmodel.TransDomainBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err) + } + } + + return domainBlocks, nil +} + +func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Follow, error) { + followsUnique := make(map[string]*transmodel.Follow) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follows owned by given account + whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}} + following := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err) + } + for _, follow := range following { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + + // 2. export follows that target given account + whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}} + followed := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err) + } + for _, follow := range followed { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + } + + // now return all the follows we found + follows := []*transmodel.Follow{} + for _, follow := range followsUnique { + follows = append(follows, follow) + } + + return follows, nil +} + +func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.FollowRequest, error) { + frsUnique := make(map[string]*transmodel.FollowRequest) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follow requests owned by given account + whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}} + requesting := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err) + } + for _, fr := range requesting { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + + // 2. export follow requests that target given account + whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}} + requested := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err) + } + for _, fr := range requested { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + } + + // now return all the followRequests we found + followRequests := []*transmodel.FollowRequest{} + for _, fr := range frsUnique { + followRequests = append(followRequests, fr) + } + + return followRequests, nil +} + +func (e *exporter) exportInstances(ctx context.Context, file *os.File) ([]*transmodel.Instance, error) { + instances := []*transmodel.Instance{} + + if err := e.db.GetAll(ctx, &instances); err != nil { + return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err) + } + + for _, u := range instances { + u.Type = transmodel.TransInstance + if err := e.simpleEncode(ctx, file, u, u.ID); err != nil { + return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err) + } + } + + return instances, nil +} + +func (e *exporter) exportUsers(ctx context.Context, file *os.File) ([]*transmodel.User, error) { + users := []*transmodel.User{} + + if err := e.db.GetAll(ctx, &users); err != nil { + return nil, fmt.Errorf("exportUsers: error selecting users: %s", err) + } + + for _, u := range users { + u.Type = transmodel.TransUser + if err := e.simpleEncode(ctx, file, u, u.ID); err != nil { + return nil, fmt.Errorf("exportUsers: error encoding user: %s", err) + } + } + + return users, nil +} diff --git a/internal/trans/exporter.go b/internal/trans/exporter.go new file mode 100644 index 000000000..3dc0558f5 --- /dev/null +++ b/internal/trans/exporter.go @@ -0,0 +1,46 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// Exporter wraps functionality for exporting entries from the database to a file. +type Exporter interface { + ExportMinimal(ctx context.Context, path string) error +} + +type exporter struct { + db db.DB + log *logrus.Logger + writtenIDs map[string]bool +} + +// NewExporter returns a new Exporter that will use the given db and logger. +func NewExporter(db db.DB, log *logrus.Logger) Exporter { + return &exporter{ + db: db, + log: log, + writtenIDs: make(map[string]bool), + } +} diff --git a/internal/trans/exportminimal.go b/internal/trans/exportminimal.go new file mode 100644 index 000000000..a9ace9b91 --- /dev/null +++ b/internal/trans/exportminimal.go @@ -0,0 +1,150 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +func (e *exporter) ExportMinimal(ctx context.Context, path string) error { + if path == "" { + return errors.New("ExportMinimal: path empty") + } + + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err) + } + + // export all local accounts we have in the database + localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err) + } + + // export all blocks that relate to local accounts + blocks, err := e.exportBlocks(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err) + } + + // for each block, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, b := range blocks { + _, alreadyWritten := e.writtenIDs[b.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[b.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err) + } + } + } + + // export all follows that relate to local accounts + follows, err := e.exportFollows(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follows: %s", err) + } + + // for each follow, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, follow := range follows { + _, alreadyWritten := e.writtenIDs[follow.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[follow.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err) + } + } + } + + // export all follow requests that relate to local accounts + followRequests, err := e.exportFollowRequests(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err) + } + + // for each follow request, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, fr := range followRequests { + _, alreadyWritten := e.writtenIDs[fr.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[fr.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err) + } + } + } + + // export all domain blocks + if _, err := e.exportDomainBlocks(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err) + } + + // export all users + if _, err := e.exportUsers(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting users: %s", err) + } + + // export all instances + if _, err := e.exportInstances(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting instances: %s", err) + } + + // export all SUSPENDED accounts to make sure the suspension sticks across db migration etc + whereSuspended := []db.Where{{ + Key: "suspended_at", + Not: true, + Value: nil, + }} + if _, err := e.exportAccounts(ctx, whereSuspended, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err) + } + + return neatClose(file) +} diff --git a/internal/trans/exportminimal_test.go b/internal/trans/exportminimal_test.go new file mode 100644 index 000000000..2bffffcfe --- /dev/null +++ b/internal/trans/exportminimal_test.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +type ExportMinimalTestSuite struct { + TransTestSuite +} + +func (suite *ExportMinimalTestSuite) TestExportMinimalOK() { + // use a temporary file path that will be cleaned when the test is closed + tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString()) + + // export to the tempFilePath + exporter := trans.NewExporter(suite.db, suite.log) + err := exporter.ExportMinimal(context.Background(), tempFilePath) + suite.NoError(err) + + // we should have some bytes in that file now + b, err := os.ReadFile(tempFilePath) + suite.NoError(err) + suite.NotEmpty(b) + fmt.Println(string(b)) +} + +func TestExportMinimalTestSuite(t *testing.T) { + suite.Run(t, &ExportMinimalTestSuite{}) +} diff --git a/internal/trans/import.go b/internal/trans/import.go new file mode 100644 index 000000000..74624e540 --- /dev/null +++ b/internal/trans/import.go @@ -0,0 +1,146 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (i *importer) Import(ctx context.Context, path string) error { + if path == "" { + return errors.New("Export: path empty") + } + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("Import: couldn't export to %s: %s", path, err) + } + + decoder := json.NewDecoder(file) + decoder.UseNumber() + + for { + entry := transmodel.Entry{} + err := decoder.Decode(&entry) + if err != nil { + if err == io.EOF { + i.log.Infof("Import: reached end of file") + return neatClose(file) + } + return fmt.Errorf("Import: error decoding in readLoop: %s", err) + } + if err := i.inputEntry(ctx, entry); err != nil { + return fmt.Errorf("Import: error inputting entry: %s", err) + } + } +} + +func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error { + t, ok := entry[transmodel.TypeKey].(string) + if !ok { + return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json") + } + + switch transmodel.Type(t) { + case transmodel.TransAccount: + account, err := i.accountDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into account: %s", err) + } + if err := i.putInDB(ctx, account); err != nil { + return fmt.Errorf("inputEntry: error adding account to database: %s", err) + } + i.log.Infof("inputEntry: added account with id %s", account.ID) + return nil + case transmodel.TransBlock: + block, err := i.blockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding block to database: %s", err) + } + i.log.Infof("inputEntry: added block with id %s", block.ID) + return nil + case transmodel.TransDomainBlock: + block, err := i.domainBlockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding domain block to database: %s", err) + } + i.log.Infof("inputEntry: added domain block with id %s", block.ID) + return nil + case transmodel.TransFollow: + follow, err := i.followDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err) + } + if err := i.putInDB(ctx, follow); err != nil { + return fmt.Errorf("inputEntry: error adding follow to database: %s", err) + } + i.log.Infof("inputEntry: added follow with id %s", follow.ID) + return nil + case transmodel.TransFollowRequest: + fr, err := i.followRequestDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err) + } + if err := i.putInDB(ctx, fr); err != nil { + return fmt.Errorf("inputEntry: error adding follow request to database: %s", err) + } + i.log.Infof("inputEntry: added follow request with id %s", fr.ID) + return nil + case transmodel.TransInstance: + inst, err := i.instanceDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err) + } + if err := i.putInDB(ctx, inst); err != nil { + return fmt.Errorf("inputEntry: error adding instance to database: %s", err) + } + i.log.Infof("inputEntry: added instance with id %s", inst.ID) + return nil + case transmodel.TransUser: + user, err := i.userDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into user: %s", err) + } + if err := i.putInDB(ctx, user); err != nil { + return fmt.Errorf("inputEntry: error adding user to database: %s", err) + } + i.log.Infof("inputEntry: added user with id %s", user.ID) + return nil + } + + i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t) + return nil +} + +func (i *importer) putInDB(ctx context.Context, entry interface{}) error { + return i.db.Put(ctx, entry) +} diff --git a/internal/trans/import_test.go b/internal/trans/import_test.go new file mode 100644 index 000000000..137a5fae1 --- /dev/null +++ b/internal/trans/import_test.go @@ -0,0 +1,91 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/trans" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type ImportMinimalTestSuite struct { + TransTestSuite +} + +func (suite *ImportMinimalTestSuite) TestImportMinimalOK() { + ctx := context.Background() + + // use a temporary file path + tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString()) + + // export to the tempFilePath + exporter := trans.NewExporter(suite.db, suite.log) + err := exporter.ExportMinimal(ctx, tempFilePath) + suite.NoError(err) + + // we should have some bytes in that file now + b, err := os.ReadFile(tempFilePath) + suite.NoError(err) + suite.NotEmpty(b) + fmt.Println(string(b)) + + // create a new database with just the tables created, no entries + testrig.StandardDBTeardown(suite.db) + newDB := testrig.NewTestDB() + testrig.CreateTestTables(newDB) + + importer := trans.NewImporter(newDB, suite.log) + err = importer.Import(ctx, tempFilePath) + suite.NoError(err) + + // we should have some accounts in the database + accounts := []*gtsmodel.Account{} + err = newDB.GetAll(ctx, &accounts) + suite.NoError(err) + suite.NotEmpty(accounts) + + // we should have some blocks in the database + blocks := []*gtsmodel.Block{} + err = newDB.GetAll(ctx, &blocks) + suite.NoError(err) + suite.NotEmpty(blocks) + + // we should have some follows in the database + follows := []*gtsmodel.Follow{} + err = newDB.GetAll(ctx, &follows) + suite.NoError(err) + suite.NotEmpty(follows) + + // we should have some domain blocks in the database + domainBlocks := []*gtsmodel.DomainBlock{} + err = newDB.GetAll(ctx, &domainBlocks) + suite.NoError(err) + suite.NotEmpty(domainBlocks) +} + +func TestImportMinimalTestSuite(t *testing.T) { + suite.Run(t, &ImportMinimalTestSuite{}) +} diff --git a/internal/trans/importer.go b/internal/trans/importer.go new file mode 100644 index 000000000..ea8866f53 --- /dev/null +++ b/internal/trans/importer.go @@ -0,0 +1,44 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// Importer wraps functionality for importing entries from a file into the database. +type Importer interface { + Import(ctx context.Context, path string) error +} + +type importer struct { + db db.DB + log *logrus.Logger +} + +// NewImporter returns a new Importer interface that uses the given db and logger. +func NewImporter(db db.DB, log *logrus.Logger) Importer { + return &importer{ + db: db, + log: log, + } +} diff --git a/internal/trans/model/account.go b/internal/trans/model/account.go new file mode 100644 index 000000000..2350048d7 --- /dev/null +++ b/internal/trans/model/account.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "crypto/rsa" + "time" +) + +// Account represents the minimum viable representation of an account for export/import. +type Account struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Username string `json:"username" bun:",nullzero"` + DisplayName string `json:"displayName,omitempty" bun:",nullzero"` + Note string `json:"note,omitempty" bun:",nullzero"` + Domain string `json:"domain,omitempty" bun:",nullzero"` + HeaderRemoteURL string `json:"headerRemoteURL,omitempty" bun:",nullzero"` + AvatarRemoteURL string `json:"avatarRemoteURL,omitempty" bun:",nullzero"` + Locked bool `json:"locked"` + Language string `json:"language,omitempty" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + URL string `json:"url" bun:",nullzero"` + InboxURI string `json:"inboxURI" bun:",nullzero"` + OutboxURI string `json:"outboxURI" bun:",nullzero"` + FollowingURI string `json:"followingUri" bun:",nullzero"` + FollowersURI string `json:"followersUri" bun:",nullzero"` + FeaturedCollectionURI string `json:"featuredCollectionUri" bun:",nullzero"` + ActorType string `json:"actorType" bun:",nullzero"` + PrivateKey *rsa.PrivateKey `json:"-" mapstructure:"-"` + PrivateKeyString string `json:"privateKey,omitempty" mapstructure:"privateKey" bun:"-"` + PublicKey *rsa.PublicKey `json:"-" mapstructure:"-"` + PublicKeyString string `json:"publicKey,omitempty" mapstructure:"publicKey" bun:"-"` + PublicKeyURI string `json:"publicKeyUri" bun:",nullzero"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` + SuspensionOrigin string `json:"suspensionOrigin,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/block.go b/internal/trans/model/block.go new file mode 100644 index 000000000..313e6a7cd --- /dev/null +++ b/internal/trans/model/block.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import "time" + +// Block represents an account block as serialized in an exported file. +type Block struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/domainblock.go b/internal/trans/model/domainblock.go new file mode 100644 index 000000000..de2bcd00a --- /dev/null +++ b/internal/trans/model/domainblock.go @@ -0,0 +1,34 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import "time" + +// DomainBlock represents a domain block as serialized in an exported file. +type DomainBlock struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` + CreatedByAccountID string `json:"createdByAccountID" bun:",nullzero"` + PrivateComment string `json:"privateComment,omitempty" bun:",nullzero"` + PublicComment string `json:"publicComment,omitempty" bun:",nullzero"` + Obfuscate bool `json:"obfuscate" bun:",nullzero"` + SubscriptionID string `json:"subscriptionID,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/follow.go b/internal/trans/model/follow.go new file mode 100644 index 000000000..b94f2600d --- /dev/null +++ b/internal/trans/model/follow.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import "time" + +// Follow represents an account follow as serialized in an export file. +type Follow struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/followrequest.go b/internal/trans/model/followrequest.go new file mode 100644 index 000000000..844bcb7af --- /dev/null +++ b/internal/trans/model/followrequest.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import "time" + +// FollowRequest represents an account follow request as serialized in an export file. +type FollowRequest struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/instance.go b/internal/trans/model/instance.go new file mode 100644 index 000000000..a75aa65bf --- /dev/null +++ b/internal/trans/model/instance.go @@ -0,0 +1,43 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "time" +) + +// Instance represents an instance entry as serialized in an export file. +type Instance struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` + Title string `json:"title,omitempty" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` + DomainBlockID string `json:"domainBlockID,omitempty" bun:",nullzero"` + ShortDescription string `json:"shortDescription,omitempty" bun:",nullzero"` + Description string `json:"description,omitempty" bun:",nullzero"` + Terms string `json:"terms,omitempty" bun:",nullzero"` + ContactEmail string `json:"contactEmail,omitempty" bun:",nullzero"` + ContactAccountUsername string `json:"contactAccountUsername,omitempty" bun:",nullzero"` + ContactAccountID string `json:"contactAccountID,omitempty" bun:",nullzero"` + Reputation int64 `json:"reputation"` + Version string `json:"version,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/type.go b/internal/trans/model/type.go new file mode 100644 index 000000000..76f57c843 --- /dev/null +++ b/internal/trans/model/type.go @@ -0,0 +1,41 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +// TypeKey should be set on a TransEntry to indicate the type of entry it is. +const TypeKey = "type" + +// Type describes the type of a trans entry, and how it should be read/serialized. +type Type string + +// Type of the trans entry. Describes how it should be read from file. +const ( + TransAccount Type = "account" + TransBlock Type = "block" + TransDomainBlock Type = "domainBlock" + TransEmailDomainBlock Type = "emailDomainBlock" + TransFollow Type = "follow" + TransFollowRequest Type = "followRequest" + TransInstance Type = "instance" + TransUser Type = "user" +) + +// Entry is used for deserializing trans entries into a rough interface so that +// the TypeKey can be fetched, before continuing with full parsing. +type Entry map[string]interface{} diff --git a/internal/trans/model/user.go b/internal/trans/model/user.go new file mode 100644 index 000000000..293b124a2 --- /dev/null +++ b/internal/trans/model/user.go @@ -0,0 +1,50 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "time" +) + +// User represents a local instance user as serialized to an export file. +type User struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Email string `json:"email,omitempty" bun:",nullzero"` + AccountID string `json:"accountID" bun:",nullzero"` + EncryptedPassword string `json:"encryptedPassword" bun:",nullzero"` + CurrentSignInAt *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"` + LastSignInAt *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"` + InviteID string `json:"inviteID,omitempty" bun:",nullzero"` + ChosenLanguages []string `json:"chosenLanguages,omitempty" bun:",nullzero"` + FilteredLanguages []string `json:"filteredLanguage,omitempty" bun:",nullzero"` + Locale string `json:"locale" bun:",nullzero"` + LastEmailedAt time.Time `json:"lastEmailedAt,omitempty" bun:",nullzero"` + ConfirmationToken string `json:"confirmationToken,omitempty" bun:",nullzero"` + ConfirmationSentAt *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"` + ConfirmedAt *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"` + UnconfirmedEmail string `json:"unconfirmedEmail,omitempty" bun:",nullzero"` + Moderator bool `json:"moderator"` + Admin bool `json:"admin"` + Disabled bool `json:"disabled"` + Approved bool `json:"approved"` + ResetPasswordToken string `json:"resetPasswordToken,omitempty" bun:",nullzero"` + ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/trans_test.go b/internal/trans/trans_test.go new file mode 100644 index 000000000..4a231486b --- /dev/null +++ b/internal/trans/trans_test.go @@ -0,0 +1,42 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans_test + +import ( + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type TransTestSuite struct { + suite.Suite + db db.DB + log *logrus.Logger +} + +func (suite *TransTestSuite) SetupTest() { + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + testrig.StandardDBSetup(suite.db, nil) +} + +func (suite *TransTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} diff --git a/internal/trans/util.go b/internal/trans/util.go new file mode 100644 index 000000000..4ccc1a4b6 --- /dev/null +++ b/internal/trans/util.go @@ -0,0 +1,32 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package trans + +import ( + "fmt" + "os" +) + +func neatClose(f *os.File) error { + if err := f.Close(); err != nil { + return fmt.Errorf("error closing file: %s", err) + } + + return nil +} diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 580f999bc..9b87e03d3 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -325,6 +325,11 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab // advanced visibility for this status // TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL + // for now we just set everything to true + status.Federated = true + status.Boostable = true + status.Replyable = true + status.Likeable = true // sensitive // TODO: this is a bool diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index 23839b9a8..b6a425732 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -70,7 +70,10 @@ func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boost BoostOfID: s.ID, BoostOfAccountID: s.AccountID, Visibility: s.Visibility, - VisibilityAdvanced: s.VisibilityAdvanced, + Federated: s.Federated, + Boostable: s.Boostable, + Replyable: s.Replyable, + Likeable: s.Likeable, // attach these here for convenience -- the boosted status/account won't go in the DB // but they're needed in the processor and for the frontend. Since we have them, we can diff --git a/internal/validate/status_test.go b/internal/validate/status_test.go index 7c85414b1..333cc408d 100644 --- a/internal/validate/status_test.go +++ b/internal/validate/status_test.go @@ -63,15 +63,13 @@ func happyStatus() *gtsmodel.Status { Language: "en", CreatedWithApplicationID: "01FEBBZHF4GFVRXSJVXD0JTZZ2", CreatedWithApplication: nil, - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, - Text: "Test status! #hello", - Pinned: false, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, + Text: "Test status! #hello", + Pinned: false, } } |