Import export (#194)

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* update bun library -> v1.0.4

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library -> v4.3.1-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* handle oauth token scope, fix user.SigninCount + token.UserID

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library --> v4.3.2-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update sqlite library -> v1.13.0

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* review changes

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* review changes

Co-authored-by: kim (grufwub) <grufwub@gmail.com>
Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>

2 files changed, 148 insertions, 90 deletions

diff --git a/testrig/db.go b/testrig/db.go
index 7cb4f7645..268ba16b7 100644
--- a/testrig/db.go
+++ b/testrig/db.go
@@ -72,6 +72,16 @@ func NewTestDB() db.DB {
 	return testDB
 }
 
+// CreateTestTables creates prerequisite test tables in the database, but doesn't populate them.
+func CreateTestTables(db db.DB) {
+	ctx := context.Background()
+	for _, m := range testModels {
+		if err := db.CreateTable(ctx, m); err != nil {
+			logrus.Panicf("error creating table for %+v: %s", m, err)
+		}
+	}
+}
+
 // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests.
 //
 // The accounts parameter is provided in case the db should be populated with a certain set of accounts.
@@ -85,13 +95,9 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 		logrus.Panic("db setup: db was nil")
 	}
 
-	ctx := context.Background()
+	CreateTestTables(db)
 
-	for _, m := range testModels {
-		if err := db.CreateTable(ctx, m); err != nil {
-			logrus.Panicf("error creating table for %+v: %s", m, err)
-		}
-	}
+	ctx := context.Background()
 
 	for _, v := range NewTestTokens() {
 		if err := db.Put(ctx, v); err != nil {
@@ -111,6 +117,18 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 		}
 	}
 
+	for _, v := range NewTestBlocks() {
+		if err := db.Put(ctx, v); err != nil {
+			logrus.Panic(err)
+		}
+	}
+
+	for _, v := range NewTestDomainBlocks() {
+		if err := db.Put(ctx, v); err != nil {
+			logrus.Panic(err)
+		}
+	}
+
 	for _, v := range NewTestUsers() {
 		if err := db.Put(ctx, v); err != nil {
 			logrus.Panic(err)
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index e4daed12c..91335c2a3 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -736,6 +736,19 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
 	}
 }
 
+func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock {
+	return map[string]*gtsmodel.DomainBlock{
+		"replyguys.com": {
+			ID:                 "01FF22EQM7X8E3RX1XGPN7S87D",
+			Domain:             "replyguys.com",
+			CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
+			PrivateComment:     "i blocked this domain because they keep replying with pushy + unwarranted linux advice",
+			PublicComment:      "reply-guying to tech posts",
+			Obfuscate:          false,
+		},
+	}
+}
+
 type filenames struct {
 	Original string
 	Small    string
@@ -803,13 +816,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"admin_account_status_2": {
 			ID:                       "01F8MHAAY43M6RJ473VQFCVH37",
@@ -828,13 +839,36 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
+		},
+		"admin_account_status_3": {
+			ID:                       "01FF25D5Q0DH7CHD57CTRS6WK0",
+			URI:                      "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
+			URL:                      "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
+			Content:                  "hi @the_mighty_zork welcome to the instance!",
+			CreatedAt:                time.Now().Add(-46 * time.Hour),
+			UpdatedAt:                time.Now().Add(-46 * time.Hour),
+			Local:                    true,
+			AccountURI:               "http://localhost:8080/users/admin",
+			MentionIDs:               []string{"01FF26A6BGEKCZFWNEHXB2ZZ6M"},
+			AccountID:                "01F8MH17FWEB39HZJ76B6VXSKF",
+			InReplyToID:              "01F8MHAMCHF6Y650WCRSCP4WMY",
+			InReplyToAccountID:       "01F8MH1H7YV1Z7D2C8K2730QBF",
+			InReplyToURI:             "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
+			BoostOfID:                "",
+			Visibility:               gtsmodel.VisibilityPublic,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_1": {
 			ID:                       "01F8MHAMCHF6Y650WCRSCP4WMY",
@@ -853,13 +887,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_2": {
 			ID:                       "01F8MHAYFKS4KMXF8K5Y1C0KRN",
@@ -878,13 +910,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: false,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                false,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_3": {
 			ID:                       "01F8MHBBN8120SYH7D5S050MGK",
@@ -903,13 +933,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: false,
-				Replyable: false,
-				Likeable:  false,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                false,
+			Replyable:                false,
+			Likeable:                 false,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_4": {
 			ID:                       "01F8MH82FYRXD2RC6108DAJ5HB",
@@ -929,13 +957,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_5": {
 			ID:                       "01FCTA44PW9H1TB328S9AQXKDS",
@@ -955,13 +981,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_2_status_1": {
 			ID:                       "01F8MHBQCBTDKN6X5VHGMMN4MA",
@@ -980,13 +1004,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_2_status_2": {
 			ID:                       "01F8MHC0H0A7XHTVH5F596ZKBM",
@@ -1005,13 +1027,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: false,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                false,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_2_status_3": {
 			ID:                       "01F8MHC8VWDRBQR0N1BATDDEM5",
@@ -1030,13 +1050,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: false,
-				Likeable:  false,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                false,
+			Likeable:                 false,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_2_status_4": {
 			ID:                       "01F8MHCP5P2NWYQ416SBA0XSEV",
@@ -1055,12 +1073,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: false,
-				Boostable: false,
-				Replyable: true,
-				Likeable:  true,
-			},
+			Federated:                false,
+			Boostable:                false,
+			Replyable:                true,
+			Likeable:                 true,
+
 			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_5": {
@@ -1083,13 +1100,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-				Federated: true,
-				Boostable: true,
-				Replyable: true,
-				Likeable:  true,
-			},
-			ActivityStreamsType: ap.ObjectNote,
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 	}
 }
@@ -1149,6 +1164,18 @@ func NewTestMentions() map[string]*gtsmodel.Mention {
 			TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
 			TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
 		},
+		"admin_account_mention_zork": {
+			ID:               "01FF26A6BGEKCZFWNEHXB2ZZ6M",
+			StatusID:         "01FF25D5Q0DH7CHD57CTRS6WK0",
+			CreatedAt:        time.Now().Add(-46 * time.Hour),
+			UpdatedAt:        time.Now().Add(-46 * time.Hour),
+			OriginAccountID:  "01F8MH17FWEB39HZJ76B6VXSKF",
+			OriginAccountURI: "http://localhost:8080/users/admin",
+			TargetAccountID:  "01F8MH1H7YV1Z7D2C8K2730QBF",
+			NameString:       "@the_mighty_zork",
+			TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
+			TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
+		},
 	}
 }
 
@@ -1215,6 +1242,19 @@ func NewTestFollows() map[string]*gtsmodel.Follow {
 	}
 }
 
+func NewTestBlocks() map[string]*gtsmodel.Block {
+	return map[string]*gtsmodel.Block{
+		"local_account_2_block_remote_account_1": {
+			ID:              "01FEXXET6XXMF7G2V3ASZP3YQW",
+			CreatedAt:       time.Now().Add(-1 * time.Hour),
+			UpdatedAt:       time.Now().Add(-1 * time.Hour),
+			URI:             "http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW",
+			AccountID:       "01F8MH5NBDF2MV7CTC4Q5128HF",
+			TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX",
+		},
+	}
+}
+
 // ActivityWithSignature wraps a pub.Activity along with its signature headers, for testing.
 type ActivityWithSignature struct {
 	Activity        pub.Activity
@@ -1374,7 +1414,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
 		DateHeader:      date,
 	}
 
-	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5")
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0")
 	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
 	fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{
 		SignatureHeader: sig,