[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

4 files changed, 152 insertions, 19 deletions

diff --git a/testrig/config.go b/testrig/config.go
index 673ed46b6..0a957a831 100644
--- a/testrig/config.go
+++ b/testrig/config.go
@@ -99,6 +99,8 @@ func testDefaults() config.Configuration {
 				TagStr: "en-gb",
 			},
 		},
+		InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
+		InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 
 		AccountsRegistrationOpen: true,
 		AccountsReasonRequired:   true,
diff --git a/testrig/processor.go b/testrig/processor.go
index e098de33a..bbb8d9d1d 100644
--- a/testrig/processor.go
+++ b/testrig/processor.go
@@ -26,15 +26,27 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestProcessor returns a Processor suitable for testing purposes.
 // The passed in state will have its worker functions set appropriately,
 // but the state will not be initialized.
-func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor {
+func NewTestProcessor(
+	state *state.State,
+	federator *federation.Federator,
+	emailSender email.Sender,
+	mediaManager *media.Manager,
+) *processing.Processor {
+
 	return processing.NewProcessor(
 		cleaner.New(state),
+		subscriptions.New(
+			state,
+			federator.TransportController(),
+			typeutils.NewConverter(state),
+		),
 		typeutils.NewConverter(state),
 		federator,
 		NewTestOauthServer(state.DB),
diff --git a/testrig/teststructs.go b/testrig/teststructs.go
index b88e37d55..58986bffa 100644
--- a/testrig/teststructs.go
+++ b/testrig/teststructs.go
@@ -18,6 +18,7 @@
 package testrig
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -25,6 +26,8 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -38,12 +41,13 @@ import (
 // and worker queues, which was causing issues
 // when running all tests at once.
 type TestStructs struct {
-	State         *state.State
-	Common        *common.Processor
-	Processor     *processing.Processor
-	HTTPClient    *MockHTTPClient
-	TypeConverter *typeutils.Converter
-	EmailSender   email.Sender
+	State               *state.State
+	Common              *common.Processor
+	Processor           *processing.Processor
+	HTTPClient          *MockHTTPClient
+	TypeConverter       *typeutils.Converter
+	EmailSender         email.Sender
+	TransportController transport.Controller
 }
 
 func SetupTestStructs(
@@ -56,6 +60,7 @@ func SetupTestStructs(
 
 	db := NewTestDB(&state)
 	state.DB = db
+	state.AdminActions = admin.New(db, &state.Workers)
 
 	storage := NewInMemoryStorage()
 	state.Storage = storage
@@ -89,6 +94,7 @@ func SetupTestStructs(
 
 	processor := processing.NewProcessor(
 		cleaner.New(&state),
+		subscriptions.New(&state, transportController, typeconverter),
 		typeconverter,
 		federator,
 		oauthServer,
@@ -105,12 +111,13 @@ func SetupTestStructs(
 	StandardStorageSetup(storage, rMediaPath)
 
 	return &TestStructs{
-		State:         &state,
-		Common:        &common,
-		Processor:     processor,
-		HTTPClient:    httpClient,
-		TypeConverter: typeconverter,
-		EmailSender:   emailSender,
+		State:               &state,
+		Common:              &common,
+		Processor:           processor,
+		HTTPClient:          httpClient,
+		TypeConverter:       typeconverter,
+		EmailSender:         emailSender,
+		TransportController: transportController,
 	}
 }
 
diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go
index 385c620db..8faed93ad 100644
--- a/testrig/transportcontroller.go
+++ b/testrig/transportcontroller.go
@@ -41,6 +41,8 @@ import (
 const (
 	applicationJSON         = "application/json"
 	applicationActivityJSON = "application/activity+json"
+	textCSV                 = "text/csv"
+	textPlain               = "text/plain"
 )
 
 // NewTestTransportController returns a test transport controller with the given http client.
@@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes         = []byte(`{"error":"404 not found"}`)
 			responseContentType   = applicationJSON
 			responseContentLength = len(responseBytes)
+			extraHeaders          = make(map[string]string, 0)
 			reqURLString          = req.URL.String()
 		)
 
@@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseContentType = applicationJSON
 			responseContentLength = len(responseBytes)
 		} else if strings.Contains(reqURLString, ".well-known/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".well-known/host-meta") {
-			responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req)
+		} else if strings.Contains(reqURLString, "lists.example.org") {
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req)
 		} else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok {
 			// the request is for a note that we have stored
 			noteI, err := streams.Serialize(note)
@@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 		}
 
 		log.Debugf(nil, "returning response %s", string(responseBytes))
+
 		reader := bytes.NewReader(responseBytes)
 		readCloser := io.NopCloser(reader)
+
+		header := http.Header{
+			"Content-Type": {responseContentType},
+		}
+		for k, v := range extraHeaders {
+			header.Add(k, v)
+		}
+
 		return &http.Response{
 			Request:       req,
 			StatusCode:    responseCode,
 			Body:          readCloser,
 			ContentLength: int64(responseContentLength),
-			Header:        http.Header{"Content-Type": {responseContentType}},
+			Header:        header,
 		}, nil
 	}
 
@@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) (
 	return m.do(req)
 }
 
-func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func HostMetaResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var hm *apimodel.HostMeta
 
 	if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" {
@@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte
 	return
 }
 
-func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func WebfingerResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var wfr *apimodel.WellKnownResponse
 
 	switch req.URL.String() {
@@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
 	responseContentLength = len(wfrJSON)
 	return
 }
+
+func DomainPermissionSubscriptionResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
+
+	const (
+		csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
+bumfaces.net,suspend,false,false,big jerks,false
+peepee.poopoo,suspend,false,false,harassment,false
+nothanks.com,suspend,false,false,,false`
+		csvRespETag = "bigbums6969"
+
+		textResp = `bumfaces.net
+peepee.poopoo
+nothanks.com`
+		textRespETag = "this is a legit etag i swear"
+
+		jsonResp = `[
+  {
+    "domain": "bumfaces.net",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "big jerks"
+  },
+    {
+    "domain": "peepee.poopoo",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "harassment"
+  },
+    {
+    "domain": "nothanks.com",
+    "suspended_at": "2020-05-13T13:29:12.000Z"
+  }
+]`
+		jsonRespETag = "don't modify me daddy"
+	)
+
+	switch req.URL.String() {
+	case "https://lists.example.org/baddies.csv":
+		extraHeaders = map[string]string{"ETag": csvRespETag}
+		if req.Header.Get("If-None-Match") == csvRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(csvResp)
+			responseContentType = textCSV
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.txt":
+		extraHeaders = map[string]string{"ETag": textRespETag}
+		if req.Header.Get("If-None-Match") == textRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(textResp)
+			responseContentType = textPlain
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.json":
+		extraHeaders = map[string]string{"ETag": jsonRespETag}
+		if req.Header.Get("If-None-Match") == jsonRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(jsonResp)
+			responseContentType = applicationJSON
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	default:
+		responseCode = http.StatusNotFound
+		responseBytes = []byte(`{"error":"not found"}`)
+		responseContentType = applicationJSON
+		responseContentLength = len(responseBytes)
+	}
+
+	return
+}