[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

3 files changed, 137 insertions, 0 deletions

diff --git a/internal/transport/derefdomainpermlist.go b/internal/transport/derefdomainpermlist.go
new file mode 100644
index 000000000..c81117bc6
--- /dev/null
+++ b/internal/transport/derefdomainpermlist.go
@@ -0,0 +1,121 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package transport
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type DereferenceDomainPermissionsResp struct {
+	// Set only if response was 200 OK.
+	// It's up to the caller to close
+	// this when they're done with it.
+	Body io.ReadCloser
+
+	// True if response
+	// was 304 Not Modified.
+	Unmodified bool
+
+	// May be set
+	// if 200 or 304.
+	ETag string
+}
+
+func (t *transport) DereferenceDomainPermissions(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	skipCache bool,
+) (*DereferenceDomainPermissionsResp, error) {
+	// Prepare new HTTP request to endpoint
+	req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set basic auth header if necessary.
+	if permSub.FetchUsername != "" || permSub.FetchPassword != "" {
+		req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword)
+	}
+
+	// Set relevant Accept headers.
+	// Allow fallback in case target doesn't
+	// negotiate content type correctly.
+	req.Header.Add("Accept-Charset", "utf-8")
+	req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*")
+
+	// If skipCache is true, we want to skip setting Cache
+	// headers so that we definitely don't get a 304 back.
+	if !skipCache {
+		// If we've successfully fetched this list
+		// before, set If-Modified-Since to last
+		// success to make the request conditional.
+		//
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
+		if !permSub.SuccessfullyFetchedAt.IsZero() {
+			timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat)
+			req.Header.Add("If-Modified-Since", timeStr)
+		}
+
+		// If we've got an ETag stored for this list, set
+		// If-None-Match to make the request conditional.
+		// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources.
+		if len(permSub.ETag) != 0 {
+			req.Header.Add("If-None-Match", permSub.ETag)
+		}
+	}
+
+	// Perform the HTTP request
+	rsp, err := t.GET(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have an unexpected / error response,
+	// wrap + return as error. This will also drain
+	// and close the response body for us.
+	if rsp.StatusCode != http.StatusOK &&
+		rsp.StatusCode != http.StatusNotModified {
+		err := gtserror.NewFromResponse(rsp)
+		return nil, err
+	}
+
+	// Check already if we were given an ETag
+	// we can use, as ETag is often returned
+	// even on 304 Not Modified responses.
+	permsResp := &DereferenceDomainPermissionsResp{
+		ETag: rsp.Header.Get("Etag"),
+	}
+
+	if rsp.StatusCode == http.StatusNotModified {
+		// Nothing has changed on the remote side
+		// since we last fetched, so there's nothing
+		// to do and we don't need to read the body.
+		rsp.Body.Close()
+		permsResp.Unmodified = true
+	} else {
+		// Return the live body to the caller.
+		permsResp.Body = rsp.Body
+	}
+
+	return permsResp, nil
+}
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
index 7f7e985fc..45d43ff18 100644
--- a/internal/transport/transport.go
+++ b/internal/transport/transport.go
@@ -78,6 +78,20 @@ type Transport interface {
 	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 
+	// DereferenceDomainPermissions dereferences the
+	// permissions list present at the given permSub's URI.
+	//
+	// If "force", then If-Modified-Since and If-None-Match
+	// headers will *NOT* be sent with the outgoing request.
+	//
+	// If err == nil and Unmodified == false, then it's up
+	// to the caller to close the returned io.ReadCloser.
+	DereferenceDomainPermissions(
+		ctx context.Context,
+		permSub *gtsmodel.DomainPermissionSubscription,
+		force bool,
+	) (*DereferenceDomainPermissionsResp, error)
+
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error)
 }
diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go
index 3a884d53f..c51c0755f 100644
--- a/internal/transport/transport_test.go
+++ b/internal/transport/transport_test.go
@@ -21,6 +21,7 @@ import (
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage