diff options
| -rw-r--r-- | cmd/gotosocial/action/server/server.go | 21 | ||||
| -rw-r--r-- | cmd/gotosocial/action/testrig/testrig.go | 8 | ||||
| -rw-r--r-- | internal/api/fileserver.go | 107 | ||||
| -rw-r--r-- | internal/api/fileserver/fileserver.go | 4 | ||||
| -rw-r--r-- | web/source/settings/admin/emoji/local/overview.js | 5 |
5 files changed, 115 insertions, 30 deletions
diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 537f338b1..de9b3b3f1 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -104,6 +104,13 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error creating instance instance: %s", err) } + // Get the instance account + // (we'll need this later). + instanceAccount, err := dbService.GetInstanceAccount(ctx, "") + if err != nil { + return fmt.Errorf("error retrieving instance account: %w", err) + } + // Open the storage backend storage, err := gtsstorage.AutoConfig() if err != nil { @@ -311,16 +318,17 @@ var Start action.GTSAction = func(ctx context.Context) error { // rate limiting rlLimit := config.GetAdvancedRateLimitRequests() rlExceptions := config.GetAdvancedRateLimitExceptions() - clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api - s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP) - fsLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates + clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api + s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP) + fsMainLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates + fsEmojiLimit := middleware.RateLimit(rlLimit*2, rlExceptions) // fileserver (emojis only, use high limit) // throttling cpuMultiplier := config.GetAdvancedThrottlingMultiplier() retryAfter := config.GetAdvancedThrottlingRetryAfter() clThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // client api s2sThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // server-to-server (AP) - fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates + fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates / emojis pkThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // throttle public key endpoint separately gzip := middleware.Gzip() // applied to all except fileserver @@ -330,12 +338,13 @@ var Start action.GTSAction = func(ctx context.Context) error { authModule.Route(router, clLimit, clThrottle, gzip) clientModule.Route(router, clLimit, clThrottle, gzip) metricsModule.Route(router, clLimit, clThrottle, gzip) - fileserverModule.Route(router, fsLimit, fsThrottle) + fileserverModule.Route(router, fsMainLimit, fsThrottle) + fileserverModule.RouteEmojis(router, instanceAccount.ID, fsEmojiLimit, fsThrottle) wellKnownModule.Route(router, gzip, s2sLimit, s2sThrottle) nodeInfoModule.Route(router, s2sLimit, s2sThrottle, gzip) activityPubModule.Route(router, s2sLimit, s2sThrottle, gzip) activityPubModule.RoutePublicKey(router, s2sLimit, pkThrottle, gzip) - webModule.Route(router, fsLimit, fsThrottle, gzip) + webModule.Route(router, fsMainLimit, fsThrottle, gzip) // Start the GoToSocial server. server := gotosocial.NewServer(dbService, router, cleaner) diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index d6bc92215..bf2c74f2f 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -83,6 +83,13 @@ var Start action.GTSAction = func(ctx context.Context) error { testrig.StandardDBSetup(state.DB, nil) + // Get the instance account + // (we'll need this later). + instanceAccount, err := state.DB.GetInstanceAccount(ctx, "") + if err != nil { + return fmt.Errorf("error retrieving instance account: %w", err) + } + if os.Getenv("GTS_STORAGE_BACKEND") == "s3" { var err error state.Storage, err = storage.NewS3Storage() @@ -225,6 +232,7 @@ var Start action.GTSAction = func(ctx context.Context) error { clientModule.Route(router) metricsModule.Route(router) fileserverModule.Route(router) + fileserverModule.RouteEmojis(router, instanceAccount.ID) wellKnownModule.Route(router) nodeInfoModule.Route(router) activityPubModule.Route(router) diff --git a/internal/api/fileserver.go b/internal/api/fileserver.go index 59f38c362..e0377b0e6 100644 --- a/internal/api/fileserver.go +++ b/internal/api/fileserver.go @@ -21,6 +21,7 @@ import ( "github.com/gin-gonic/gin" "github.com/superseriousbusiness/gotosocial/internal/api/fileserver" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/middleware" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/router" @@ -30,36 +31,98 @@ type Fileserver struct { fileserver *fileserver.Module } -func (f *Fileserver) Route(r *router.Router, m ...gin.HandlerFunc) { - fileserverGroup := r.AttachGroup("fileserver") - - // Attach middlewares appropriate for this group. - fileserverGroup.Use(m...) - // If we're using local storage or proxying s3, we can set a - // long max-age + immutable on all file requests to reflect - // that we never host different files at the same URL (since - // ULIDs are generated per piece of media), so we can - // easily prevent clients having to fetch files repeatedly. +// Attach cache middleware appropriate for file serving. +func useFSCacheMiddleware(grp *gin.RouterGroup) { + // If we're using local storage or proxying s3 (ie., serving + // from here) we can set a long max-age + immutable on file + // requests to reflect that we never host different files at + // the same URL (since ULIDs are generated per piece of media), + // so we can prevent clients having to fetch files repeatedly. // - // If we *are* using non-proxying s3, however, the max age - // must be set dynamically within the request handler, - // based on how long the signed URL has left to live before - // it expires. This ensures that clients won't cache expired - // links. This is done within fileserver/servefile.go, so we - // should not set the middleware here in that case. + // If we *are* using non-proxying s3, however (ie., not serving + // from here) the max age must be set dynamically within the + // request handler, based on how long the signed URL has left + // to live before it expires. This ensures that clients won't + // cache expired links. This is done within fileserver/servefile.go + // so we should not set the middleware here in that case. // // See: // // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching#avoiding_revalidation // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#immutable - if config.GetStorageBackend() == "local" || config.GetStorageS3Proxy() { - fileserverGroup.Use(middleware.CacheControl(middleware.CacheControlConfig{ - Directives: []string{"private", "max-age=604800", "immutable"}, - Vary: []string{"Range"}, // Cache partial ranges separately. - })) + servingFromHere := config.GetStorageBackend() == "local" || config.GetStorageS3Proxy() + if !servingFromHere { + return } - f.fileserver.Route(fileserverGroup.Handle) + grp.Use(middleware.CacheControl(middleware.CacheControlConfig{ + Directives: []string{"private", "max-age=604800", "immutable"}, + Vary: []string{"Range"}, // Cache partial ranges separately. + })) +} + +// Route the "main" fileserver group +// that handles everything except emojis. +func (f *Fileserver) Route( + r *router.Router, + m ...gin.HandlerFunc, +) { + const fsGroupPath = "fileserver" + + "/:" + fileserver.AccountIDKey + + "/:" + fileserver.MediaTypeKey + fsGroup := r.AttachGroup(fsGroupPath) + + // Attach provided + + // cache middlewares. + fsGroup.Use(m...) + useFSCacheMiddleware(fsGroup) + + f.fileserver.Route(fsGroup.Handle) +} + +// Route the "emojis" fileserver +// group to handle emojis specifically. +// +// instanceAccount ID is required because +// that is the ID under which all emoji +// files are stored, and from which all +// emoji file requests are therefore served. +func (f *Fileserver) RouteEmojis( + r *router.Router, + instanceAcctID string, + m ...gin.HandlerFunc, +) { + var fsEmojiGroupPath = "fileserver" + + "/" + instanceAcctID + + "/" + string(media.TypeEmoji) + fsEmojiGroup := r.AttachGroup(fsEmojiGroupPath) + + // Inject the instance account and emoji media + // type params into the gin context manually, + // since we know we're only going to be serving + // emojis (stored under the instance account ID) + // from this group. This allows us to use the + // same handler functions for both the "main" + // fileserver handler and the emojis handler. + fsEmojiGroup.Use(func(c *gin.Context) { + c.Params = append(c.Params, []gin.Param{ + { + Key: fileserver.AccountIDKey, + Value: instanceAcctID, + }, + { + Key: fileserver.MediaTypeKey, + Value: string(media.TypeEmoji), + }, + }...) + }) + + // Attach provided + + // cache middlewares. + fsEmojiGroup.Use(m...) + useFSCacheMiddleware(fsEmojiGroup) + + f.fileserver.Route(fsEmojiGroup.Handle) } func NewFileserver(p *processing.Processor) *Fileserver { diff --git a/internal/api/fileserver/fileserver.go b/internal/api/fileserver/fileserver.go index 3620ea63f..db15ce2e0 100644 --- a/internal/api/fileserver/fileserver.go +++ b/internal/api/fileserver/fileserver.go @@ -33,8 +33,8 @@ const ( MediaSizeKey = "media_size" // FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg FileNameKey = "file_name" - // FileServePath is the fileserve path minus the 'fileserver' prefix. - FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey + // FileServePath is the fileserve path minus the 'fileserver/:account_id/:media_type' prefix. + FileServePath = "/:" + MediaSizeKey + "/:" + FileNameKey ) type Module struct { diff --git a/web/source/settings/admin/emoji/local/overview.js b/web/source/settings/admin/emoji/local/overview.js index 757f07c43..44b11f584 100644 --- a/web/source/settings/admin/emoji/local/overview.js +++ b/web/source/settings/admin/emoji/local/overview.js @@ -64,6 +64,11 @@ module.exports = function EmojiOverview({ }) { You can either upload them here directly, or copy from those already present on other (known) instances through the <Link to={`./remote`}>Remote Emoji</Link> page. </p> + <p> + <strong>Be warned!</strong> If you upload more than about 300-400 custom emojis in + total on your instance, this may lead to rate-limiting issues for users and clients + if they try to load all the emoji images at once (which is what many clients do). + </p> {content} </> ); |