diff options
| -rw-r--r-- | docs/configuration/storage.md | 14 | ||||
| -rw-r--r-- | internal/cleaner/media.go | 2 | ||||
| -rw-r--r-- | internal/config/config.go | 1 | ||||
| -rw-r--r-- | internal/config/helpers.gen.go | 35 | ||||
| -rw-r--r-- | internal/storage/storage.go | 10 | ||||
| -rwxr-xr-x | test/envparsing.sh | 3 |
6 files changed, 64 insertions, 1 deletions
diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 04873ee36..0c5b1e9ba 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -101,6 +101,20 @@ storage-s3-secret-key: "" # Default: "" storage-s3-bucket: "" + +# String. Key prefix to use for the S3 storage. +# This is optional. +# +# Prefix must end with a trailing slash. +# +# This is useful if you want to store multiple instances in the same bucket, +# or if you want to store your data in a subdirectory of the bucket. +# This has no effect if the storage backend isn't "s3". +# +# Examples: ["gts-instance1/", "gts-instance2/"] +# Default: "" +storage-s3-key-prefix: "" + # String. Bucket lookup type. # # If you know what kind of bucket lookup type you need you can specify it here. diff --git a/internal/cleaner/media.go b/internal/cleaner/media.go index 6384ba368..043d7cf5f 100644 --- a/internal/cleaner/media.go +++ b/internal/cleaner/media.go @@ -20,6 +20,7 @@ package cleaner import ( "context" "errors" + "strings" "time" "code.superseriousbusiness.org/gotosocial/internal/db" @@ -96,6 +97,7 @@ func (m *Media) PruneOrphaned(ctx context.Context) (int, error) { // All media files in storage will have path fitting: {$account}/{$type}/{$size}/{$id}.{$ext} if err := m.state.Storage.WalkKeys(ctx, func(path string) error { // Check for our expected fileserver path format. + path = strings.TrimPrefix(path, m.state.Storage.KeyPrefix) if !regexes.FilePath.MatchString(path) { log.Warnf(ctx, "unexpected storage item: %s", path) return nil diff --git a/internal/config/config.go b/internal/config/config.go index f051ab005..d9293e062 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -135,6 +135,7 @@ type Configuration struct { StorageS3Proxy bool `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"` StorageS3RedirectURL string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."` StorageS3BucketLookup string `name:"storage-s3-bucket-lookup" usage:"S3 bucket lookup type to use. Can be 'auto', 'dns' or 'path'. Defaults to 'auto'."` + StorageS3KeyPrefix string `name:"storage-s3-key-prefix" usage:"Prefix to use for S3 keys. This is useful for separating multiple instances sharing the same S3 bucket."` StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"` StatusesPollMaxOptions int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"` diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 38f33c67e..0803bab08 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -104,6 +104,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) { flags.Bool("storage-s3-proxy", cfg.StorageS3Proxy, "Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL") flags.String("storage-s3-redirect-url", cfg.StorageS3RedirectURL, "Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL.") flags.String("storage-s3-bucket-lookup", cfg.StorageS3BucketLookup, "S3 bucket lookup type to use. Can be 'auto', 'dns' or 'path'. Defaults to 'auto'.") + flags.String("storage-s3-key-prefix", cfg.StorageS3KeyPrefix, "Prefix to use for S3 keys. This is useful for separating multiple instances sharing the same S3 bucket.") flags.Int("statuses-max-chars", cfg.StatusesMaxChars, "Max permitted characters for posted statuses, including content warning") flags.Int("statuses-poll-max-options", cfg.StatusesPollMaxOptions, "Max amount of options permitted on a poll") flags.Int("statuses-poll-option-max-chars", cfg.StatusesPollOptionMaxChars, "Max amount of characters for a poll option") @@ -285,6 +286,7 @@ func (cfg *Configuration) MarshalMap() map[string]any { cfgmap["storage-s3-proxy"] = cfg.StorageS3Proxy cfgmap["storage-s3-redirect-url"] = cfg.StorageS3RedirectURL cfgmap["storage-s3-bucket-lookup"] = cfg.StorageS3BucketLookup + cfgmap["storage-s3-key-prefix"] = cfg.StorageS3KeyPrefix cfgmap["statuses-max-chars"] = cfg.StatusesMaxChars cfgmap["statuses-poll-max-options"] = cfg.StatusesPollMaxOptions cfgmap["statuses-poll-option-max-chars"] = cfg.StatusesPollOptionMaxChars @@ -1029,6 +1031,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error { } } + if ival, ok := cfgmap["storage-s3-key-prefix"]; ok { + var err error + cfg.StorageS3KeyPrefix, err = cast.ToStringE(ival) + if err != nil { + return fmt.Errorf("error casting %#v -> string for 'storage-s3-key-prefix': %w", ival, err) + } + } + if ival, ok := cfgmap["statuses-max-chars"]; ok { var err error cfg.StatusesMaxChars, err = cast.ToIntE(ival) @@ -3793,6 +3803,31 @@ func GetStorageS3BucketLookup() string { return global.GetStorageS3BucketLookup( // SetStorageS3BucketLookup safely sets the value for global configuration 'StorageS3BucketLookup' field func SetStorageS3BucketLookup(v string) { global.SetStorageS3BucketLookup(v) } +// StorageS3KeyPrefixFlag returns the flag name for the 'StorageS3KeyPrefix' field +func StorageS3KeyPrefixFlag() string { return "storage-s3-key-prefix" } + +// GetStorageS3KeyPrefix safely fetches the Configuration value for state's 'StorageS3KeyPrefix' field +func (st *ConfigState) GetStorageS3KeyPrefix() (v string) { + st.mutex.RLock() + v = st.config.StorageS3KeyPrefix + st.mutex.RUnlock() + return +} + +// SetStorageS3KeyPrefix safely sets the Configuration value for state's 'StorageS3KeyPrefix' field +func (st *ConfigState) SetStorageS3KeyPrefix(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3KeyPrefix = v + st.reloadToViper() +} + +// GetStorageS3KeyPrefix safely fetches the value for global configuration 'StorageS3KeyPrefix' field +func GetStorageS3KeyPrefix() string { return global.GetStorageS3KeyPrefix() } + +// SetStorageS3KeyPrefix safely sets the value for global configuration 'StorageS3KeyPrefix' field +func SetStorageS3KeyPrefix(v string) { global.SetStorageS3KeyPrefix(v) } + // StatusesMaxCharsFlag returns the flag name for the 'StatusesMaxChars' field func StatusesMaxCharsFlag() string { return "statuses-max-chars" } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 780d2ca5d..32e9ccd27 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -78,27 +78,32 @@ type Driver struct { // S3-only parameters Proxy bool Bucket string + KeyPrefix string PresignedCache *ttl.Cache[string, PresignedURL] RedirectURL string } // Get returns the byte value for key in storage. func (d *Driver) Get(ctx context.Context, key string) ([]byte, error) { + key = d.KeyPrefix + key return d.Storage.ReadBytes(ctx, key) } // GetStream returns an io.ReadCloser for the value bytes at key in the storage. func (d *Driver) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + key = d.KeyPrefix + key return d.Storage.ReadStream(ctx, key) } // Put writes the supplied value bytes at key in the storage func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error) { + key = d.KeyPrefix + key return d.Storage.WriteBytes(ctx, key, value) } // PutFile moves the contents of file at path, to storage.Driver{} under given key (with content-type if supported). func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) (int64, error) { + key = d.KeyPrefix + key // Open file at path for reading. file, err := os.Open(filepath) if err != nil { @@ -144,11 +149,13 @@ func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) // Delete attempts to remove the supplied key (and corresponding value) from storage. func (d *Driver) Delete(ctx context.Context, key string) error { + key = d.KeyPrefix + key return d.Storage.Remove(ctx, key) } // Has checks if the supplied key is in the storage. func (d *Driver) Has(ctx context.Context, key string) (bool, error) { + key = d.KeyPrefix + key stat, err := d.Storage.Stat(ctx, key) return (stat != nil), err } @@ -156,6 +163,7 @@ func (d *Driver) Has(ctx context.Context, key string) (bool, error) { // WalkKeys walks the keys in the storage. func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error { return d.Storage.WalkKeys(ctx, storage.WalkKeysOpts{ + Prefix: d.KeyPrefix, Step: func(entry storage.Entry) error { return walk(entry.Key) }, @@ -164,6 +172,7 @@ func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error { // URL will return a presigned GET object URL, but only if running on S3 storage with proxying disabled. func (d *Driver) URL(ctx context.Context, key string) *PresignedURL { + key = d.KeyPrefix + key // Check whether S3 *without* proxying is enabled s3, ok := d.Storage.(*s3.S3Storage) if !ok || d.Proxy { @@ -349,6 +358,7 @@ func NewS3Storage() (*Driver, error) { return &Driver{ Proxy: config.GetStorageS3Proxy(), Bucket: config.GetStorageS3BucketName(), + KeyPrefix: config.GetStorageS3KeyPrefix(), Storage: s3, PresignedCache: presignedCache, RedirectURL: redirectURL, diff --git a/test/envparsing.sh b/test/envparsing.sh index 4fe13b6ac..3f8a55fda 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -187,6 +187,7 @@ EXPECT=$(cat << "EOF" "storage-s3-bucket": "gts", "storage-s3-bucket-lookup": "auto", "storage-s3-endpoint": "localhost:9000", + "storage-s3-key-prefix": "", "storage-s3-proxy": true, "storage-s3-redirect-url": "", "storage-s3-secret-key": "miniostorage", @@ -208,7 +209,7 @@ EXPECT=$(cat << "EOF" EOF ) -# Set all the environment variables to +# Set all the environment variables to # ensure that these are parsed without panic OUTPUT=$(GTS_LOG_LEVEL='info' \ GTS_LOG_TIMESTAMP_FORMAT="banana" \ |