diff options
| author | 2022-11-24 08:35:46 +0000 | |
|---|---|---|
| committer | 2022-11-24 09:35:46 +0100 | |
| commit | fcb9c0bb8bed51ffb856b8e47f4e047ddd75eb67 (patch) | |
| tree | 933c9e1ed2b37e3ca2ee371c0b53f2c1ac561cc5 /internal/storage | |
| parent | [feature/performance] Fail fast when doing remote transport calls inside inco... (diff) | |
| download | gotosocial-fcb9c0bb8bed51ffb856b8e47f4e047ddd75eb67.tar.xz | |
[chore] cleanup storage implementation, no need for multiple interface types (#1131)
Signed-off-by: kim <grufwub@gmail.com>
Signed-off-by: kim <grufwub@gmail.com>
Diffstat (limited to 'internal/storage')
| -rw-r--r-- | internal/storage/local.go | 34 | ||||
| -rw-r--r-- | internal/storage/s3.go | 50 | ||||
| -rw-r--r-- | internal/storage/storage.go | 79 | 
3 files changed, 53 insertions, 110 deletions
| diff --git a/internal/storage/local.go b/internal/storage/local.go deleted file mode 100644 index 9a5f971a2..000000000 --- a/internal/storage/local.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -   GoToSocial -   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - -   This program is free software: you can redistribute it and/or modify -   it under the terms of the GNU Affero General Public License as published by -   the Free Software Foundation, either version 3 of the License, or -   (at your option) any later version. - -   This program is distributed in the hope that it will be useful, -   but WITHOUT ANY WARRANTY; without even the implied warranty of -   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -   GNU Affero General Public License for more details. - -   You should have received a copy of the GNU Affero General Public License -   along with this program.  If not, see <http://www.gnu.org/licenses/>. -*/ - -package storage - -import ( -	"context" -	"net/url" - -	"codeberg.org/gruf/go-store/v2/kv" -) - -type Local struct { -	*kv.KVStore -} - -func (l *Local) URL(ctx context.Context, key string) *url.URL { -	return nil -} diff --git a/internal/storage/s3.go b/internal/storage/s3.go deleted file mode 100644 index 1ead7efe9..000000000 --- a/internal/storage/s3.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -   GoToSocial -   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - -   This program is free software: you can redistribute it and/or modify -   it under the terms of the GNU Affero General Public License as published by -   the Free Software Foundation, either version 3 of the License, or -   (at your option) any later version. - -   This program is distributed in the hope that it will be useful, -   but WITHOUT ANY WARRANTY; without even the implied warranty of -   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -   GNU Affero General Public License for more details. - -   You should have received a copy of the GNU Affero General Public License -   along with this program.  If not, see <http://www.gnu.org/licenses/>. -*/ - -package storage - -import ( -	"context" -	"mime" -	"net/url" -	"path" -	"time" - -	"codeberg.org/gruf/go-store/v2/kv" -	"codeberg.org/gruf/go-store/v2/storage" -) - -type S3 struct { -	Proxy   bool -	Bucket  string -	Storage *storage.S3Storage -	*kv.KVStore -} - -func (s *S3) URL(ctx context.Context, key string) *url.URL { -	if s.Proxy { -		return nil -	} - -	// it's safe to ignore the error here, as we just fall back to fetching the file if URL request fails -	url, _ := s.Storage.Client().PresignedGetObject(ctx, s.Bucket, key, time.Hour, url.Values{ -		"response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, -	}) - -	return url -} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 71d4774f7..498ea873a 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -20,11 +20,11 @@ package storage  import (  	"context" -	"errors"  	"fmt" -	"io" +	"mime"  	"net/url"  	"path" +	"time"  	"codeberg.org/gruf/go-store/v2/kv"  	"codeberg.org/gruf/go-store/v2/storage" @@ -33,32 +33,50 @@ import (  	"github.com/superseriousbusiness/gotosocial/internal/config"  ) -var ( -	ErrNotSupported  = errors.New("driver does not suppport functionality") -	ErrAlreadyExists = storage.ErrAlreadyExists -) +// ErrAlreadyExists is a ptr to underlying storage.ErrAlreadyExists, +// to put the related errors in the same package as our storage wrapper. +var ErrAlreadyExists = storage.ErrAlreadyExists + +// Driver wraps a kv.KVStore to also provide S3 presigned GET URLs. +type Driver struct { +	// Underlying storage +	*kv.KVStore +	Storage storage.Storage -// Driver implements the functionality to store and retrieve blobs -// (images,video,audio) -type Driver interface { -	Get(ctx context.Context, key string) ([]byte, error) -	GetStream(ctx context.Context, key string) (io.ReadCloser, error) -	PutStream(ctx context.Context, key string, r io.Reader) error -	Put(ctx context.Context, key string, value []byte) error -	Delete(ctx context.Context, key string) error -	URL(ctx context.Context, key string) *url.URL +	// S3-only parameters +	Proxy  bool +	Bucket string  } -func AutoConfig() (Driver, error) { -	switch config.GetStorageBackend() { +// URL will return a presigned GET object URL, but only if running on S3 storage with proxying disabled. +func (d *Driver) URL(ctx context.Context, key string) *url.URL { +	// Check whether S3 *without* proxying is enabled +	s3, ok := d.Storage.(*storage.S3Storage) +	if !ok || d.Proxy { +		return nil +	} + +	// If URL request fails, fallback is to fetch the file. So ignore the error here +	url, _ := s3.Client().PresignedGetObject(ctx, d.Bucket, key, time.Hour, url.Values{ +		"response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, +	}) + +	return url +} + +func AutoConfig() (*Driver, error) { +	var st storage.Storage + +	switch backend := config.GetStorageBackend(); backend {  	case "s3": +		// Load runtime configuration  		endpoint := config.GetStorageS3Endpoint()  		access := config.GetStorageS3AccessKey()  		secret := config.GetStorageS3SecretKey()  		secure := config.GetStorageS3UseSSL()  		bucket := config.GetStorageS3BucketName() -		proxy := config.GetStorageS3Proxy() +		// Open the s3 storage implementation  		s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{  			CoreOpts: minio.Options{  				Creds:  credentials.NewStaticV4(access, secret, ""), @@ -75,15 +93,14 @@ func AutoConfig() (Driver, error) {  			return nil, fmt.Errorf("error opening s3 storage: %w", err)  		} -		return &S3{ -			Proxy:   proxy, -			Bucket:  bucket, -			Storage: s3, -			KVStore: kv.New(s3), -		}, nil +		// Set storage impl +		st = s3 +  	case "local": +		// Load runtime configuration  		basePath := config.GetStorageLocalBasePath() +		// Open the disk storage implementation  		disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{  			// Put the store lockfile in the storage dir itself.  			// Normally this would not be safe, since we could end up @@ -96,7 +113,17 @@ func AutoConfig() (Driver, error) {  			return nil, fmt.Errorf("error opening disk storage: %w", err)  		} -		return &Local{kv.New(disk)}, nil +		// Set storage impl +		st = disk + +	default: +		return nil, fmt.Errorf("invalid storage backend: %s", backend)  	} -	return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend()) + +	return &Driver{ +		KVStore: kv.New(st), +		Proxy:   config.GetStorageS3Proxy(), +		Bucket:  config.GetStorageS3BucketName(), +		Storage: st, +	}, nil  } | 
