diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/storage/storage.go | 102 | ||||
| -rw-r--r-- | internal/storage/storage_common.go | 121 | ||||
| -rw-r--r-- | internal/storage/storage_nos3.go | 75 |
3 files changed, 202 insertions, 96 deletions
diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 12e2d8fec..2c47017b3 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -15,13 +15,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see <http://www.gnu.org/licenses/>. +//go:build !nos3 + package storage import ( "context" - "errors" "fmt" - "io" "mime" "net/url" "os" @@ -31,10 +31,8 @@ import ( "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/log" - "codeberg.org/gruf/go-bytesize" "codeberg.org/gruf/go-cache/v3/ttl" "codeberg.org/gruf/go-storage" - "codeberg.org/gruf/go-storage/disk" "codeberg.org/gruf/go-storage/s3" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -45,33 +43,11 @@ const ( urlCacheExpiryFrequency = time.Minute * 5 ) -// PresignedURL represents a pre signed S3 URL with -// an expiry time. -type PresignedURL struct { - *url.URL - Expiry time.Time // link expires at this time -} - -// IsInvalidKey returns whether error is an invalid-key -// type error returned by the underlying storage library. -func IsInvalidKey(err error) bool { - return errors.Is(err, storage.ErrInvalidKey) -} - -// IsAlreadyExist returns whether error is an already-exists -// type error returned by the underlying storage library. -func IsAlreadyExist(err error) bool { - return errors.Is(err, storage.ErrAlreadyExists) -} - -// IsNotFound returns whether error is a not-found error -// type returned by the underlying storage library. -func IsNotFound(err error) bool { - return errors.Is(err, storage.ErrNotFound) -} - -// Driver wraps a kv.KVStore to also provide S3 presigned GET URLs. +// Driver wraps a storage.Storage to +// provide optimized write operations +// and extra S3-specific utilities. type Driver struct { + // Underlying storage Storage storage.Storage @@ -82,21 +58,6 @@ type Driver struct { RedirectURL string } -// Get returns the byte value for key in storage. -func (d *Driver) Get(ctx context.Context, key string) ([]byte, error) { - return d.Storage.ReadBytes(ctx, key) -} - -// GetStream returns an io.ReadCloser for the value bytes at key in the storage. -func (d *Driver) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { - return d.Storage.ReadStream(ctx, key) -} - -// Put writes the supplied value bytes at key in the storage -func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error) { - return d.Storage.WriteBytes(ctx, key, value) -} - // PutFile moves the contents of file at path, to storage.Driver{} under given key (with content-type if supported). func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) (int64, error) { @@ -143,26 +104,6 @@ func (d *Driver) PutFile(ctx context.Context, key, filepath, contentType string) return sz, err } -// Delete attempts to remove the supplied key (and corresponding value) from storage. -func (d *Driver) Delete(ctx context.Context, key string) error { - return d.Storage.Remove(ctx, key) -} - -// Has checks if the supplied key is in the storage. -func (d *Driver) Has(ctx context.Context, key string) (bool, error) { - stat, err := d.Storage.Stat(ctx, key) - return (stat != nil), err -} - -// WalkKeys walks the keys in the storage. -func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error { - return d.Storage.WalkKeys(ctx, storage.WalkKeysOpts{ - Step: func(entry storage.Entry) error { - return walk(entry.Key) - }, - }) -} - // URL will return a presigned GET object URL, but only if running on S3 storage with proxying disabled. func (d *Driver) URL(ctx context.Context, key string) *PresignedURL { @@ -277,37 +218,6 @@ func (d *Driver) ProbeCSPUri(ctx context.Context) (string, error) { return uStripped.String(), nil } -func AutoConfig() (*Driver, error) { - switch backend := config.GetStorageBackend(); backend { - case "s3": - return NewS3Storage() - case "local": - return NewFileStorage() - default: - return nil, fmt.Errorf("invalid storage backend: %s", backend) - } -} - -func NewFileStorage() (*Driver, error) { - // Load runtime configuration - basePath := config.GetStorageLocalBasePath() - - // Use default disk config with - // increased write buffer size. - diskCfg := disk.DefaultConfig() - diskCfg.WriteBufSize = int(16 * bytesize.KiB) - - // Open the disk storage implementation - disk, err := disk.Open(basePath, &diskCfg) - if err != nil { - return nil, fmt.Errorf("error opening disk storage: %w", err) - } - - return &Driver{ - Storage: disk, - }, nil -} - func NewS3Storage() (*Driver, error) { // Load runtime configuration endpoint := config.GetStorageS3Endpoint() diff --git a/internal/storage/storage_common.go b/internal/storage/storage_common.go new file mode 100644 index 000000000..3e1311525 --- /dev/null +++ b/internal/storage/storage_common.go @@ -0,0 +1,121 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "time" + + "code.superseriousbusiness.org/gotosocial/internal/config" + "codeberg.org/gruf/go-bytesize" + "codeberg.org/gruf/go-storage" + "codeberg.org/gruf/go-storage/disk" +) + +// PresignedURL represents a pre signed S3 URL with +// an expiry time. +type PresignedURL struct { + *url.URL + Expiry time.Time // link expires at this time +} + +// IsInvalidKey returns whether error is an invalid-key +// type error returned by the underlying storage library. +func IsInvalidKey(err error) bool { + return errors.Is(err, storage.ErrInvalidKey) +} + +// IsAlreadyExist returns whether error is an already-exists +// type error returned by the underlying storage library. +func IsAlreadyExist(err error) bool { + return errors.Is(err, storage.ErrAlreadyExists) +} + +// IsNotFound returns whether error is a not-found error +// type returned by the underlying storage library. +func IsNotFound(err error) bool { + return errors.Is(err, storage.ErrNotFound) +} + +// Get returns the byte value for key in storage. +func (d *Driver) Get(ctx context.Context, key string) ([]byte, error) { + return d.Storage.ReadBytes(ctx, key) +} + +// GetStream returns an io.ReadCloser for the value bytes at key in the storage. +func (d *Driver) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + return d.Storage.ReadStream(ctx, key) +} + +// Put writes the supplied value bytes at key in the storage +func (d *Driver) Put(ctx context.Context, key string, value []byte) (int, error) { + return d.Storage.WriteBytes(ctx, key, value) +} + +// Delete attempts to remove the supplied key (and corresponding value) from storage. +func (d *Driver) Delete(ctx context.Context, key string) error { + return d.Storage.Remove(ctx, key) +} + +// Has checks if the supplied key is in the storage. +func (d *Driver) Has(ctx context.Context, key string) (bool, error) { + stat, err := d.Storage.Stat(ctx, key) + return (stat != nil), err +} + +// WalkKeys walks the keys in the storage. +func (d *Driver) WalkKeys(ctx context.Context, walk func(string) error) error { + return d.Storage.WalkKeys(ctx, storage.WalkKeysOpts{ + Step: func(entry storage.Entry) error { + return walk(entry.Key) + }, + }) +} + +func AutoConfig() (*Driver, error) { + switch backend := config.GetStorageBackend(); backend { + case "s3": + return NewS3Storage() + case "local": + return NewFileStorage() + default: + return nil, fmt.Errorf("invalid storage backend: %s", backend) + } +} + +func NewFileStorage() (*Driver, error) { + // Load runtime configuration + basePath := config.GetStorageLocalBasePath() + + // Use default disk config with + // increased write buffer size. + diskCfg := disk.DefaultConfig() + diskCfg.WriteBufSize = int(16 * bytesize.KiB) + + // Open the disk storage implementation + disk, err := disk.Open(basePath, &diskCfg) + if err != nil { + return nil, fmt.Errorf("error opening disk storage: %w", err) + } + + return &Driver{Storage: disk}, nil +} diff --git a/internal/storage/storage_nos3.go b/internal/storage/storage_nos3.go new file mode 100644 index 000000000..73a95d50f --- /dev/null +++ b/internal/storage/storage_nos3.go @@ -0,0 +1,75 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +//go:build nos3 + +package storage + +import ( + "context" + "errors" + "os" + + "code.superseriousbusiness.org/gotosocial/internal/gtserror" + "code.superseriousbusiness.org/gotosocial/internal/log" + "codeberg.org/gruf/go-storage/disk" +) + +// Driver wraps a disk.DiskStorage to +// provide optimized write operations. +type Driver struct{ Storage *disk.DiskStorage } + +// PutFile: see PutFile() in storage.go. +func (d *Driver) PutFile(ctx context.Context, key, filepath, _ string) (int64, error) { + + // Open file at path for reading. + file, err := os.Open(filepath) + if err != nil { + return 0, gtserror.Newf("error opening file %s: %w", filepath, err) + } + + // Write the file data to storage under key. Note + // that for disk.DiskStorage{} this should end up + // being a highly optimized Linux sendfile syscall. + sz, err := d.Storage.WriteStream(ctx, key, file) + + // Wrap write error. + if err != nil { + err = gtserror.Newf("error writing file %s: %w", key, err) + } + + // Close the file: done with it. + if e := file.Close(); e != nil { + log.Errorf(ctx, "error closing file %s: %v", filepath, e) + } + + return sz, err +} + +// URL: not implemented for 'nos3'. +func (d *Driver) URL(ctx context.Context, key string) *PresignedURL { + return nil +} + +// ProbeCSPUri: not implemented for 'nos3'. +func (d *Driver) ProbeCSPUri(ctx context.Context) (string, error) { + return "", nil +} + +func NewS3Storage() (*Driver, error) { + return nil, errors.New("gotosocial was compiled without S3 storage support") +} |