diff options
Diffstat (limited to 'vendor/codeberg.org/gruf/go-storage/disk/disk.go')
-rw-r--r-- | vendor/codeberg.org/gruf/go-storage/disk/disk.go | 467 |
1 files changed, 0 insertions, 467 deletions
diff --git a/vendor/codeberg.org/gruf/go-storage/disk/disk.go b/vendor/codeberg.org/gruf/go-storage/disk/disk.go deleted file mode 100644 index b11346503..000000000 --- a/vendor/codeberg.org/gruf/go-storage/disk/disk.go +++ /dev/null @@ -1,467 +0,0 @@ -package disk - -import ( - "bytes" - "context" - "errors" - "io" - "io/fs" - "os" - "path" - "strings" - "syscall" - - "codeberg.org/gruf/go-fastcopy" - "codeberg.org/gruf/go-fastpath/v2" - "codeberg.org/gruf/go-storage" - "codeberg.org/gruf/go-storage/internal" -) - -// ensure DiskStorage conforms to storage.Storage. -var _ storage.Storage = (*DiskStorage)(nil) - -// DefaultConfig returns the default DiskStorage configuration. -func DefaultConfig() Config { - return defaultConfig -} - -// immutable default configuration. -var defaultConfig = Config{ - OpenRead: OpenArgs{syscall.O_RDONLY, 0o644}, - OpenWrite: OpenArgs{syscall.O_CREAT | syscall.O_WRONLY, 0o644}, - MkdirPerms: 0o755, - WriteBufSize: 4096, -} - -// OpenArgs defines args passed -// in a syscall.Open() operation. -type OpenArgs struct { - Flags int - Perms uint32 -} - -// Config defines options to be -// used when opening a DiskStorage. -type Config struct { - - // OpenRead are the arguments passed - // to syscall.Open() when opening a - // file for read operations. - OpenRead OpenArgs - - // OpenWrite are the arguments passed - // to syscall.Open() when opening a - // file for write operations. - OpenWrite OpenArgs - - // MkdirPerms are the permissions used - // when creating necessary sub-dirs in - // a storage key with slashes. - MkdirPerms uint32 - - // WriteBufSize is the buffer size - // to use when writing file streams. - WriteBufSize int -} - -// getDiskConfig returns valid (and owned!) Config for given ptr. -func getDiskConfig(cfg *Config) Config { - if cfg == nil { - // use defaults. - return defaultConfig - } - - // Ensure non-zero syscall args. - if cfg.OpenRead.Flags == 0 { - cfg.OpenRead.Flags = defaultConfig.OpenRead.Flags - } - if cfg.OpenRead.Perms == 0 { - cfg.OpenRead.Perms = defaultConfig.OpenRead.Perms - } - if cfg.OpenWrite.Flags == 0 { - cfg.OpenWrite.Flags = defaultConfig.OpenWrite.Flags - } - if cfg.OpenWrite.Perms == 0 { - cfg.OpenWrite.Perms = defaultConfig.OpenWrite.Perms - } - if cfg.MkdirPerms == 0 { - cfg.MkdirPerms = defaultConfig.MkdirPerms - } - - // Ensure valid write buf. - if cfg.WriteBufSize <= 0 { - cfg.WriteBufSize = defaultConfig.WriteBufSize - } - - return Config{ - OpenRead: cfg.OpenRead, - OpenWrite: cfg.OpenWrite, - MkdirPerms: cfg.MkdirPerms, - WriteBufSize: cfg.WriteBufSize, - } -} - -// DiskStorage is a Storage implementation -// that stores directly to a filesystem. -type DiskStorage struct { - path string // path is the root path of this store - pool fastcopy.CopyPool // pool is the prepared io copier with buffer pool - cfg Config // cfg is the supplied configuration for this store -} - -// Open opens a DiskStorage instance for given folder path and configuration. -func Open(path string, cfg *Config) (*DiskStorage, error) { - // Check + set config defaults. - config := getDiskConfig(cfg) - - // Clean provided storage path, ensure - // final '/' to help with path trimming. - pb := internal.GetPathBuilder() - path = pb.Clean(path) + "/" - internal.PutPathBuilder(pb) - - // Ensure directories up-to path exist. - perms := fs.FileMode(config.MkdirPerms) - err := os.MkdirAll(path, perms) - if err != nil { - return nil, err - } - - // Prepare DiskStorage. - st := &DiskStorage{ - path: path, - cfg: config, - } - - // Set fastcopy pool buffer size. - st.pool.Buffer(config.WriteBufSize) - - return st, nil -} - -// Clean: implements Storage.Clean(). -func (st *DiskStorage) Clean(ctx context.Context) error { - // Check context still valid. - if err := ctx.Err(); err != nil { - return err - } - - // Clean unused directories. - return cleanDirs(st.path, OpenArgs{ - Flags: syscall.O_RDONLY, - }) -} - -// ReadBytes: implements Storage.ReadBytes(). -func (st *DiskStorage) ReadBytes(ctx context.Context, key string) ([]byte, error) { - // Get stream reader for key - rc, err := st.ReadStream(ctx, key) - if err != nil { - return nil, err - } - - // Read all data to memory. - data, err := io.ReadAll(rc) - if err != nil { - _ = rc.Close() - return nil, err - } - - // Close storage stream reader. - if err := rc.Close(); err != nil { - return nil, err - } - - return data, nil -} - -// ReadStream: implements Storage.ReadStream(). -func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) { - // Generate file path for key. - kpath, err := st.Filepath(key) - if err != nil { - return nil, err - } - - // Check context still valid. - if err := ctx.Err(); err != nil { - return nil, err - } - - // Attempt to open file with read args. - file, err := open(kpath, st.cfg.OpenRead) - if err != nil { - - if err == syscall.ENOENT { - // Translate not-found errors and wrap with key. - err = internal.ErrWithKey(storage.ErrNotFound, key) - } - - return nil, err - } - - return file, nil -} - -// WriteBytes: implements Storage.WriteBytes(). -func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { - n, err := st.WriteStream(ctx, key, bytes.NewReader(value)) - return int(n), err -} - -// WriteStream: implements Storage.WriteStream(). -func (st *DiskStorage) WriteStream(ctx context.Context, key string, stream io.Reader) (int64, error) { - // Acquire path builder buffer. - pb := internal.GetPathBuilder() - - // Generate the file path for given key. - kpath, subdir, err := st.filepath(pb, key) - if err != nil { - return 0, err - } - - // Done with path buffer. - internal.PutPathBuilder(pb) - - // Check context still valid. - if err := ctx.Err(); err != nil { - return 0, err - } - - if subdir { - // Get dir of key path. - dir := path.Dir(kpath) - - // Note that subdir will only be set if - // the transformed key (without base path) - // contains any slashes. This is not a - // definitive check, but it allows us to - // skip a syscall if mkdirall not needed! - perms := fs.FileMode(st.cfg.MkdirPerms) - err = os.MkdirAll(dir, perms) - if err != nil { - return 0, err - } - } - - // Attempt to open file with write args. - file, err := open(kpath, st.cfg.OpenWrite) - if err != nil { - - if st.cfg.OpenWrite.Flags&syscall.O_EXCL != 0 && - err == syscall.EEXIST { - // Translate already exists errors and wrap with key. - err = internal.ErrWithKey(storage.ErrAlreadyExists, key) - } - - return 0, err - } - - // Copy provided stream to file interface. - n, err := st.pool.Copy(file, stream) - if err != nil { - _ = file.Close() - return n, err - } - - // Finally, close file. - return n, file.Close() -} - -// Stat implements Storage.Stat(). -func (st *DiskStorage) Stat(ctx context.Context, key string) (*storage.Entry, error) { - // Generate file path for key. - kpath, err := st.Filepath(key) - if err != nil { - return nil, err - } - - // Check context still valid. - if err := ctx.Err(); err != nil { - return nil, err - } - - // Stat file on disk. - stat, err := stat(kpath) - if stat == nil { - return nil, err - } - - return &storage.Entry{ - Key: key, - Size: stat.Size, - }, nil -} - -// Remove implements Storage.Remove(). -func (st *DiskStorage) Remove(ctx context.Context, key string) error { - // Generate file path for key. - kpath, err := st.Filepath(key) - if err != nil { - return err - } - - // Check context still valid. - if err := ctx.Err(); err != nil { - return err - } - - // Stat file on disk. - stat, err := stat(kpath) - if err != nil { - return err - } - - // Not-found (or handled - // as) error situations. - if stat == nil { - return internal.ErrWithKey(storage.ErrNotFound, key) - } else if stat.Mode&syscall.S_IFREG == 0 { - err := errors.New("storage/disk: not a regular file") - return internal.ErrWithKey(err, key) - } - - // Remove at path (we know this is file). - if err := unlink(kpath); err != nil { - - if err == syscall.ENOENT { - // Translate not-found errors and wrap with key. - err = internal.ErrWithKey(storage.ErrNotFound, key) - } - - return err - } - - return nil -} - -// WalkKeys implements Storage.WalkKeys(). -func (st *DiskStorage) WalkKeys(ctx context.Context, opts storage.WalkKeysOpts) error { - if opts.Step == nil { - panic("nil step fn") - } - - // Check context still valid. - if err := ctx.Err(); err != nil { - return err - } - - // Acquire path builder for walk. - pb := internal.GetPathBuilder() - defer internal.PutPathBuilder(pb) - - // Dir to walk. - dir := st.path - - if opts.Prefix != "" { - // Convert key prefix to one of our storage filepaths. - pathprefix, subdir, err := st.filepath(pb, opts.Prefix) - if err != nil { - return internal.ErrWithMsg(err, "prefix error") - } - - if subdir { - // Note that subdir will only be set if - // the transformed key (without base path) - // contains any slashes. This is not a - // definitive check, but it allows us to - // update the directory we walk in case - // it might narrow search parameters! - dir = path.Dir(pathprefix) - } - - // Set updated storage - // path prefix in opts. - opts.Prefix = pathprefix - } - - // Only need to open dirs as read-only. - args := OpenArgs{Flags: syscall.O_RDONLY} - - return walkDir(pb, dir, args, func(kpath string, fsentry fs.DirEntry) error { - if !fsentry.Type().IsRegular() { - // Ignore anything but - // regular file types. - return nil - } - - // Get full item path (without root). - kpath = pb.Join(kpath, fsentry.Name()) - - // Perform a fast filter check against storage path prefix (if set). - if opts.Prefix != "" && !strings.HasPrefix(kpath, opts.Prefix) { - return nil // ignore - } - - // Storage key without base. - key := kpath[len(st.path):] - - // Ignore filtered keys. - if opts.Filter != nil && - !opts.Filter(key) { - return nil // ignore - } - - // Load file info. This should already - // be loaded due to the underlying call - // to os.File{}.ReadDir() populating them. - info, err := fsentry.Info() - if err != nil { - return err - } - - // Perform provided walk function - return opts.Step(storage.Entry{ - Key: key, - Size: info.Size(), - }) - }) -} - -// Filepath checks and returns a formatted Filepath for given key. -func (st *DiskStorage) Filepath(key string) (path string, err error) { - pb := internal.GetPathBuilder() - path, _, err = st.filepath(pb, key) - internal.PutPathBuilder(pb) - return -} - -// filepath performs the "meat" of Filepath(), returning also if path *may* be a subdir of base. -func (st *DiskStorage) filepath(pb *fastpath.Builder, key string) (path string, subdir bool, err error) { - // Fast check for whether this may be a - // sub-directory. This is not a definitive - // check, it's only for a fastpath check. - subdir = strings.ContainsRune(key, '/') - - // Build from base. - pb.Append(st.path) - pb.Append(key) - - // Take COPY of bytes. - path = string(pb.B) - - // Check for dir traversal outside base. - if isDirTraversal(st.path, path) { - err = internal.ErrWithKey(storage.ErrInvalidKey, key) - } - - return -} - -// isDirTraversal will check if rootPlusPath is a dir traversal outside of root, -// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath). -func isDirTraversal(root, rootPlusPath string) bool { - switch { - // Root is $PWD, check for traversal out of - case root == ".": - return strings.HasPrefix(rootPlusPath, "../") - - // The path MUST be prefixed by root - case !strings.HasPrefix(rootPlusPath, root): - return true - - // In all other cases, check not equal - default: - return len(root) == len(rootPlusPath) - } -} |