summaryrefslogtreecommitdiff
path: root/vendor/codeberg.org/gruf/go-store/storage/disk.go
blob: 9b5430437c506d736410beedb8d0318e1cac04f7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
package storage

import (
	"io"
	"io/fs"
	"os"
	"path"
	"syscall"

	"codeberg.org/gruf/go-bytes"
	"codeberg.org/gruf/go-pools"
	"codeberg.org/gruf/go-store/util"
)

// DefaultDiskConfig is the default DiskStorage configuration
var DefaultDiskConfig = &DiskConfig{
	Overwrite:    true,
	WriteBufSize: 4096,
	Transform:    NopTransform(),
	Compression:  NoCompression(),
}

// DiskConfig defines options to be used when opening a DiskStorage
type DiskConfig struct {
	// Transform is the supplied key<-->path KeyTransform
	Transform KeyTransform

	// WriteBufSize is the buffer size to use when writing file streams (PutStream)
	WriteBufSize int

	// Overwrite allows overwriting values of stored keys in the storage
	Overwrite bool

	// Compression is the Compressor to use when reading / writing files, default is no compression
	Compression Compressor
}

// getDiskConfig returns a valid DiskConfig for supplied ptr
func getDiskConfig(cfg *DiskConfig) DiskConfig {
	// If nil, use default
	if cfg == nil {
		cfg = DefaultDiskConfig
	}

	// Assume nil transform == none
	if cfg.Transform == nil {
		cfg.Transform = NopTransform()
	}

	// Assume nil compress == none
	if cfg.Compression == nil {
		cfg.Compression = NoCompression()
	}

	// Assume 0 buf size == use default
	if cfg.WriteBufSize < 1 {
		cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize
	}

	// Return owned config copy
	return DiskConfig{
		Transform:    cfg.Transform,
		WriteBufSize: cfg.WriteBufSize,
		Overwrite:    cfg.Overwrite,
		Compression:  cfg.Compression,
	}
}

// DiskStorage is a Storage implementation that stores directly to a filesystem
type DiskStorage struct {
	path   string           // path is the root path of this store
	bufp   pools.BufferPool // bufp is the buffer pool for this DiskStorage
	config DiskConfig       // cfg is the supplied configuration for this store
	lock   *LockableFile    // lock is the opened lockfile for this storage instance
}

// OpenFile opens a DiskStorage instance for given folder path and configuration
func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) {
	// Acquire path builder
	pb := util.GetPathBuilder()
	defer util.PutPathBuilder(pb)

	// Clean provided path, ensure ends in '/' (should
	// be dir, this helps with file path trimming later)
	storePath := pb.Join(path, "store") + "/"

	// Get checked config
	config := getDiskConfig(cfg)

	// Attempt to open dir path
	file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms)
	if err != nil {
		// If not a not-exist error, return
		if !os.IsNotExist(err) {
			return nil, err
		}

		// Attempt to make store path dirs
		err = os.MkdirAll(storePath, defaultDirPerms)
		if err != nil {
			return nil, err
		}

		// Reopen dir now it's been created
		file, err = os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms)
		if err != nil {
			return nil, err
		}
	}
	defer file.Close()

	// Double check this is a dir (NOT a file!)
	stat, err := file.Stat()
	if err != nil {
		return nil, err
	} else if !stat.IsDir() {
		return nil, errPathIsFile
	}

	// Open and acquire storage lock for path
	lock, err := OpenLock(pb.Join(path, LockFile))
	if err != nil {
		return nil, err
	} else if err := lock.Lock(); err != nil {
		return nil, err
	}

	// Return new DiskStorage
	return &DiskStorage{
		path:   storePath,
		bufp:   pools.NewBufferPool(config.WriteBufSize),
		config: config,
		lock:   lock,
	}, nil
}

// Clean implements Storage.Clean()
func (st *DiskStorage) Clean() error {
	return util.CleanDirs(st.path)
}

// ReadBytes implements Storage.ReadBytes()
func (st *DiskStorage) ReadBytes(key string) ([]byte, error) {
	// Get stream reader for key
	rc, err := st.ReadStream(key)
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	// Read all bytes and return
	return io.ReadAll(rc)
}

// ReadStream implements Storage.ReadStream()
func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error) {
	// Get file path for key
	kpath, err := st.filepath(key)
	if err != nil {
		return nil, err
	}

	// Attempt to open file (replace ENOENT with our own)
	file, err := open(kpath, defaultFileROFlags)
	if err != nil {
		return nil, errSwapNotFound(err)
	}

	// Wrap the file in a compressor
	cFile, err := st.config.Compression.Reader(file)
	if err != nil {
		file.Close() // close this here, ignore error
		return nil, err
	}

	// Wrap compressor to ensure file close
	return util.ReadCloserWithCallback(cFile, func() {
		file.Close()
	}), nil
}

// WriteBytes implements Storage.WriteBytes()
func (st *DiskStorage) WriteBytes(key string, value []byte) error {
	return st.WriteStream(key, bytes.NewReader(value))
}

// WriteStream implements Storage.WriteStream()
func (st *DiskStorage) WriteStream(key string, r io.Reader) error {
	// Get file path for key
	kpath, err := st.filepath(key)
	if err != nil {
		return err
	}

	// Ensure dirs leading up to file exist
	err = os.MkdirAll(path.Dir(kpath), defaultDirPerms)
	if err != nil {
		return err
	}

	// Prepare to swap error if need-be
	errSwap := errSwapNoop

	// Build file RW flags
	flags := defaultFileRWFlags
	if !st.config.Overwrite {
		flags |= syscall.O_EXCL

		// Catch + replace err exist
		errSwap = errSwapExist
	}

	// Attempt to open file
	file, err := open(kpath, flags)
	if err != nil {
		return errSwap(err)
	}
	defer file.Close()

	// Wrap the file in a compressor
	cFile, err := st.config.Compression.Writer(file)
	if err != nil {
		return err
	}
	defer cFile.Close()

	// Acquire write buffer
	buf := st.bufp.Get()
	defer st.bufp.Put(buf)
	buf.Grow(st.config.WriteBufSize)

	// Copy reader to file
	_, err = io.CopyBuffer(cFile, r, buf.B)
	return err
}

// Stat implements Storage.Stat()
func (st *DiskStorage) Stat(key string) (bool, error) {
	// Get file path for key
	kpath, err := st.filepath(key)
	if err != nil {
		return false, err
	}

	// Check for file on disk
	return stat(kpath)
}

// Remove implements Storage.Remove()
func (st *DiskStorage) Remove(key string) error {
	// Get file path for key
	kpath, err := st.filepath(key)
	if err != nil {
		return err
	}

	// Attempt to remove file
	return os.Remove(kpath)
}

// Close implements Storage.Close()
func (st *DiskStorage) Close() error {
	defer st.lock.Close()
	return st.lock.Unlock()
}

// WalkKeys implements Storage.WalkKeys()
func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error {
	// Acquire path builder
	pb := util.GetPathBuilder()
	defer util.PutPathBuilder(pb)

	// Walk dir for entries
	return util.WalkDir(pb, st.path, func(kpath string, fsentry fs.DirEntry) {
		if fsentry.Type().IsRegular() {
			// Only deal with regular files

			// Get full item path (without root)
			kpath = pb.Join(kpath, fsentry.Name())[len(st.path):]

			// Perform provided walk function
			opts.WalkFn(entry(st.config.Transform.PathToKey(kpath)))
		}
	})
}

// filepath checks and returns a formatted filepath for given key
func (st *DiskStorage) filepath(key string) (string, error) {
	// Acquire path builder
	pb := util.GetPathBuilder()
	defer util.PutPathBuilder(pb)

	// Calculate transformed key path
	key = st.config.Transform.KeyToPath(key)

	// Generated joined root path
	pb.AppendString(st.path)
	pb.AppendString(key)

	// Check for dir traversal outside of root
	if util.IsDirTraversal(st.path, pb.StringPtr()) {
		return "", ErrInvalidKey
	}

	return pb.String(), nil
}