diff options
Diffstat (limited to 'vendor/github.com/dsoprea/go-utility/v2/filesystem')
18 files changed, 1400 insertions, 0 deletions
diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md b/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md new file mode 100644 index 000000000..eb03fea7c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/README.md @@ -0,0 +1,64 @@ +[](https://godoc.org/github.com/dsoprea/go-utility/filesystem) +[](https://travis-ci.org/dsoprea/go-utility) +[](https://coveralls.io/github/dsoprea/go-utility?branch=master) +[](https://goreportcard.com/report/github.com/dsoprea/go-utility) + +# bounceback + +An `io.ReadSeeker` and `io.WriteSeeker` that returns to the right place before +reading or writing. Useful when the same file resource is being reused for reads +or writes throughout that file. + +# list_files + +A recursive path walker that supports filters. + +# seekable_buffer + +A memory structure that satisfies `io.ReadWriteSeeker`. + +# copy_bytes_between_positions + +Given an `io.ReadWriteSeeker`, copy N bytes from one position to an earlier +position. + +# read_counter, write_counter + +Wrap `io.Reader` and `io.Writer` structs in order to report how many bytes were +transferred. + +# readseekwritecloser + +Provides the ReadWriteSeekCloser interface that combines a RWS and a Closer. +Also provides a no-op wrapper to augment a plain RWS with a closer. + +# boundedreadwriteseek + +Wraps a ReadWriteSeeker such that no seeks can be at an offset less than a +specific-offset. + +# calculateseek + +Provides a reusable function with which to calculate seek offsets. + +# progress_wrapper + +Provides `io.Reader` and `io.Writer` wrappers that also trigger callbacks after +each call. The reader wrapper also invokes the callback upon EOF. + +# does_exist + +Check whether a file/directory exists using a file-path. + +# graceful_copy + +Do a copy but correctly handle short-writes and reads that might return a non- +zero read count *and* EOF. + +# readseeker_to_readerat + +A wrapper that allows an `io.ReadSeeker` to be used as a `io.ReaderAt`. + +# simplefileinfo + +An implementation of `os.FileInfo` to support testing. diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go new file mode 100644 index 000000000..1112a10ef --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/bounceback.go @@ -0,0 +1,273 @@ +package rifs + +import ( + "fmt" + "io" + + "github.com/dsoprea/go-logging" +) + +// BouncebackStats describes operation counts. +type BouncebackStats struct { + reads int + writes int + seeks int + syncs int +} + +func (bbs BouncebackStats) String() string { + return fmt.Sprintf( + "BouncebackStats<READS=(%d) WRITES=(%d) SEEKS=(%d) SYNCS=(%d)>", + bbs.reads, bbs.writes, bbs.seeks, bbs.syncs) +} + +type bouncebackBase struct { + currentPosition int64 + + stats BouncebackStats +} + +// Position returns the position that we're supposed to be at. +func (bb *bouncebackBase) Position() int64 { + + // TODO(dustin): Add test + + return bb.currentPosition +} + +// StatsReads returns the number of reads that have been attempted. +func (bb *bouncebackBase) StatsReads() int { + + // TODO(dustin): Add test + + return bb.stats.reads +} + +// StatsWrites returns the number of write operations. +func (bb *bouncebackBase) StatsWrites() int { + + // TODO(dustin): Add test + + return bb.stats.writes +} + +// StatsSeeks returns the number of seeks. +func (bb *bouncebackBase) StatsSeeks() int { + + // TODO(dustin): Add test + + return bb.stats.seeks +} + +// StatsSyncs returns the number of corrective seeks ("bounce-backs"). +func (bb *bouncebackBase) StatsSyncs() int { + + // TODO(dustin): Add test + + return bb.stats.syncs +} + +// Seek does a seek to an arbitrary place in the `io.ReadSeeker`. +func (bb *bouncebackBase) seek(s io.Seeker, offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // If the seek is relative, make sure we're where we're supposed to be *first*. + if whence != io.SeekStart { + err = bb.checkPosition(s) + log.PanicIf(err) + } + + bb.stats.seeks++ + + newPosition, err = s.Seek(offset, whence) + log.PanicIf(err) + + // Update our internal tracking. + bb.currentPosition = newPosition + + return newPosition, nil +} + +func (bb *bouncebackBase) checkPosition(s io.Seeker) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // Make sure we're where we're supposed to be. + + // This should have no overhead, and enables us to collect stats. + realCurrentPosition, err := s.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + if realCurrentPosition != bb.currentPosition { + bb.stats.syncs++ + + _, err = s.Seek(bb.currentPosition, io.SeekStart) + log.PanicIf(err) + } + + return nil +} + +// BouncebackReader wraps a ReadSeeker, keeps track of our position, and +// seeks back to it before writing. This allows an underlying ReadWriteSeeker +// with an unstable position can still be used for a prolonged series of writes. +type BouncebackReader struct { + rs io.ReadSeeker + + bouncebackBase +} + +// NewBouncebackReader returns a `*BouncebackReader` struct. +func NewBouncebackReader(rs io.ReadSeeker) (br *BouncebackReader, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + initialPosition, err := rs.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + bb := bouncebackBase{ + currentPosition: initialPosition, + } + + br = &BouncebackReader{ + rs: rs, + bouncebackBase: bb, + } + + return br, nil +} + +// Seek does a seek to an arbitrary place in the `io.ReadSeeker`. +func (br *BouncebackReader) Seek(offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newPosition, err = br.bouncebackBase.seek(br.rs, offset, whence) + log.PanicIf(err) + + return newPosition, nil +} + +// Seek does a standard read. +func (br *BouncebackReader) Read(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + br.bouncebackBase.stats.reads++ + + err = br.bouncebackBase.checkPosition(br.rs) + log.PanicIf(err) + + // Do read. + + n, err = br.rs.Read(p) + if err != nil { + if err == io.EOF { + return 0, io.EOF + } + + log.Panic(err) + } + + // Update our internal tracking. + br.bouncebackBase.currentPosition += int64(n) + + return n, nil +} + +// BouncebackWriter wraps a WriteSeeker, keeps track of our position, and +// seeks back to it before writing. This allows an underlying ReadWriteSeeker +// with an unstable position can still be used for a prolonged series of writes. +type BouncebackWriter struct { + ws io.WriteSeeker + + bouncebackBase +} + +// NewBouncebackWriter returns a new `BouncebackWriter` struct. +func NewBouncebackWriter(ws io.WriteSeeker) (bw *BouncebackWriter, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + initialPosition, err := ws.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + bb := bouncebackBase{ + currentPosition: initialPosition, + } + + bw = &BouncebackWriter{ + ws: ws, + bouncebackBase: bb, + } + + return bw, nil +} + +// Seek puts us at a specific position in the internal writer for the next +// write/seek. +func (bw *BouncebackWriter) Seek(offset int64, whence int) (newPosition int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newPosition, err = bw.bouncebackBase.seek(bw.ws, offset, whence) + log.PanicIf(err) + + return newPosition, nil +} + +// Write performs a write against the internal `WriteSeeker` starting at the +// position that we're supposed to be at. +func (bw *BouncebackWriter) Write(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + bw.bouncebackBase.stats.writes++ + + // Make sure we're where we're supposed to be. + + realCurrentPosition, err := bw.ws.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + if realCurrentPosition != bw.bouncebackBase.currentPosition { + bw.bouncebackBase.stats.seeks++ + + _, err = bw.ws.Seek(bw.bouncebackBase.currentPosition, io.SeekStart) + log.PanicIf(err) + } + + // Do write. + + n, err = bw.ws.Write(p) + log.PanicIf(err) + + // Update our internal tracking. + bw.bouncebackBase.currentPosition += int64(n) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go new file mode 100644 index 000000000..3d2e840fa --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseekcloser.go @@ -0,0 +1,95 @@ +package rifs + +import ( + "io" + + "github.com/dsoprea/go-logging" +) + +// BoundedReadWriteSeekCloser wraps a RWS that is also a closer with boundaries. +// This proxies the RWS methods to the inner BRWS inside. +type BoundedReadWriteSeekCloser struct { + io.Closer + *BoundedReadWriteSeeker +} + +// NewBoundedReadWriteSeekCloser returns a new BoundedReadWriteSeekCloser. +func NewBoundedReadWriteSeekCloser(rwsc ReadWriteSeekCloser, minimumOffset int64, staticFileSize int64) (brwsc *BoundedReadWriteSeekCloser, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + bs, err := NewBoundedReadWriteSeeker(rwsc, minimumOffset, staticFileSize) + log.PanicIf(err) + + brwsc = &BoundedReadWriteSeekCloser{ + Closer: rwsc, + BoundedReadWriteSeeker: bs, + } + + return brwsc, nil +} + +// Seek forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + newOffset, err = rwsc.BoundedReadWriteSeeker.Seek(offset, whence) + log.PanicIf(err) + + return newOffset, nil +} + +// Read forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Read(buffer []byte) (readCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + readCount, err = rwsc.BoundedReadWriteSeeker.Read(buffer) + if err != nil { + if err == io.EOF { + return 0, err + } + + log.Panic(err) + } + + return readCount, nil +} + +// Write forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Write(buffer []byte) (writtenCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + writtenCount, err = rwsc.BoundedReadWriteSeeker.Write(buffer) + log.PanicIf(err) + + return writtenCount, nil +} + +// Close forwards calls to the inner RWS. +func (rwsc *BoundedReadWriteSeekCloser) Close() (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + err = rwsc.Closer.Close() + log.PanicIf(err) + + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go new file mode 100644 index 000000000..d29657b05 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/boundedreadwriteseeker.go @@ -0,0 +1,156 @@ +package rifs + +import ( + "errors" + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +var ( + // ErrSeekBeyondBound is returned when a seek is requested beyond the + // statically-given file-size. No writes or seeks beyond boundaries are + // supported with a statically-given file size. + ErrSeekBeyondBound = errors.New("seek beyond boundary") +) + +// BoundedReadWriteSeeker is a thin filter that ensures that no seeks can be done +// to offsets smaller than the one we were given. This supports libraries that +// might be expecting to read from the front of the stream being used on data +// that is in the middle of a stream instead. +type BoundedReadWriteSeeker struct { + io.ReadWriteSeeker + + currentOffset int64 + minimumOffset int64 + + staticFileSize int64 +} + +// NewBoundedReadWriteSeeker returns a new BoundedReadWriteSeeker instance. +func NewBoundedReadWriteSeeker(rws io.ReadWriteSeeker, minimumOffset int64, staticFileSize int64) (brws *BoundedReadWriteSeeker, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if minimumOffset < 0 { + log.Panicf("BoundedReadWriteSeeker minimum offset must be zero or larger: (%d)", minimumOffset) + } + + // We'll always started at a relative offset of zero. + _, err = rws.Seek(minimumOffset, os.SEEK_SET) + log.PanicIf(err) + + brws = &BoundedReadWriteSeeker{ + ReadWriteSeeker: rws, + + currentOffset: 0, + minimumOffset: minimumOffset, + + staticFileSize: staticFileSize, + } + + return brws, nil +} + +// Seek moves the offset to the given offset. Prevents offset from ever being +// moved left of `brws.minimumOffset`. +func (brws *BoundedReadWriteSeeker) Seek(offset int64, whence int) (updatedOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + fileSize := brws.staticFileSize + + // If we weren't given a static file-size, look it up whenever it is needed. + if whence == os.SEEK_END && fileSize == 0 { + realFileSizeRaw, err := brws.ReadWriteSeeker.Seek(0, os.SEEK_END) + log.PanicIf(err) + + fileSize = realFileSizeRaw - brws.minimumOffset + } + + updatedOffset, err = CalculateSeek(brws.currentOffset, offset, whence, fileSize) + log.PanicIf(err) + + if brws.staticFileSize != 0 && updatedOffset > brws.staticFileSize { + //updatedOffset = int64(brws.staticFileSize) + + // NOTE(dustin): Presumably, this will only be disruptive to writes that are beyond the boundaries, which, if we're being used at all, should already account for the boundary and prevent this error from ever happening. So, time will tell how disruptive this is. + return 0, ErrSeekBeyondBound + } + + if updatedOffset != brws.currentOffset { + updatedRealOffset := updatedOffset + brws.minimumOffset + + _, err = brws.ReadWriteSeeker.Seek(updatedRealOffset, os.SEEK_SET) + log.PanicIf(err) + + brws.currentOffset = updatedOffset + } + + return updatedOffset, nil +} + +// Read forwards writes to the inner RWS. +func (brws *BoundedReadWriteSeeker) Read(buffer []byte) (readCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if brws.staticFileSize != 0 { + availableCount := brws.staticFileSize - brws.currentOffset + if availableCount == 0 { + return 0, io.EOF + } + + if int64(len(buffer)) > availableCount { + buffer = buffer[:availableCount] + } + } + + readCount, err = brws.ReadWriteSeeker.Read(buffer) + brws.currentOffset += int64(readCount) + + if err != nil { + if err == io.EOF { + return 0, err + } + + log.Panic(err) + } + + return readCount, nil +} + +// Write forwards writes to the inner RWS. +func (brws *BoundedReadWriteSeeker) Write(buffer []byte) (writtenCount int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if brws.staticFileSize != 0 { + log.Panicf("writes can not be performed if a static file-size was given") + } + + writtenCount, err = brws.ReadWriteSeeker.Write(buffer) + brws.currentOffset += int64(writtenCount) + + log.PanicIf(err) + + return writtenCount, nil +} + +// MinimumOffset returns the configured minimum-offset. +func (brws *BoundedReadWriteSeeker) MinimumOffset() int64 { + return brws.minimumOffset +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go new file mode 100644 index 000000000..cd59d727c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/calculate_seek.go @@ -0,0 +1,52 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// SeekType is a convenience type to associate the different seek-types with +// printable descriptions. +type SeekType int + +// String returns a descriptive string. +func (n SeekType) String() string { + if n == io.SeekCurrent { + return "SEEK-CURRENT" + } else if n == io.SeekEnd { + return "SEEK-END" + } else if n == io.SeekStart { + return "SEEK-START" + } + + log.Panicf("unknown seek-type: (%d)", n) + return "" +} + +// CalculateSeek calculates an offset in a file-stream given the parameters. +func CalculateSeek(currentOffset int64, delta int64, whence int, fileSize int64) (finalOffset int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + finalOffset = 0 + } + }() + + if whence == os.SEEK_SET { + finalOffset = delta + } else if whence == os.SEEK_CUR { + finalOffset = currentOffset + delta + } else if whence == os.SEEK_END { + finalOffset = fileSize + delta + } else { + log.Panicf("whence not valid: (%d)", whence) + } + + if finalOffset < 0 { + finalOffset = 0 + } + + return finalOffset, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go new file mode 100644 index 000000000..256333d40 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/common.go @@ -0,0 +1,15 @@ +package rifs + +import ( + "os" + "path" +) + +var ( + appPath string +) + +func init() { + goPath := os.Getenv("GOPATH") + appPath = path.Join(goPath, "src", "github.com", "dsoprea", "go-utility", "filesystem") +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go new file mode 100644 index 000000000..89ee9a92c --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/copy_bytes_between_positions.go @@ -0,0 +1,40 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// CopyBytesBetweenPositions will copy bytes from one position in the given RWS +// to an earlier position in the same RWS. +func CopyBytesBetweenPositions(rws io.ReadWriteSeeker, fromPosition, toPosition int64, count int) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if fromPosition <= toPosition { + log.Panicf("from position (%d) must be larger than to position (%d)", fromPosition, toPosition) + } + + br, err := NewBouncebackReader(rws) + log.PanicIf(err) + + _, err = br.Seek(fromPosition, os.SEEK_SET) + log.PanicIf(err) + + bw, err := NewBouncebackWriter(rws) + log.PanicIf(err) + + _, err = bw.Seek(toPosition, os.SEEK_SET) + log.PanicIf(err) + + written, err := io.CopyN(bw, br, int64(count)) + log.PanicIf(err) + + n = int(written) + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go new file mode 100644 index 000000000..f5e6cd20a --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/does_exist.go @@ -0,0 +1,19 @@ +package rifs + +import ( + "os" +) + +// DoesExist returns true if we can open the given file/path without error. We +// can't simply use `os.IsNotExist()` because we'll get a different error when +// the parent directory doesn't exist, and really the only important thing is if +// it exists *and* it's readable. +func DoesExist(filepath string) bool { + f, err := os.Open(filepath) + if err != nil { + return false + } + + f.Close() + return true +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go new file mode 100644 index 000000000..8705e5fe0 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/graceful_copy.go @@ -0,0 +1,54 @@ +package rifs + +import ( + "fmt" + "io" +) + +const ( + defaultCopyBufferSize = 1024 * 1024 +) + +// GracefulCopy willcopy while enduring lesser normal issues. +// +// - We'll ignore EOF if the read byte-count is more than zero. Only an EOF when +// zero bytes were read will terminate the loop. +// +// - Ignore short-writes. If less bytes were written than the bytes that were +// given, we'll keep trying until done. +func GracefulCopy(w io.Writer, r io.Reader, buffer []byte) (copyCount int, err error) { + if buffer == nil { + buffer = make([]byte, defaultCopyBufferSize) + } + + for { + readCount, err := r.Read(buffer) + if err != nil { + if err != io.EOF { + err = fmt.Errorf("read error: %s", err.Error()) + return 0, err + } + + // Only break on EOF if no bytes were actually read. + if readCount == 0 { + break + } + } + + writeBuffer := buffer[:readCount] + + for len(writeBuffer) > 0 { + writtenCount, err := w.Write(writeBuffer) + if err != nil { + err = fmt.Errorf("write error: %s", err.Error()) + return 0, err + } + + writeBuffer = writeBuffer[writtenCount:] + } + + copyCount += readCount + } + + return copyCount, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go new file mode 100644 index 000000000..bcdbd67cb --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/list_files.go @@ -0,0 +1,143 @@ +package rifs + +import ( + "io" + "os" + "path" + + "github.com/dsoprea/go-logging" +) + +// FileListFilterPredicate is the callback predicate used for filtering. +type FileListFilterPredicate func(parent string, child os.FileInfo) (hit bool, err error) + +// VisitedFile is one visited file. +type VisitedFile struct { + Filepath string + Info os.FileInfo + Index int +} + +// ListFiles feeds a continuous list of files from a recursive folder scan. An +// optional predicate can be provided in order to filter. When done, the +// `filesC` channel is closed. If there's an error, the `errC` channel will +// receive it. +func ListFiles(rootPath string, cb FileListFilterPredicate) (filesC chan VisitedFile, count int, errC chan error) { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + log.Panic(err) + } + }() + + // Make sure the path exists. + + f, err := os.Open(rootPath) + log.PanicIf(err) + + f.Close() + + // Do our thing. + + filesC = make(chan VisitedFile, 100) + errC = make(chan error, 1) + index := 0 + + go func() { + defer func() { + if state := recover(); state != nil { + err := log.Wrap(state.(error)) + errC <- err + } + }() + + queue := []string{rootPath} + for len(queue) > 0 { + // Pop the next folder to process off the queue. + var thisPath string + thisPath, queue = queue[0], queue[1:] + + // Skip path if a symlink. + + fi, err := os.Lstat(thisPath) + log.PanicIf(err) + + if (fi.Mode() & os.ModeSymlink) > 0 { + continue + } + + // Read information. + + folderF, err := os.Open(thisPath) + if err != nil { + errC <- log.Wrap(err) + return + } + + // Iterate through children. + + for { + children, err := folderF.Readdir(1000) + if err == io.EOF { + break + } else if err != nil { + errC <- log.Wrap(err) + return + } + + for _, child := range children { + filepath := path.Join(thisPath, child.Name()) + + // Skip if a file symlink. + + fi, err := os.Lstat(filepath) + log.PanicIf(err) + + if (fi.Mode() & os.ModeSymlink) > 0 { + continue + } + + // If a predicate was given, determine if this child will be + // left behind. + if cb != nil { + hit, err := cb(thisPath, child) + + if err != nil { + errC <- log.Wrap(err) + return + } + + if hit == false { + continue + } + } + + index++ + + // Push file to channel. + + vf := VisitedFile{ + Filepath: filepath, + Info: child, + Index: index, + } + + filesC <- vf + + // If a folder, queue for later processing. + + if child.IsDir() == true { + queue = append(queue, filepath) + } + } + } + + folderF.Close() + } + + close(filesC) + close(errC) + }() + + return filesC, index, errC +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go new file mode 100644 index 000000000..0a064c53d --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/progress_wrapper.go @@ -0,0 +1,93 @@ +package rifs + +import ( + "io" + "time" + + "github.com/dsoprea/go-logging" +) + +// ProgressFunc receives progress updates. +type ProgressFunc func(n int, duration time.Duration, isEof bool) error + +// WriteProgressWrapper wraps a reader and calls a callback after each read with +// count and duration info. +type WriteProgressWrapper struct { + w io.Writer + progressCb ProgressFunc +} + +// NewWriteProgressWrapper returns a new WPW instance. +func NewWriteProgressWrapper(w io.Writer, progressCb ProgressFunc) io.Writer { + return &WriteProgressWrapper{ + w: w, + progressCb: progressCb, + } +} + +// Write does a write and calls the callback. +func (wpw *WriteProgressWrapper) Write(buffer []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + startAt := time.Now() + + n, err = wpw.w.Write(buffer) + log.PanicIf(err) + + duration := time.Since(startAt) + + err = wpw.progressCb(n, duration, false) + log.PanicIf(err) + + return n, nil +} + +// ReadProgressWrapper wraps a reader and calls a callback after each read with +// count and duration info. +type ReadProgressWrapper struct { + r io.Reader + progressCb ProgressFunc +} + +// NewReadProgressWrapper returns a new RPW instance. +func NewReadProgressWrapper(r io.Reader, progressCb ProgressFunc) io.Reader { + return &ReadProgressWrapper{ + r: r, + progressCb: progressCb, + } +} + +// Read reads data and calls the callback. +func (rpw *ReadProgressWrapper) Read(buffer []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + startAt := time.Now() + + n, err = rpw.r.Read(buffer) + + duration := time.Since(startAt) + + if err != nil { + if err == io.EOF { + errInner := rpw.progressCb(n, duration, true) + log.PanicIf(errInner) + + return n, err + } + + log.Panic(err) + } + + err = rpw.progressCb(n, duration, false) + log.PanicIf(err) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go new file mode 100644 index 000000000..d878ca4e6 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/read_counter.go @@ -0,0 +1,36 @@ +package rifs + +import ( + "io" +) + +// ReadCounter proxies read requests and maintains a counter of bytes read. +type ReadCounter struct { + r io.Reader + counter int +} + +// NewReadCounter returns a new `ReadCounter` struct wrapping a `Reader`. +func NewReadCounter(r io.Reader) *ReadCounter { + return &ReadCounter{ + r: r, + } +} + +// Count returns the total number of bytes read. +func (rc *ReadCounter) Count() int { + return rc.counter +} + +// Reset resets the counter to zero. +func (rc *ReadCounter) Reset() { + rc.counter = 0 +} + +// Read forwards a read to the underlying `Reader` while bumping the counter. +func (rc *ReadCounter) Read(b []byte) (n int, err error) { + n, err = rc.r.Read(b) + rc.counter += n + + return n, err +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go new file mode 100644 index 000000000..3f3ec44dd --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readseeker_to_readerat.go @@ -0,0 +1,63 @@ +package rifs + +import ( + "io" + + "github.com/dsoprea/go-logging" +) + +// ReadSeekerToReaderAt is a wrapper that allows a ReadSeeker to masquerade as a +// ReaderAt. +type ReadSeekerToReaderAt struct { + rs io.ReadSeeker +} + +// NewReadSeekerToReaderAt returns a new ReadSeekerToReaderAt instance. +func NewReadSeekerToReaderAt(rs io.ReadSeeker) *ReadSeekerToReaderAt { + return &ReadSeekerToReaderAt{ + rs: rs, + } +} + +// ReadAt is a wrapper that satisfies the ReaderAt interface. +// +// Note that a requirement of ReadAt is that it doesn't have an effect on the +// offset in the underlying resource as well as that concurrent calls can be +// made to it. Since we're capturing the current offset in the underlying +// resource and then seeking back to it before returning, it is the +// responsibility of the caller to serialize (i.e. use a mutex with) these +// requests in order to eliminate race-conditions in the parallel-usage +// scenario. +// +// Note also that, since ReadAt() is going to be called on a particular +// instance, that instance is going to internalize a file resource, that file- +// resource is provided by the OS, and [most] OSs are only gonna support one +// file-position per resource, locking is already going to be a necessary +// internal semantic of a ReaderAt implementation. +func (rstra *ReadSeekerToReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + originalOffset, err := rstra.rs.Seek(0, io.SeekCurrent) + log.PanicIf(err) + + defer func() { + _, err := rstra.rs.Seek(originalOffset, io.SeekStart) + log.PanicIf(err) + }() + + _, err = rstra.rs.Seek(offset, io.SeekStart) + log.PanicIf(err) + + // Note that all errors will be wrapped, here. The usage of this method is + // such that typically no specific errors would be expected as part of + // normal operation (in which case we'd check for those first and return + // them directly). + n, err = io.ReadFull(rstra.rs, p) + log.PanicIf(err) + + return n, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go new file mode 100644 index 000000000..c583a8024 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/readwriteseekcloser.go @@ -0,0 +1,29 @@ +package rifs + +import ( + "io" +) + +// ReadWriteSeekCloser satisfies `io.ReadWriteSeeker` and `io.Closer` +// interfaces. +type ReadWriteSeekCloser interface { + io.ReadWriteSeeker + io.Closer +} + +type readWriteSeekNoopCloser struct { + io.ReadWriteSeeker +} + +// ReadWriteSeekNoopCloser wraps a `io.ReadWriteSeeker` with a no-op Close() +// call. +func ReadWriteSeekNoopCloser(rws io.ReadWriteSeeker) ReadWriteSeekCloser { + return readWriteSeekNoopCloser{ + ReadWriteSeeker: rws, + } +} + +// Close does nothing but allows the RWS to satisfy `io.Closer`.:wq +func (readWriteSeekNoopCloser) Close() (err error) { + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go new file mode 100644 index 000000000..5d41bb5df --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/seekable_buffer.go @@ -0,0 +1,146 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// SeekableBuffer is a simple memory structure that satisfies +// `io.ReadWriteSeeker`. +type SeekableBuffer struct { + data []byte + position int64 +} + +// NewSeekableBuffer is a factory that returns a `*SeekableBuffer`. +func NewSeekableBuffer() *SeekableBuffer { + data := make([]byte, 0) + + return &SeekableBuffer{ + data: data, + } +} + +// NewSeekableBufferWithBytes is a factory that returns a `*SeekableBuffer`. +func NewSeekableBufferWithBytes(originalData []byte) *SeekableBuffer { + data := make([]byte, len(originalData)) + copy(data, originalData) + + return &SeekableBuffer{ + data: data, + } +} + +func len64(data []byte) int64 { + return int64(len(data)) +} + +// Bytes returns the underlying slice. +func (sb *SeekableBuffer) Bytes() []byte { + return sb.data +} + +// Len returns the number of bytes currently stored. +func (sb *SeekableBuffer) Len() int { + return len(sb.data) +} + +// Write does a standard write to the internal slice. +func (sb *SeekableBuffer) Write(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + // The current position we're already at is past the end of the data we + // actually have. Extend our buffer up to our current position. + if sb.position > len64(sb.data) { + extra := make([]byte, sb.position-len64(sb.data)) + sb.data = append(sb.data, extra...) + } + + positionFromEnd := len64(sb.data) - sb.position + tailCount := positionFromEnd - len64(p) + + var tailBytes []byte + if tailCount > 0 { + tailBytes = sb.data[len64(sb.data)-tailCount:] + sb.data = append(sb.data[:sb.position], p...) + } else { + sb.data = append(sb.data[:sb.position], p...) + } + + if tailBytes != nil { + sb.data = append(sb.data, tailBytes...) + } + + dataSize := len64(p) + sb.position += dataSize + + return int(dataSize), nil +} + +// Read does a standard read against the internal slice. +func (sb *SeekableBuffer) Read(p []byte) (n int, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if sb.position >= len64(sb.data) { + return 0, io.EOF + } + + n = copy(p, sb.data[sb.position:]) + sb.position += int64(n) + + return n, nil +} + +// Truncate either chops or extends the internal buffer. +func (sb *SeekableBuffer) Truncate(size int64) (err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + sizeInt := int(size) + if sizeInt < len(sb.data)-1 { + sb.data = sb.data[:sizeInt] + } else { + new := make([]byte, sizeInt-len(sb.data)) + sb.data = append(sb.data, new...) + } + + return nil +} + +// Seek does a standard seek on the internal slice. +func (sb *SeekableBuffer) Seek(offset int64, whence int) (n int64, err error) { + defer func() { + if state := recover(); state != nil { + err = log.Wrap(state.(error)) + } + }() + + if whence == os.SEEK_SET { + sb.position = offset + } else if whence == os.SEEK_END { + sb.position = len64(sb.data) + offset + } else if whence == os.SEEK_CUR { + sb.position += offset + } else { + log.Panicf("seek whence is not valid: (%d)", whence) + } + + if sb.position < 0 { + sb.position = 0 + } + + return sb.position, nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go new file mode 100644 index 000000000..a227b0b00 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/simplefileinfo.go @@ -0,0 +1,69 @@ +package rifs + +import ( + "os" + "time" +) + +// SimpleFileInfo is a simple `os.FileInfo` implementation useful for testing +// with the bare minimum. +type SimpleFileInfo struct { + filename string + isDir bool + size int64 + mode os.FileMode + modTime time.Time +} + +// NewSimpleFileInfoWithFile returns a new file-specific SimpleFileInfo. +func NewSimpleFileInfoWithFile(filename string, size int64, mode os.FileMode, modTime time.Time) *SimpleFileInfo { + return &SimpleFileInfo{ + filename: filename, + isDir: false, + size: size, + mode: mode, + modTime: modTime, + } +} + +// NewSimpleFileInfoWithDirectory returns a new directory-specific +// SimpleFileInfo. +func NewSimpleFileInfoWithDirectory(filename string, modTime time.Time) *SimpleFileInfo { + return &SimpleFileInfo{ + filename: filename, + isDir: true, + mode: os.ModeDir, + modTime: modTime, + } +} + +// Name returns the base name of the file. +func (sfi *SimpleFileInfo) Name() string { + return sfi.filename +} + +// Size returns the length in bytes for regular files; system-dependent for +// others. +func (sfi *SimpleFileInfo) Size() int64 { + return sfi.size +} + +// Mode returns the file mode bits. +func (sfi *SimpleFileInfo) Mode() os.FileMode { + return sfi.mode +} + +// ModTime returns the modification time. +func (sfi *SimpleFileInfo) ModTime() time.Time { + return sfi.modTime +} + +// IsDir returns true if a directory. +func (sfi *SimpleFileInfo) IsDir() bool { + return sfi.isDir +} + +// Sys returns internal state. +func (sfi *SimpleFileInfo) Sys() interface{} { + return nil +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go new file mode 100644 index 000000000..4b33b41a9 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/utility.go @@ -0,0 +1,17 @@ +package rifs + +import ( + "io" + "os" + + "github.com/dsoprea/go-logging" +) + +// GetOffset returns the current offset of the Seeker and just panics if unable +// to find it. +func GetOffset(s io.Seeker) int64 { + offsetRaw, err := s.Seek(0, os.SEEK_CUR) + log.PanicIf(err) + + return offsetRaw +} diff --git a/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go b/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go new file mode 100644 index 000000000..dc39901d5 --- /dev/null +++ b/vendor/github.com/dsoprea/go-utility/v2/filesystem/write_counter.go @@ -0,0 +1,36 @@ +package rifs + +import ( + "io" +) + +// WriteCounter proxies write requests and maintains a counter of bytes written. +type WriteCounter struct { + w io.Writer + counter int +} + +// NewWriteCounter returns a new `WriteCounter` struct wrapping a `Writer`. +func NewWriteCounter(w io.Writer) *WriteCounter { + return &WriteCounter{ + w: w, + } +} + +// Count returns the total number of bytes read. +func (wc *WriteCounter) Count() int { + return wc.counter +} + +// Reset resets the counter to zero. +func (wc *WriteCounter) Reset() { + wc.counter = 0 +} + +// Write forwards a write to the underlying `Writer` while bumping the counter. +func (wc *WriteCounter) Write(b []byte) (n int, err error) { + n, err = wc.w.Write(b) + wc.counter += n + + return n, err +} |