From cc4f773b0e0f45cdb28727853b9d253234a93b56 Mon Sep 17 00:00:00 2001 From: Daenney Date: Fri, 7 Jun 2024 15:06:43 +0200 Subject: [chore] Update WASM go-sqlite3 to v0.16.1 (#2976) This includes support for journal mode set to WAL on the BSDs. Relates to: #1753, #2962 --- vendor/github.com/ncruces/go-sqlite3/go.work.sum | 8 +- .../go-sqlite3/internal/alloc/alloc_other.go | 9 + .../go-sqlite3/internal/alloc/alloc_slice.go | 24 ++ .../go-sqlite3/internal/alloc/alloc_unix.go | 67 ++++++ .../go-sqlite3/internal/alloc/alloc_windows.go | 76 ++++++ .../go-sqlite3/internal/util/alloc_other.go | 9 - .../go-sqlite3/internal/util/alloc_slice.go | 25 -- .../ncruces/go-sqlite3/internal/util/alloc_unix.go | 67 ------ .../go-sqlite3/internal/util/alloc_windows.go | 76 ------ .../ncruces/go-sqlite3/internal/util/const.go | 2 +- .../ncruces/go-sqlite3/internal/util/mmap.go | 5 +- .../ncruces/go-sqlite3/internal/util/mmap_other.go | 7 +- vendor/github.com/ncruces/go-sqlite3/stmt.go | 4 +- vendor/github.com/ncruces/go-sqlite3/value.go | 4 +- vendor/github.com/ncruces/go-sqlite3/vfs/README.md | 18 +- vendor/github.com/ncruces/go-sqlite3/vfs/api.go | 4 +- vendor/github.com/ncruces/go-sqlite3/vfs/const.go | 1 + vendor/github.com/ncruces/go-sqlite3/vfs/file.go | 3 + .../github.com/ncruces/go-sqlite3/vfs/os_linux.go | 2 +- .../ncruces/go-sqlite3/vfs/os_std_sleep.go | 9 - .../ncruces/go-sqlite3/vfs/os_windows.go | 15 +- vendor/github.com/ncruces/go-sqlite3/vfs/shm.go | 43 ++-- .../github.com/ncruces/go-sqlite3/vfs/shm_bsd.go | 259 +++++++++++++++++++++ .../github.com/ncruces/go-sqlite3/vfs/shm_other.go | 2 +- vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go | 12 +- 25 files changed, 500 insertions(+), 251 deletions(-) create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_other.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_slice.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_unix.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_windows.go delete mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_other.go delete mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_slice.go delete mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go delete mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go delete mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go (limited to 'vendor/github.com/ncruces/go-sqlite3') diff --git a/vendor/github.com/ncruces/go-sqlite3/go.work.sum b/vendor/github.com/ncruces/go-sqlite3/go.work.sum index 4deb7b7f3..27b395cc7 100644 --- a/vendor/github.com/ncruces/go-sqlite3/go.work.sum +++ b/vendor/github.com/ncruces/go-sqlite3/go.work.sum @@ -1,9 +1,7 @@ -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_other.go b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_other.go new file mode 100644 index 000000000..ded8da108 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_other.go @@ -0,0 +1,9 @@ +//go:build !(unix || windows) || sqlite3_nosys + +package alloc + +import "github.com/tetratelabs/wazero/experimental" + +func Virtual(cap, max uint64) experimental.LinearMemory { + return Slice(cap, max) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_slice.go b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_slice.go new file mode 100644 index 000000000..5072ca9c1 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_slice.go @@ -0,0 +1,24 @@ +//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys + +package alloc + +import "github.com/tetratelabs/wazero/experimental" + +func Slice(cap, _ uint64) experimental.LinearMemory { + return &sliceMemory{make([]byte, 0, cap)} +} + +type sliceMemory struct { + buf []byte +} + +func (b *sliceMemory) Free() {} + +func (b *sliceMemory) Reallocate(size uint64) []byte { + if cap := uint64(cap(b.buf)); size > cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_unix.go b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_unix.go new file mode 100644 index 000000000..39a3a38cc --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_unix.go @@ -0,0 +1,67 @@ +//go:build unix && !sqlite3_nosys + +package alloc + +import ( + "math" + + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/unix" +) + +func Virtual(_, max uint64) experimental.LinearMemory { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + max = (max + rnd) &^ rnd + + if max > math.MaxInt { + // This ensures int(max) overflows to a negative value, + // and unix.Mmap returns EINVAL. + max = math.MaxUint64 + } + + // Reserve max bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + panic(err) + } + return &mmappedMemory{buf: b[:0]} +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type mmappedMemory struct { + buf []byte +} + +func (m *mmappedMemory) Reallocate(size uint64) []byte { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size && size < res { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := (size + rnd) &^ rnd + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + panic(err) + } + + // Update committed memory. + m.buf = m.buf[:new] + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)] +} + +func (m *mmappedMemory) Free() { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + panic(err) + } + m.buf = nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_windows.go b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_windows.go new file mode 100644 index 000000000..27d875f2e --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/alloc/alloc_windows.go @@ -0,0 +1,76 @@ +//go:build !sqlite3_nosys + +package alloc + +import ( + "math" + "reflect" + "unsafe" + + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/windows" +) + +func Virtual(_, max uint64) experimental.LinearMemory { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + max = (max + rnd) &^ rnd + + if max > math.MaxInt { + // This ensures uintptr(max) overflows to a large value, + // and windows.VirtualAlloc returns an error. + max = math.MaxUint64 + } + + // Reserve max bytes of address space, to ensure we won't need to move it. + // This does not commit memory. + r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE) + if err != nil { + panic(err) + } + + mem := virtualMemory{addr: r} + // SliceHeader, although deprecated, avoids a go vet warning. + sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf)) + sh.Cap = int(max) + sh.Data = r + return &mem +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type virtualMemory struct { + buf []byte + addr uintptr +} + +func (m *virtualMemory) Reallocate(size uint64) []byte { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size && size < res { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + new := (size + rnd) &^ rnd + + // Commit additional memory up to new bytes. + _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) + if err != nil { + panic(err) + } + + // Update committed memory. + m.buf = m.buf[:new] + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)] +} + +func (m *virtualMemory) Free() { + err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) + if err != nil { + panic(err) + } + m.addr = 0 +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_other.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_other.go deleted file mode 100644 index ba16efc02..000000000 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_other.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !(unix || windows) || sqlite3_nosys - -package util - -import "github.com/tetratelabs/wazero/experimental" - -func virtualAlloc(cap, max uint64) experimental.LinearMemory { - return sliceAlloc(cap, max) -} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_slice.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_slice.go deleted file mode 100644 index b8cc1453c..000000000 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_slice.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys - -package util - -import "github.com/tetratelabs/wazero/experimental" - -func sliceAlloc(cap, max uint64) experimental.LinearMemory { - return &sliceBuffer{make([]byte, cap), max} -} - -type sliceBuffer struct { - buf []byte - max uint64 -} - -func (b *sliceBuffer) Free() {} - -func (b *sliceBuffer) Reallocate(size uint64) []byte { - if cap := uint64(cap(b.buf)); size > cap { - b.buf = append(b.buf[:cap], make([]byte, size-cap)...) - } else { - b.buf = b.buf[:size] - } - return b.buf -} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go deleted file mode 100644 index 2b1d3916b..000000000 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build unix && !sqlite3_nosys - -package util - -import ( - "math" - - "github.com/tetratelabs/wazero/experimental" - "golang.org/x/sys/unix" -) - -func virtualAlloc(cap, max uint64) experimental.LinearMemory { - // Round up to the page size. - rnd := uint64(unix.Getpagesize() - 1) - max = (max + rnd) &^ rnd - - if max > math.MaxInt { - // This ensures int(max) overflows to a negative value, - // and unix.Mmap returns EINVAL. - max = math.MaxUint64 - } - - // Reserve max bytes of address space, to ensure we won't need to move it. - // A protected, private, anonymous mapping should not commit memory. - b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON) - if err != nil { - panic(err) - } - return &mmappedMemory{buf: b[:0]} -} - -// The slice covers the entire mmapped memory: -// - len(buf) is the already committed memory, -// - cap(buf) is the reserved address space. -type mmappedMemory struct { - buf []byte -} - -func (m *mmappedMemory) Reallocate(size uint64) []byte { - com := uint64(len(m.buf)) - res := uint64(cap(m.buf)) - if com < size && size < res { - // Round up to the page size. - rnd := uint64(unix.Getpagesize() - 1) - new := (size + rnd) &^ rnd - - // Commit additional memory up to new bytes. - err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) - if err != nil { - panic(err) - } - - // Update committed memory. - m.buf = m.buf[:new] - } - // Limit returned capacity because bytes beyond - // len(m.buf) have not yet been committed. - return m.buf[:size:len(m.buf)] -} - -func (m *mmappedMemory) Free() { - err := unix.Munmap(m.buf[:cap(m.buf)]) - if err != nil { - panic(err) - } - m.buf = nil -} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go deleted file mode 100644 index 8936173b4..000000000 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build !sqlite3_nosys - -package util - -import ( - "math" - "reflect" - "unsafe" - - "github.com/tetratelabs/wazero/experimental" - "golang.org/x/sys/windows" -) - -func virtualAlloc(cap, max uint64) experimental.LinearMemory { - // Round up to the page size. - rnd := uint64(windows.Getpagesize() - 1) - max = (max + rnd) &^ rnd - - if max > math.MaxInt { - // This ensures uintptr(max) overflows to a large value, - // and windows.VirtualAlloc returns an error. - max = math.MaxUint64 - } - - // Reserve max bytes of address space, to ensure we won't need to move it. - // This does not commit memory. - r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE) - if err != nil { - panic(err) - } - - mem := virtualMemory{addr: r} - // SliceHeader, although deprecated, avoids a go vet warning. - sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf)) - sh.Cap = int(max) // Not a bug. - sh.Data = r - return &mem -} - -// The slice covers the entire mmapped memory: -// - len(buf) is the already committed memory, -// - cap(buf) is the reserved address space. -type virtualMemory struct { - buf []byte - addr uintptr -} - -func (m *virtualMemory) Reallocate(size uint64) []byte { - com := uint64(len(m.buf)) - res := uint64(cap(m.buf)) - if com < size && size < res { - // Round up to the page size. - rnd := uint64(windows.Getpagesize() - 1) - new := (size + rnd) &^ rnd - - // Commit additional memory up to new bytes. - _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) - if err != nil { - panic(err) - } - - // Update committed memory. - m.buf = m.buf[:new] - } - // Limit returned capacity because bytes beyond - // len(m.buf) have not yet been committed. - return m.buf[:size:len(m.buf)] -} - -func (m *virtualMemory) Free() { - err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) - if err != nil { - panic(err) - } - m.addr = 0 -} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go index 86bb9749d..5e89018dd 100644 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go @@ -1,6 +1,6 @@ package util -// https://sqlite.com/matrix/rescode.html +// https://sqlite.com/rescode.html const ( OK = 0 /* Successful result */ diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go index 6783c9612..a4fa2a25a 100644 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go @@ -1,4 +1,4 @@ -//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) +//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) package util @@ -7,6 +7,7 @@ import ( "os" "unsafe" + "github.com/ncruces/go-sqlite3/internal/alloc" "github.com/tetratelabs/wazero/api" "github.com/tetratelabs/wazero/experimental" "golang.org/x/sys/unix" @@ -14,7 +15,7 @@ import ( func withAllocator(ctx context.Context) context.Context { return experimental.WithMemoryAllocator(ctx, - experimental.MemoryAllocatorFunc(virtualAlloc)) + experimental.MemoryAllocatorFunc(alloc.Virtual)) } type mmapState struct { diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go index 1e81c9fd3..a0a3ba67d 100644 --- a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go @@ -1,10 +1,11 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys +//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys package util import ( "context" + "github.com/ncruces/go-sqlite3/internal/alloc" "github.com/tetratelabs/wazero/experimental" ) @@ -14,8 +15,8 @@ func withAllocator(ctx context.Context) context.Context { return experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory { if cap == max { - return virtualAlloc(cap, max) + return alloc.Virtual(cap, max) } - return sliceAlloc(cap, max) + return alloc.Slice(cap, max) })) } diff --git a/vendor/github.com/ncruces/go-sqlite3/stmt.go b/vendor/github.com/ncruces/go-sqlite3/stmt.go index 63c2085d0..ac40e3802 100644 --- a/vendor/github.com/ncruces/go-sqlite3/stmt.go +++ b/vendor/github.com/ncruces/go-sqlite3/stmt.go @@ -441,12 +441,12 @@ func (s *Stmt) ColumnOriginName(col int) string { // ColumnBool returns the value of the result column as a bool. // The leftmost column of the result set has the index 0. // SQLite does not have a separate boolean storage class. -// Instead, boolean values are retrieved as integers, +// Instead, boolean values are retrieved as numbers, // with 0 converted to false and any other value to true. // // https://sqlite.org/c3ref/column_blob.html func (s *Stmt) ColumnBool(col int) bool { - return s.ColumnInt64(col) != 0 + return s.ColumnFloat(col) != 0 } // ColumnInt returns the value of the result column as an int. diff --git a/vendor/github.com/ncruces/go-sqlite3/value.go b/vendor/github.com/ncruces/go-sqlite3/value.go index 61d3cbf70..d0edf215b 100644 --- a/vendor/github.com/ncruces/go-sqlite3/value.go +++ b/vendor/github.com/ncruces/go-sqlite3/value.go @@ -68,12 +68,12 @@ func (v Value) NumericType() Datatype { // Bool returns the value as a bool. // SQLite does not have a separate boolean storage class. -// Instead, boolean values are retrieved as integers, +// Instead, boolean values are retrieved as numbers, // with 0 converted to false and any other value to true. // // https://sqlite.org/c3ref/value_blob.html func (v Value) Bool() bool { - return v.Int64() != 0 + return v.Float() != 0 } // Int returns the value as an int. diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/README.md b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md index 88059a41b..741a1b6a4 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/README.md +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md @@ -46,7 +46,7 @@ to check if your build supports file locking. ### Write-Ahead Logging -On 64-bit Linux and macOS, this module uses `mmap` to implement +On 64-bit Unix, this module uses `mmap` to implement [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), like SQLite. @@ -54,6 +54,11 @@ To allow `mmap` to work, each connection needs to reserve up to 4GB of address s To limit the address space each connection reserves, use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go). +With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2) +a WAL database can only be accessed by a single proccess. +Other processes that attempt to access a database locked with BSD locks, +will fail with the `SQLITE_PROTOCOL` error code. + Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm), and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases. To use `EXCLUSIVE` locking mode with the @@ -79,8 +84,9 @@ The VFS can be customized with a few build tags: - `sqlite3_noshm` disables shared memory on all platforms. > [!IMPORTANT] -> The default configuration of this package is compatible with -> the standard [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses); -> `sqlite3_flock` is compatible with the [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style). -> If incompatible file locking is used, accessing databases concurrently with _other_ SQLite libraries -> will eventually corrupt data. +> The default configuration of this package is compatible with the standard +> [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses); +> `sqlite3_flock` builds are compatible with the +> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style). +> If incompatible file locking is used, accessing databases concurrently with +> _other_ SQLite libraries will eventually corrupt data. diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/api.go b/vendor/github.com/ncruces/go-sqlite3/vfs/api.go index 19c22ae8f..e133e8be9 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/api.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/api.go @@ -168,8 +168,8 @@ type FileSharedMemory interface { // SharedMemory is a shared-memory WAL-index implementation. // Use [NewSharedMemory] to create a shared-memory. type SharedMemory interface { - shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error) - shmLock(int32, int32, _ShmFlag) error + shmMap(context.Context, api.Module, int32, int32, bool) (uint32, _ErrorCode) + shmLock(int32, int32, _ShmFlag) _ErrorCode shmUnmap(bool) io.Closer } diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/const.go b/vendor/github.com/ncruces/go-sqlite3/vfs/const.go index 7f409f35f..f7217af96 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/const.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/const.go @@ -47,6 +47,7 @@ const ( _IOERR_SHMMAP _ErrorCode = util.IOERR_SHMMAP _IOERR_SEEK _ErrorCode = util.IOERR_SEEK _IOERR_DELETE_NOENT _ErrorCode = util.IOERR_DELETE_NOENT + _IOERR_GETTEMPPATH _ErrorCode = util.IOERR_GETTEMPPATH _IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC _IOERR_COMMIT_ATOMIC _ErrorCode = util.IOERR_COMMIT_ATOMIC _IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/file.go b/vendor/github.com/ncruces/go-sqlite3/vfs/file.go index ca8cf84f3..93a2f7ece 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/file.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/file.go @@ -95,6 +95,9 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error f, err = osutil.OpenFile(name.String(), oflags, 0666) } if err != nil { + if name == nil { + return nil, flags, _IOERR_GETTEMPPATH + } if errors.Is(err, syscall.EISDIR) { return nil, flags, _CANTOPEN_ISDIR } diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go index 11e683a04..7bb78c0af 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go @@ -56,7 +56,7 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d if timeout < time.Since(before) { break } - osSleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) + time.Sleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) } } return osLockErrorCode(err, def) diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go deleted file mode 100644 index c6bc40769..000000000 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !windows || sqlite3_nosys - -package vfs - -import "time" - -func osSleep(d time.Duration) { - time.Sleep(d) -} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go index 5c68754f8..83b952b16 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go @@ -136,7 +136,7 @@ func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def if timeout < time.Since(before) { break } - osSleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) + time.Sleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) } } return osLockErrorCode(err, def) @@ -171,16 +171,3 @@ func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { } return def } - -func osSleep(d time.Duration) { - if d > 0 { - period := max(1, d/(5*time.Millisecond)) - if period < 16 { - windows.TimeBeginPeriod(uint32(period)) - } - time.Sleep(d) - if period < 16 { - windows.TimeEndPeriod(uint32(period)) - } - } -} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go index 2b76dd5dc..58da34df4 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go @@ -51,12 +51,7 @@ type vfsShm struct { readOnly bool } -func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) { - // Ensure size is a multiple of the OS page size. - if int(size)&(unix.Getpagesize()-1) != 0 { - return 0, _IOERR_SHMMAP - } - +func (s *vfsShm) shmOpen() _ErrorCode { if s.File == nil { var flag int if s.readOnly { @@ -67,28 +62,40 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext f, err := os.OpenFile(s.path, flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666) if err != nil { - return 0, _CANTOPEN + return _CANTOPEN } s.File = f } // Dead man's switch. if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK { - return 0, _IOERR_LOCK + return _IOERR_LOCK } else if lock == unix.F_WRLCK { - return 0, _BUSY + return _BUSY } else if lock == unix.F_UNLCK { if s.readOnly { - return 0, _READONLY_CANTINIT + return _READONLY_CANTINIT } if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK { - return 0, rc + return rc } if err := s.Truncate(0); err != nil { - return 0, _IOERR_SHMOPEN + return _IOERR_SHMOPEN } } if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK { + return rc + } + return _OK +} + +func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if rc := s.shmOpen(); rc != _OK { return 0, rc } @@ -99,7 +106,7 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext } if n := (int64(id) + 1) * int64(size); n > o { if !extend { - return 0, nil + return 0, _OK } err := osAllocate(s.File, n) if err != nil { @@ -115,13 +122,13 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext } r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot) if err != nil { - return 0, err + return 0, _IOERR_SHMMAP } s.regions = append(s.regions, r) - return r.Ptr, nil + return r.Ptr, _OK } -func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error { +func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode { // Argument check. if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK { panic(util.AssertErr()) @@ -165,9 +172,9 @@ func (s *vfsShm) shmUnmap(delete bool) { s.regions = s.regions[:0] // Close the file. - defer s.Close() if delete { - os.Remove(s.Name()) + os.Remove(s.path) } + s.Close() s.File = nil } diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go new file mode 100644 index 000000000..3b45b3087 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go @@ -0,0 +1,259 @@ +//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) + +package vfs + +import ( + "context" + "io" + "os" + "sync" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" + "golang.org/x/sys/unix" +) + +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = true + +const _SHM_NLOCK = 8 + +func (f *vfsFile) SharedMemory() SharedMemory { return f.shm } + +// NewSharedMemory returns a shared-memory WAL-index +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, +// or not appropriate for the given flags. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. +func NewSharedMemory(path string, flags OpenFlag) SharedMemory { + if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 { + return nil + } + return &vfsShm{ + path: path, + readOnly: flags&OPEN_READONLY != 0, + } +} + +type vfsShmFile struct { + *os.File + info os.FileInfo + + // +checklocks:vfsShmFilesMtx + refs int + + // +checklocks:lockMtx + lock [_SHM_NLOCK]int16 + lockMtx sync.Mutex +} + +var ( + // +checklocks:vfsShmFilesMtx + vfsShmFiles []*vfsShmFile + vfsShmFilesMtx sync.Mutex +) + +type vfsShm struct { + *vfsShmFile + path string + lock [_SHM_NLOCK]bool + regions []*util.MappedRegion + readOnly bool +} + +func (s *vfsShm) Close() error { + if s.vfsShmFile == nil { + return nil + } + + // Unlock everything. + s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK) + + vfsShmFilesMtx.Lock() + defer vfsShmFilesMtx.Unlock() + + // Decrease reference count. + if s.vfsShmFile.refs > 1 { + s.vfsShmFile.refs-- + s.vfsShmFile = nil + return nil + } + for i, g := range vfsShmFiles { + if g == s.vfsShmFile { + vfsShmFiles[i] = nil + break + } + } + + err := s.File.Close() + s.vfsShmFile = nil + return err +} + +func (s *vfsShm) shmOpen() (rc _ErrorCode) { + if s.vfsShmFile != nil { + return _OK + } + + // Open file read-write, as it will be shared. + f, err := os.OpenFile(s.path, + unix.O_RDWR|unix.O_CREAT|unix.O_NOFOLLOW, 0666) + if err != nil { + return _CANTOPEN + } + // Close if file if it's not nil. + defer func() { f.Close() }() + + fi, err := f.Stat() + if err != nil { + return _IOERR_FSTAT + } + + vfsShmFilesMtx.Lock() + defer vfsShmFilesMtx.Unlock() + + // Find a shared file, increase the reference count. + for _, g := range vfsShmFiles { + if g != nil && os.SameFile(fi, g.info) { + g.refs++ + s.vfsShmFile = g + return _OK + } + } + + // Lock and truncate the file, if not readonly. + if s.readOnly { + rc = _READONLY_CANTINIT + } else { + if rc := osWriteLock(f, 0, 0, 0); rc != _OK { + return rc + } + if err := f.Truncate(0); err != nil { + return _IOERR_SHMOPEN + } + } + + // Add the new shared file. + s.vfsShmFile = &vfsShmFile{ + File: f, + info: fi, + refs: 1, + } + f = nil + add := true + for i, g := range vfsShmFiles { + if g == nil { + vfsShmFiles[i] = s.vfsShmFile + add = false + } + } + if add { + vfsShmFiles = append(vfsShmFiles, s.vfsShmFile) + } + return rc +} + +func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if rc := s.shmOpen(); rc != _OK { + return 0, rc + } + + // Check if file is big enough. + o, err := s.Seek(0, io.SeekEnd) + if err != nil { + return 0, _IOERR_SHMSIZE + } + if n := (int64(id) + 1) * int64(size); n > o { + if !extend { + return 0, _OK + } + err := osAllocate(s.File, n) + if err != nil { + return 0, _IOERR_SHMSIZE + } + } + + var prot int + if s.readOnly { + prot = unix.PROT_READ + } else { + prot = unix.PROT_READ | unix.PROT_WRITE + } + r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot) + if err != nil { + return 0, _IOERR_SHMMAP + } + s.regions = append(s.regions, r) + return r.Ptr, _OK +} + +func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode { + s.lockMtx.Lock() + defer s.lockMtx.Unlock() + + switch { + case flags&_SHM_UNLOCK != 0: + for i := offset; i < offset+n; i++ { + if s.lock[i] { + if s.vfsShmFile.lock[i] <= 0 { + s.vfsShmFile.lock[i] = 0 + } else { + s.vfsShmFile.lock[i]-- + } + } + } + case flags&_SHM_SHARED != 0: + for i := offset; i < offset+n; i++ { + if s.vfsShmFile.lock[i] < 0 { + return _BUSY + } + } + for i := offset; i < offset+n; i++ { + s.vfsShmFile.lock[i]++ + s.lock[i] = true + } + case flags&_SHM_EXCLUSIVE != 0: + for i := offset; i < offset+n; i++ { + if s.vfsShmFile.lock[i] != 0 { + return _BUSY + } + } + for i := offset; i < offset+n; i++ { + s.vfsShmFile.lock[i] = -1 + s.lock[i] = true + } + } + + return _OK +} + +func (s *vfsShm) shmUnmap(delete bool) { + if s.vfsShmFile == nil { + return + } + + // Unmap regions. + for _, r := range s.regions { + r.Unmap() + } + clear(s.regions) + s.regions = s.regions[:0] + + // Close the file. + if delete { + os.Remove(s.path) + } + s.Close() + s.vfsShmFile = nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go index 21191979e..7c8997581 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys package vfs diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go b/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go index 1887e9f22..d624aa78c 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go @@ -83,7 +83,7 @@ func vfsRandomness(ctx context.Context, mod api.Module, pVfs uint32, nByte int32 } func vfsSleep(ctx context.Context, mod api.Module, pVfs uint32, nMicro int32) _ErrorCode { - osSleep(time.Duration(nMicro) * time.Microsecond) + time.Sleep(time.Duration(nMicro) * time.Microsecond) return _OK } @@ -397,18 +397,14 @@ func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) { func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode { shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() - p, err := shm.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) - if err != nil { - return vfsErrorCode(err, _IOERR_SHMMAP) - } + p, rc := shm.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) util.WriteUint32(mod, pp, p) - return _OK + return rc } func vfsShmLock(ctx context.Context, mod api.Module, pFile uint32, offset, n int32, flags _ShmFlag) _ErrorCode { shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() - err := shm.shmLock(offset, n, flags) - return vfsErrorCode(err, _IOERR_SHMLOCK) + return shm.shmLock(offset, n, flags) } func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode { -- cgit v1.2.3