diff options
Diffstat (limited to 'vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go')
| -rw-r--r-- | vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go | 708 |
1 files changed, 0 insertions, 708 deletions
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go deleted file mode 100644 index 639429a63..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go +++ /dev/null @@ -1,708 +0,0 @@ -package wazevo - -import ( - "context" - "fmt" - "reflect" - "runtime" - "sync/atomic" - "unsafe" - - "github.com/tetratelabs/wazero/api" - "github.com/tetratelabs/wazero/experimental" - "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" - "github.com/tetratelabs/wazero/internal/expctxkeys" - "github.com/tetratelabs/wazero/internal/internalapi" - "github.com/tetratelabs/wazero/internal/wasm" - "github.com/tetratelabs/wazero/internal/wasmdebug" - "github.com/tetratelabs/wazero/internal/wasmruntime" -) - -type ( - // callEngine implements api.Function. - callEngine struct { - internalapi.WazeroOnly - stack []byte - // stackTop is the pointer to the *aligned* top of the stack. This must be updated - // whenever the stack is changed. This is passed to the assembly function - // at the very beginning of api.Function Call/CallWithStack. - stackTop uintptr - // executable is the pointer to the executable code for this function. - executable *byte - preambleExecutable *byte - // parent is the *moduleEngine from which this callEngine is created. - parent *moduleEngine - // indexInModule is the index of the function in the module. - indexInModule wasm.Index - // sizeOfParamResultSlice is the size of the parameter/result slice. - sizeOfParamResultSlice int - requiredParams int - // execCtx holds various information to be read/written by assembly functions. - execCtx executionContext - // execCtxPtr holds the pointer to the executionContext which doesn't change after callEngine is created. - execCtxPtr uintptr - numberOfResults int - stackIteratorImpl stackIterator - } - - // executionContext is the struct to be read/written by assembly functions. - executionContext struct { - // exitCode holds the wazevoapi.ExitCode describing the state of the function execution. - exitCode wazevoapi.ExitCode - // callerModuleContextPtr holds the moduleContextOpaque for Go function calls. - callerModuleContextPtr *byte - // originalFramePointer holds the original frame pointer of the caller of the assembly function. - originalFramePointer uintptr - // originalStackPointer holds the original stack pointer of the caller of the assembly function. - originalStackPointer uintptr - // goReturnAddress holds the return address to go back to the caller of the assembly function. - goReturnAddress uintptr - // stackBottomPtr holds the pointer to the bottom of the stack. - stackBottomPtr *byte - // goCallReturnAddress holds the return address to go back to the caller of the Go function. - goCallReturnAddress *byte - // stackPointerBeforeGoCall holds the stack pointer before calling a Go function. - stackPointerBeforeGoCall *uint64 - // stackGrowRequiredSize holds the required size of stack grow. - stackGrowRequiredSize uintptr - // memoryGrowTrampolineAddress holds the address of memory grow trampoline function. - memoryGrowTrampolineAddress *byte - // stackGrowCallTrampolineAddress holds the address of stack grow trampoline function. - stackGrowCallTrampolineAddress *byte - // checkModuleExitCodeTrampolineAddress holds the address of check-module-exit-code function. - checkModuleExitCodeTrampolineAddress *byte - // savedRegisters is the opaque spaces for save/restore registers. - // We want to align 16 bytes for each register, so we use [64][2]uint64. - savedRegisters [64][2]uint64 - // goFunctionCallCalleeModuleContextOpaque is the pointer to the target Go function's moduleContextOpaque. - goFunctionCallCalleeModuleContextOpaque uintptr - // tableGrowTrampolineAddress holds the address of table grow trampoline function. - tableGrowTrampolineAddress *byte - // refFuncTrampolineAddress holds the address of ref-func trampoline function. - refFuncTrampolineAddress *byte - // memmoveAddress holds the address of memmove function implemented by Go runtime. See memmove.go. - memmoveAddress uintptr - // framePointerBeforeGoCall holds the frame pointer before calling a Go function. Note: only used in amd64. - framePointerBeforeGoCall uintptr - // memoryWait32TrampolineAddress holds the address of memory_wait32 trampoline function. - memoryWait32TrampolineAddress *byte - // memoryWait32TrampolineAddress holds the address of memory_wait64 trampoline function. - memoryWait64TrampolineAddress *byte - // memoryNotifyTrampolineAddress holds the address of the memory_notify trampoline function. - memoryNotifyTrampolineAddress *byte - } -) - -func (c *callEngine) requiredInitialStackSize() int { - const initialStackSizeDefault = 10240 - stackSize := initialStackSizeDefault - paramResultInBytes := c.sizeOfParamResultSlice * 8 * 2 // * 8 because uint64 is 8 bytes, and *2 because we need both separated param/result slots. - required := paramResultInBytes + 32 + 16 // 32 is enough to accommodate the call frame info, and 16 exists just in case when []byte is not aligned to 16 bytes. - if required > stackSize { - stackSize = required - } - return stackSize -} - -func (c *callEngine) init() { - stackSize := c.requiredInitialStackSize() - if wazevoapi.StackGuardCheckEnabled { - stackSize += wazevoapi.StackGuardCheckGuardPageSize - } - c.stack = make([]byte, stackSize) - c.stackTop = alignedStackTop(c.stack) - if wazevoapi.StackGuardCheckEnabled { - c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] - } else { - c.execCtx.stackBottomPtr = &c.stack[0] - } - c.execCtxPtr = uintptr(unsafe.Pointer(&c.execCtx)) -} - -// alignedStackTop returns 16-bytes aligned stack top of given stack. -// 16 bytes should be good for all platform (arm64/amd64). -func alignedStackTop(s []byte) uintptr { - stackAddr := uintptr(unsafe.Pointer(&s[len(s)-1])) - return stackAddr - (stackAddr & (16 - 1)) -} - -// Definition implements api.Function. -func (c *callEngine) Definition() api.FunctionDefinition { - return c.parent.module.Source.FunctionDefinition(c.indexInModule) -} - -// Call implements api.Function. -func (c *callEngine) Call(ctx context.Context, params ...uint64) ([]uint64, error) { - if c.requiredParams != len(params) { - return nil, fmt.Errorf("expected %d params, but passed %d", c.requiredParams, len(params)) - } - paramResultSlice := make([]uint64, c.sizeOfParamResultSlice) - copy(paramResultSlice, params) - if err := c.callWithStack(ctx, paramResultSlice); err != nil { - return nil, err - } - return paramResultSlice[:c.numberOfResults], nil -} - -func (c *callEngine) addFrame(builder wasmdebug.ErrorBuilder, addr uintptr) (def api.FunctionDefinition, listener experimental.FunctionListener) { - eng := c.parent.parent.parent - cm := eng.compiledModuleOfAddr(addr) - if cm == nil { - // This case, the module might have been closed and deleted from the engine. - // We fall back to searching the imported modules that can be referenced from this callEngine. - - // First, we check itself. - if checkAddrInBytes(addr, c.parent.parent.executable) { - cm = c.parent.parent - } else { - // Otherwise, search all imported modules. TODO: maybe recursive, but not sure it's useful in practice. - p := c.parent - for i := range p.importedFunctions { - candidate := p.importedFunctions[i].me.parent - if checkAddrInBytes(addr, candidate.executable) { - cm = candidate - break - } - } - } - } - - if cm != nil { - index := cm.functionIndexOf(addr) - def = cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) - var sources []string - if dw := cm.module.DWARFLines; dw != nil { - sourceOffset := cm.getSourceOffset(addr) - sources = dw.Line(sourceOffset) - } - builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources) - if len(cm.listeners) > 0 { - listener = cm.listeners[index] - } - } - return -} - -// CallWithStack implements api.Function. -func (c *callEngine) CallWithStack(ctx context.Context, paramResultStack []uint64) (err error) { - if c.sizeOfParamResultSlice > len(paramResultStack) { - return fmt.Errorf("need %d params, but stack size is %d", c.sizeOfParamResultSlice, len(paramResultStack)) - } - return c.callWithStack(ctx, paramResultStack) -} - -// CallWithStack implements api.Function. -func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint64) (err error) { - snapshotEnabled := ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil - if snapshotEnabled { - ctx = context.WithValue(ctx, expctxkeys.SnapshotterKey{}, c) - } - - if wazevoapi.StackGuardCheckEnabled { - defer func() { - wazevoapi.CheckStackGuardPage(c.stack) - }() - } - - p := c.parent - ensureTermination := p.parent.ensureTermination - m := p.module - if ensureTermination { - select { - case <-ctx.Done(): - // If the provided context is already done, close the module and return the error. - m.CloseWithCtxErr(ctx) - return m.FailIfClosed() - default: - } - } - - var paramResultPtr *uint64 - if len(paramResultStack) > 0 { - paramResultPtr = ¶mResultStack[0] - } - defer func() { - r := recover() - if s, ok := r.(*snapshot); ok { - // A snapshot that wasn't handled was created by a different call engine possibly from a nested wasm invocation, - // let it propagate up to be handled by the caller. - panic(s) - } - if r != nil { - type listenerForAbort struct { - def api.FunctionDefinition - lsn experimental.FunctionListener - } - - var listeners []listenerForAbort - builder := wasmdebug.NewErrorBuilder() - def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) - if lsn != nil { - listeners = append(listeners, listenerForAbort{def, lsn}) - } - returnAddrs := unwindStack( - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), - c.execCtx.framePointerBeforeGoCall, - c.stackTop, - nil, - ) - for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it. - def, lsn = c.addFrame(builder, retAddr) - if lsn != nil { - listeners = append(listeners, listenerForAbort{def, lsn}) - } - } - err = builder.FromRecovered(r) - - for _, lsn := range listeners { - lsn.lsn.Abort(ctx, m, lsn.def, err) - } - } else { - if err != wasmruntime.ErrRuntimeStackOverflow { // Stackoverflow case shouldn't be panic (to avoid extreme stack unwinding). - err = c.parent.module.FailIfClosed() - } - } - - if err != nil { - // Ensures that we can reuse this callEngine even after an error. - c.execCtx.exitCode = wazevoapi.ExitCodeOK - } - }() - - if ensureTermination { - done := m.CloseModuleOnCanceledOrTimeout(ctx) - defer done() - } - - if c.stackTop&(16-1) != 0 { - panic("BUG: stack must be aligned to 16 bytes") - } - entrypoint(c.preambleExecutable, c.executable, c.execCtxPtr, c.parent.opaquePtr, paramResultPtr, c.stackTop) - for { - switch ec := c.execCtx.exitCode; ec & wazevoapi.ExitCodeMask { - case wazevoapi.ExitCodeOK: - return nil - case wazevoapi.ExitCodeGrowStack: - oldsp := uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) - oldTop := c.stackTop - oldStack := c.stack - var newsp, newfp uintptr - if wazevoapi.StackGuardCheckEnabled { - newsp, newfp, err = c.growStackWithGuarded() - } else { - newsp, newfp, err = c.growStack() - } - if err != nil { - return err - } - adjustClonedStack(oldsp, oldTop, newsp, newfp, c.stackTop) - // Old stack must be alive until the new stack is adjusted. - runtime.KeepAlive(oldStack) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, newsp, newfp) - case wazevoapi.ExitCodeGrowMemory: - mod := c.callerModuleInstance() - mem := mod.MemoryInstance - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - argRes := &s[0] - if res, ok := mem.Grow(uint32(*argRes)); !ok { - *argRes = uint64(0xffffffff) // = -1 in signed 32-bit integer. - } else { - *argRes = uint64(res) - } - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeTableGrow: - mod := c.callerModuleInstance() - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - tableIndex, num, ref := uint32(s[0]), uint32(s[1]), uintptr(s[2]) - table := mod.Tables[tableIndex] - s[0] = uint64(uint32(int32(table.Grow(num, ref)))) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallGoFunction: - index := wazevoapi.GoFunctionIndexFromExitCode(ec) - f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) - func() { - if snapshotEnabled { - defer snapshotRecoverFn(c) - } - f.Call(ctx, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) - }() - // Back to the native code. - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallGoFunctionWithListener: - index := wazevoapi.GoFunctionIndexFromExitCode(ec) - f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) - listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - // Call Listener.Before. - callerModule := c.callerModuleInstance() - listener := listeners[index] - hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) - def := hostModule.FunctionDefinition(wasm.Index(index)) - listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) - // Call into the Go function. - func() { - if snapshotEnabled { - defer snapshotRecoverFn(c) - } - f.Call(ctx, s) - }() - // Call Listener.After. - listener.After(ctx, callerModule, def, s) - // Back to the native code. - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallGoModuleFunction: - index := wazevoapi.GoFunctionIndexFromExitCode(ec) - f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) - mod := c.callerModuleInstance() - func() { - if snapshotEnabled { - defer snapshotRecoverFn(c) - } - f.Call(ctx, mod, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) - }() - // Back to the native code. - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallGoModuleFunctionWithListener: - index := wazevoapi.GoFunctionIndexFromExitCode(ec) - f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) - listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - // Call Listener.Before. - callerModule := c.callerModuleInstance() - listener := listeners[index] - hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) - def := hostModule.FunctionDefinition(wasm.Index(index)) - listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) - // Call into the Go function. - func() { - if snapshotEnabled { - defer snapshotRecoverFn(c) - } - f.Call(ctx, callerModule, s) - }() - // Call Listener.After. - listener.After(ctx, callerModule, def, s) - // Back to the native code. - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallListenerBefore: - stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - index := wasm.Index(stack[0]) - mod := c.callerModuleInstance() - listener := mod.Engine.(*moduleEngine).listeners[index] - def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount) - listener.Before(ctx, mod, def, stack[1:], c.stackIterator(false)) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCallListenerAfter: - stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - index := wasm.Index(stack[0]) - mod := c.callerModuleInstance() - listener := mod.Engine.(*moduleEngine).listeners[index] - def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount) - listener.After(ctx, mod, def, stack[1:]) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeCheckModuleExitCode: - // Note: this operation must be done in Go, not native code. The reason is that - // native code cannot be preempted and that means it can block forever if there are not - // enough OS threads (which we don't have control over). - if err := m.FailIfClosed(); err != nil { - panic(err) - } - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeRefFunc: - mod := c.callerModuleInstance() - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - funcIndex := wasm.Index(s[0]) - ref := mod.Engine.FunctionInstanceReference(funcIndex) - s[0] = uint64(ref) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeMemoryWait32: - mod := c.callerModuleInstance() - mem := mod.MemoryInstance - if !mem.Shared { - panic(wasmruntime.ErrRuntimeExpectedSharedMemory) - } - - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - timeout, exp, addr := int64(s[0]), uint32(s[1]), uintptr(s[2]) - base := uintptr(unsafe.Pointer(&mem.Buffer[0])) - - offset := uint32(addr - base) - res := mem.Wait32(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint32 { - addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset) - return atomic.LoadUint32((*uint32)(addr)) - }) - s[0] = res - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeMemoryWait64: - mod := c.callerModuleInstance() - mem := mod.MemoryInstance - if !mem.Shared { - panic(wasmruntime.ErrRuntimeExpectedSharedMemory) - } - - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - timeout, exp, addr := int64(s[0]), uint64(s[1]), uintptr(s[2]) - base := uintptr(unsafe.Pointer(&mem.Buffer[0])) - - offset := uint32(addr - base) - res := mem.Wait64(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint64 { - addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset) - return atomic.LoadUint64((*uint64)(addr)) - }) - s[0] = uint64(res) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeMemoryNotify: - mod := c.callerModuleInstance() - mem := mod.MemoryInstance - - s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) - count, addr := uint32(s[0]), s[1] - offset := uint32(uintptr(addr) - uintptr(unsafe.Pointer(&mem.Buffer[0]))) - res := mem.Notify(offset, count) - s[0] = uint64(res) - c.execCtx.exitCode = wazevoapi.ExitCodeOK - afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) - case wazevoapi.ExitCodeUnreachable: - panic(wasmruntime.ErrRuntimeUnreachable) - case wazevoapi.ExitCodeMemoryOutOfBounds: - panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) - case wazevoapi.ExitCodeTableOutOfBounds: - panic(wasmruntime.ErrRuntimeInvalidTableAccess) - case wazevoapi.ExitCodeIndirectCallNullPointer: - panic(wasmruntime.ErrRuntimeInvalidTableAccess) - case wazevoapi.ExitCodeIndirectCallTypeMismatch: - panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) - case wazevoapi.ExitCodeIntegerOverflow: - panic(wasmruntime.ErrRuntimeIntegerOverflow) - case wazevoapi.ExitCodeIntegerDivisionByZero: - panic(wasmruntime.ErrRuntimeIntegerDivideByZero) - case wazevoapi.ExitCodeInvalidConversionToInteger: - panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) - case wazevoapi.ExitCodeUnalignedAtomic: - panic(wasmruntime.ErrRuntimeUnalignedAtomic) - default: - panic("BUG") - } - } -} - -func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance { - return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr) -} - -const callStackCeiling = uintptr(50000000) // in uint64 (8 bytes) == 400000000 bytes in total == 400mb. - -func (c *callEngine) growStackWithGuarded() (newSP uintptr, newFP uintptr, err error) { - if wazevoapi.StackGuardCheckEnabled { - wazevoapi.CheckStackGuardPage(c.stack) - } - newSP, newFP, err = c.growStack() - if err != nil { - return - } - if wazevoapi.StackGuardCheckEnabled { - c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] - } - return -} - -// growStack grows the stack, and returns the new stack pointer. -func (c *callEngine) growStack() (newSP, newFP uintptr, err error) { - currentLen := uintptr(len(c.stack)) - if callStackCeiling < currentLen { - err = wasmruntime.ErrRuntimeStackOverflow - return - } - - newLen := 2*currentLen + c.execCtx.stackGrowRequiredSize + 16 // Stack might be aligned to 16 bytes, so add 16 bytes just in case. - newSP, newFP, c.stackTop, c.stack = c.cloneStack(newLen) - c.execCtx.stackBottomPtr = &c.stack[0] - return -} - -func (c *callEngine) cloneStack(l uintptr) (newSP, newFP, newTop uintptr, newStack []byte) { - newStack = make([]byte, l) - - relSp := c.stackTop - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) - relFp := c.stackTop - c.execCtx.framePointerBeforeGoCall - - // Copy the existing contents in the previous Go-allocated stack into the new one. - var prevStackAligned, newStackAligned []byte - { - //nolint:staticcheck - sh := (*reflect.SliceHeader)(unsafe.Pointer(&prevStackAligned)) - sh.Data = c.stackTop - relSp - sh.Len = int(relSp) - sh.Cap = int(relSp) - } - newTop = alignedStackTop(newStack) - { - newSP = newTop - relSp - newFP = newTop - relFp - //nolint:staticcheck - sh := (*reflect.SliceHeader)(unsafe.Pointer(&newStackAligned)) - sh.Data = newSP - sh.Len = int(relSp) - sh.Cap = int(relSp) - } - copy(newStackAligned, prevStackAligned) - return -} - -func (c *callEngine) stackIterator(onHostCall bool) experimental.StackIterator { - c.stackIteratorImpl.reset(c, onHostCall) - return &c.stackIteratorImpl -} - -// stackIterator implements experimental.StackIterator. -type stackIterator struct { - retAddrs []uintptr - retAddrCursor int - eng *engine - pc uint64 - - currentDef *wasm.FunctionDefinition -} - -func (si *stackIterator) reset(c *callEngine, onHostCall bool) { - if onHostCall { - si.retAddrs = append(si.retAddrs[:0], uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) - } else { - si.retAddrs = si.retAddrs[:0] - } - si.retAddrs = unwindStack(uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall, c.stackTop, si.retAddrs) - si.retAddrs = si.retAddrs[:len(si.retAddrs)-1] // the last return addr is the trampoline, so we skip it. - si.retAddrCursor = 0 - si.eng = c.parent.parent.parent -} - -// Next implements the same method as documented on experimental.StackIterator. -func (si *stackIterator) Next() bool { - if si.retAddrCursor >= len(si.retAddrs) { - return false - } - - addr := si.retAddrs[si.retAddrCursor] - cm := si.eng.compiledModuleOfAddr(addr) - if cm != nil { - index := cm.functionIndexOf(addr) - def := cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) - si.currentDef = def - si.retAddrCursor++ - si.pc = uint64(addr) - return true - } - return false -} - -// ProgramCounter implements the same method as documented on experimental.StackIterator. -func (si *stackIterator) ProgramCounter() experimental.ProgramCounter { - return experimental.ProgramCounter(si.pc) -} - -// Function implements the same method as documented on experimental.StackIterator. -func (si *stackIterator) Function() experimental.InternalFunction { - return si -} - -// Definition implements the same method as documented on experimental.InternalFunction. -func (si *stackIterator) Definition() api.FunctionDefinition { - return si.currentDef -} - -// SourceOffsetForPC implements the same method as documented on experimental.InternalFunction. -func (si *stackIterator) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 { - upc := uintptr(pc) - cm := si.eng.compiledModuleOfAddr(upc) - return cm.getSourceOffset(upc) -} - -// snapshot implements experimental.Snapshot -type snapshot struct { - sp, fp, top uintptr - returnAddress *byte - stack []byte - savedRegisters [64][2]uint64 - ret []uint64 - c *callEngine -} - -// Snapshot implements the same method as documented on experimental.Snapshotter. -func (c *callEngine) Snapshot() experimental.Snapshot { - returnAddress := c.execCtx.goCallReturnAddress - oldTop, oldSp := c.stackTop, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) - newSP, newFP, newTop, newStack := c.cloneStack(uintptr(len(c.stack)) + 16) - adjustClonedStack(oldSp, oldTop, newSP, newFP, newTop) - return &snapshot{ - sp: newSP, - fp: newFP, - top: newTop, - savedRegisters: c.execCtx.savedRegisters, - returnAddress: returnAddress, - stack: newStack, - c: c, - } -} - -// Restore implements the same method as documented on experimental.Snapshot. -func (s *snapshot) Restore(ret []uint64) { - s.ret = ret - panic(s) -} - -func (s *snapshot) doRestore() { - spp := *(**uint64)(unsafe.Pointer(&s.sp)) - view := goCallStackView(spp) - copy(view, s.ret) - - c := s.c - c.stack = s.stack - c.stackTop = s.top - ec := &c.execCtx - ec.stackBottomPtr = &c.stack[0] - ec.stackPointerBeforeGoCall = spp - ec.framePointerBeforeGoCall = s.fp - ec.goCallReturnAddress = s.returnAddress - ec.savedRegisters = s.savedRegisters -} - -// Error implements the same method on error. -func (s *snapshot) Error() string { - return "unhandled snapshot restore, this generally indicates restore was called from a different " + - "exported function invocation than snapshot" -} - -func snapshotRecoverFn(c *callEngine) { - if r := recover(); r != nil { - if s, ok := r.(*snapshot); ok && s.c == c { - s.doRestore() - } else { - panic(r) - } - } -} |