summaryrefslogtreecommitdiff
path: root/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64
diff options
context:
space:
mode:
authorLibravatar kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>2024-08-15 00:08:55 +0000
committerLibravatar GitHub <noreply@github.com>2024-08-15 00:08:55 +0000
commit09f24e044653b1327ac1c40f3ab150e3f0184f23 (patch)
tree1d9984d053fa5c8d1203abaa49b8752a1532ff11 /vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64
parentupdate go-fastcopy to v1.1.3 (#3200) (diff)
downloadgotosocial-09f24e044653b1327ac1c40f3ab150e3f0184f23.tar.xz
update go-ffmpreg to v0.2.5 (pulls in latest tetratelabs/wazero) (#3203)
Diffstat (limited to 'vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64')
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go9
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go15
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go27
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go2
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go30
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go56
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go271
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go25
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go241
-rw-r--r--vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go12
10 files changed, 472 insertions, 216 deletions
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go
index 4eaa13ce1..d1eaa7cd4 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go
@@ -182,9 +182,9 @@ func (m *machine) LowerReturns(rets []ssa.Value) {
// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the
// caller side of the function call.
-func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, slotBegin int64) {
+func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, slotBegin int64) {
arg := &a.Args[argIndex]
- if def != nil && def.IsFromInstr() {
+ if def.IsFromInstr() {
// Constant instructions are inlined.
if inst := def.Instr; inst.Constant() {
val := inst.Return()
@@ -228,10 +228,9 @@ func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex i
}
func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) (*instruction, *addressMode) {
- exct := m.executableContext
- exct.PendingInstructions = exct.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
mode := m.resolveAddressModeForOffset(offset, dstBits, rn, allowTmpRegUse)
- for _, instr := range exct.PendingInstructions {
+ for _, instr := range m.pendingInstructions {
cur = linkInstr(cur, instr)
}
return cur, mode
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
index 99e6bb482..06f8a4a05 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
@@ -14,7 +14,6 @@ var calleeSavedRegistersSorted = []regalloc.VReg{
// CompileGoFunctionTrampoline implements backend.Machine.
func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte {
- exct := m.executableContext
argBegin := 1 // Skips exec context by default.
if needModuleContextPtr {
argBegin++
@@ -26,7 +25,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
cur := m.allocateInstr()
cur.asNop0()
- exct.RootInstr = cur
+ m.rootInstr = cur
// Execution context is always the first argument.
execCtrPtr := x0VReg
@@ -244,7 +243,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
ret.asRet()
linkInstr(cur, ret)
- m.encode(m.executableContext.RootInstr)
+ m.encode(m.rootInstr)
return m.compiler.Buf()
}
@@ -302,20 +301,18 @@ func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []re
}
func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction {
- exct := m.executableContext
- exct.PendingInstructions = exct.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
m.lowerConstantI64(dst, v)
- for _, instr := range exct.PendingInstructions {
+ for _, instr := range m.pendingInstructions {
cur = linkInstr(cur, instr)
}
return cur
}
func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction {
- exct := m.executableContext
- exct.PendingInstructions = exct.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
m.lowerConstantI32(dst, v)
- for _, instr := range exct.PendingInstructions {
+ for _, instr := range m.pendingInstructions {
cur = linkInstr(cur, instr)
}
return cur
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go
index 7121cb538..1f563428a 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go
@@ -36,18 +36,6 @@ type (
instructionKind byte
)
-func asNop0(i *instruction) {
- i.kind = nop0
-}
-
-func setNext(i, next *instruction) {
- i.next = next
-}
-
-func setPrev(i, prev *instruction) {
- i.prev = prev
-}
-
// IsCall implements regalloc.Instr IsCall.
func (i *instruction) IsCall() bool {
return i.kind == call
@@ -63,21 +51,6 @@ func (i *instruction) IsReturn() bool {
return i.kind == ret
}
-// Next implements regalloc.Instr Next.
-func (i *instruction) Next() regalloc.Instr {
- return i.next
-}
-
-// Prev implements regalloc.Instr Prev.
-func (i *instruction) Prev() regalloc.Instr {
- return i.prev
-}
-
-// AddedBeforeRegAlloc implements regalloc.Instr AddedBeforeRegAlloc.
-func (i *instruction) AddedBeforeRegAlloc() bool {
- return i.addedBeforeRegAlloc
-}
-
type defKind byte
const (
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
index f0ede2d6a..21be9b71e 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
@@ -12,7 +12,7 @@ import (
// Encode implements backend.Machine Encode.
func (m *machine) Encode(ctx context.Context) error {
m.resolveRelativeAddresses(ctx)
- m.encode(m.executableContext.RootInstr)
+ m.encode(m.rootInstr)
if l := len(m.compiler.Buf()); l > maxFunctionExecutableSize {
return fmt.Errorf("function size exceeds the limit: %d > %d", l, maxFunctionExecutableSize)
}
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
index 048bf3204..f9df356c0 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
@@ -17,19 +17,18 @@ import (
// LowerSingleBranch implements backend.Machine.
func (m *machine) LowerSingleBranch(br *ssa.Instruction) {
- ectx := m.executableContext
switch br.Opcode() {
case ssa.OpcodeJump:
- _, _, targetBlk := br.BranchData()
+ _, _, targetBlkID := br.BranchData()
if br.IsFallthroughJump() {
return
}
b := m.allocateInstr()
- target := ectx.GetOrAllocateSSABlockLabel(targetBlk)
- if target == labelReturn {
+ targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID)
+ if targetBlk.ReturnBlock() {
b.asRet()
} else {
- b.asBr(target)
+ b.asBr(ssaBlockLabel(targetBlk))
}
m.insert(b)
case ssa.OpcodeBrTable:
@@ -40,7 +39,8 @@ func (m *machine) LowerSingleBranch(br *ssa.Instruction) {
}
func (m *machine) lowerBrTable(i *ssa.Instruction) {
- index, targets := i.BrTableData()
+ index, targetBlockIDs := i.BrTableData()
+ targetBlockCount := len(targetBlockIDs.View())
indexOperand := m.getOperand_NR(m.compiler.ValueDefinition(index), extModeNone)
// Firstly, we have to do the bounds check of the index, and
@@ -50,7 +50,7 @@ func (m *machine) lowerBrTable(i *ssa.Instruction) {
// subs wzr, index, maxIndexReg
// csel adjustedIndex, maxIndexReg, index, hs ;; if index is higher or equal than maxIndexReg.
maxIndexReg := m.compiler.AllocateVReg(ssa.TypeI32)
- m.lowerConstantI32(maxIndexReg, int32(len(targets)-1))
+ m.lowerConstantI32(maxIndexReg, int32(targetBlockCount-1))
subs := m.allocateInstr()
subs.asALU(aluOpSubS, xzrVReg, indexOperand, operandNR(maxIndexReg), false)
m.insert(subs)
@@ -61,24 +61,24 @@ func (m *machine) lowerBrTable(i *ssa.Instruction) {
brSequence := m.allocateInstr()
- tableIndex := m.addJmpTableTarget(targets)
- brSequence.asBrTableSequence(adjustedIndex, tableIndex, len(targets))
+ tableIndex := m.addJmpTableTarget(targetBlockIDs)
+ brSequence.asBrTableSequence(adjustedIndex, tableIndex, targetBlockCount)
m.insert(brSequence)
}
// LowerConditionalBranch implements backend.Machine.
func (m *machine) LowerConditionalBranch(b *ssa.Instruction) {
- exctx := m.executableContext
- cval, args, targetBlk := b.BranchData()
+ cval, args, targetBlkID := b.BranchData()
if len(args) > 0 {
panic(fmt.Sprintf(
"conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s",
- exctx.CurrentSSABlk,
- targetBlk,
+ m.currentLabelPos.sb,
+ targetBlkID,
))
}
- target := exctx.GetOrAllocateSSABlockLabel(targetBlk)
+ targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID)
+ target := ssaBlockLabel(targetBlk)
cvalDef := m.compiler.ValueDefinition(cval)
switch {
@@ -791,7 +791,7 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) {
default:
panic("TODO: lowering " + op.String())
}
- m.executableContext.FlushPendingInstructions()
+ m.FlushPendingInstructions()
}
func (m *machine) lowerShuffle(rd regalloc.VReg, rn, rm operand, lane1, lane2 uint64) {
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go
index d9fbf1789..7a398c3d0 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go
@@ -162,9 +162,9 @@ func (o operand) assignReg(v regalloc.VReg) operand {
//
// `mode` is used to extend the operand if the bit length is smaller than mode.bits().
// If the operand can be expressed as operandKindImm12, `mode` is ignored.
-func (m *machine) getOperand_Imm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) {
- if def.IsFromBlockParam() {
- return operandNR(def.BlkParamVReg)
+func (m *machine) getOperand_Imm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) {
+ if !def.IsFromInstr() {
+ return operandNR(m.compiler.VRegOf(def.V))
}
instr := def.Instr
@@ -179,9 +179,9 @@ func (m *machine) getOperand_Imm12_ER_SR_NR(def *backend.SSAValueDefinition, mod
// getOperand_MaybeNegatedImm12_ER_SR_NR is almost the same as getOperand_Imm12_ER_SR_NR, but this might negate the immediate value.
// If the immediate value is negated, the second return value is true, otherwise always false.
-func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) {
- if def.IsFromBlockParam() {
- return operandNR(def.BlkParamVReg), false
+func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) {
+ if !def.IsFromInstr() {
+ return operandNR(m.compiler.VRegOf(def.V)), false
}
instr := def.Instr
@@ -193,7 +193,7 @@ func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDef
}
signExtended := int64(c)
- if def.SSAValue().Type().Bits() == 32 {
+ if def.V.Type().Bits() == 32 {
signExtended = (signExtended << 32) >> 32
}
negatedWithoutSign := -signExtended
@@ -208,9 +208,9 @@ func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDef
// ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def).
//
// `mode` is used to extend the operand if the bit length is smaller than mode.bits().
-func (m *machine) getOperand_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) {
- if def.IsFromBlockParam() {
- return operandNR(def.BlkParamVReg)
+func (m *machine) getOperand_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) {
+ if !def.IsFromInstr() {
+ return operandNR(m.compiler.VRegOf(def.V))
}
if m.compiler.MatchInstr(def, ssa.OpcodeSExtend) || m.compiler.MatchInstr(def, ssa.OpcodeUExtend) {
@@ -251,9 +251,9 @@ func (m *machine) getOperand_ER_SR_NR(def *backend.SSAValueDefinition, mode extM
// ensureValueNR returns an operand of either operandKindSR or operandKindNR from the given value (defined by `def).
//
// `mode` is used to extend the operand if the bit length is smaller than mode.bits().
-func (m *machine) getOperand_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) {
- if def.IsFromBlockParam() {
- return operandNR(def.BlkParamVReg)
+func (m *machine) getOperand_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) {
+ if !def.IsFromInstr() {
+ return operandNR(m.compiler.VRegOf(def.V))
}
if m.compiler.MatchInstr(def, ssa.OpcodeIshl) {
@@ -273,9 +273,9 @@ func (m *machine) getOperand_SR_NR(def *backend.SSAValueDefinition, mode extMode
}
// getOperand_ShiftImm_NR returns an operand of either operandKindShiftImm or operandKindNR from the given value (defined by `def).
-func (m *machine) getOperand_ShiftImm_NR(def *backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) {
- if def.IsFromBlockParam() {
- return operandNR(def.BlkParamVReg)
+func (m *machine) getOperand_ShiftImm_NR(def backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) {
+ if !def.IsFromInstr() {
+ return operandNR(m.compiler.VRegOf(def.V))
}
instr := def.Instr
@@ -289,28 +289,18 @@ func (m *machine) getOperand_ShiftImm_NR(def *backend.SSAValueDefinition, mode e
// ensureValueNR returns an operand of operandKindNR from the given value (defined by `def).
//
// `mode` is used to extend the operand if the bit length is smaller than mode.bits().
-func (m *machine) getOperand_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) {
+func (m *machine) getOperand_NR(def backend.SSAValueDefinition, mode extMode) (op operand) {
var v regalloc.VReg
- if def.IsFromBlockParam() {
- v = def.BlkParamVReg
+ if def.IsFromInstr() && def.Instr.Constant() {
+ // We inline all the constant instructions so that we could reduce the register usage.
+ v = m.lowerConstant(def.Instr)
+ def.Instr.MarkLowered()
} else {
- instr := def.Instr
- if instr.Constant() {
- // We inline all the constant instructions so that we could reduce the register usage.
- v = m.lowerConstant(instr)
- instr.MarkLowered()
- } else {
- if n := def.N; n == 0 {
- v = m.compiler.VRegOf(instr.Return())
- } else {
- _, rs := instr.Returns()
- v = m.compiler.VRegOf(rs[n-1])
- }
- }
+ v = m.compiler.VRegOf(def.V)
}
r := v
- switch inBits := def.SSAValue().Type().Bits(); {
+ switch inBits := def.V.Type().Bits(); {
case mode == extModeNone:
case inBits == 32 && (mode == extModeZeroExtend32 || mode == extModeSignExtend32):
case inBits == 32 && mode == extModeZeroExtend64:
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go
index 5f584f928..00e6b238f 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go
@@ -3,6 +3,7 @@ package arm64
import (
"context"
"fmt"
+ "math"
"strings"
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
@@ -14,12 +15,33 @@ import (
type (
// machine implements backend.Machine.
machine struct {
- compiler backend.Compiler
- executableContext *backend.ExecutableContextT[instruction]
- currentABI *backend.FunctionABI
+ compiler backend.Compiler
+ currentABI *backend.FunctionABI
+ instrPool wazevoapi.Pool[instruction]
+ // labelPositionPool is the pool of labelPosition. The id is the label where
+ // if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID.
+ labelPositionPool wazevoapi.IDedPool[labelPosition]
- regAlloc regalloc.Allocator
- regAllocFn *backend.RegAllocFunction[*instruction, *machine]
+ // nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID
+ // so that we can have an identical label for the SSA block ID, which is useful for debugging.
+ nextLabel label
+ // rootInstr is the first instruction of the function.
+ rootInstr *instruction
+ // currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition.
+ currentLabelPos *labelPosition
+ // orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock.
+ orderedSSABlockLabelPos []*labelPosition
+ // returnLabelPos is the labelPosition for the return block.
+ returnLabelPos labelPosition
+ // perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock.
+ perBlockHead, perBlockEnd *instruction
+ // pendingInstructions are the instructions which are not yet emitted into the instruction list.
+ pendingInstructions []*instruction
+ // maxSSABlockID is the maximum ssa.BasicBlockID in the current function.
+ maxSSABlockID label
+
+ regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn]
+ regAllocFn regAllocFn
amodePool wazevoapi.Pool[addressMode]
@@ -35,6 +57,8 @@ type (
// jmpTableTargets holds the labels of the jump table targets.
jmpTableTargets [][]uint32
+ // jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table.
+ jmpTableTargetsNext int
// spillSlotSize is the size of the stack slot in bytes used for spilling registers.
// During the execution of the function, the stack looks like:
@@ -91,45 +115,132 @@ type (
nextLabel label
offset int64
}
+)
+
+type (
+ // label represents a position in the generated code which is either
+ // a real instruction or the constant InstructionPool (e.g. jump tables).
+ //
+ // This is exactly the same as the traditional "label" in assembly code.
+ label uint32
- labelPosition = backend.LabelPosition[instruction]
- label = backend.Label
+ // labelPosition represents the regions of the generated code which the label represents.
+ // This implements regalloc.Block.
+ labelPosition struct {
+ // sb is not nil if this corresponds to a ssa.BasicBlock.
+ sb ssa.BasicBlock
+ // cur is used to walk through the instructions in the block during the register allocation.
+ cur,
+ // begin and end are the first and last instructions of the block.
+ begin, end *instruction
+ // binaryOffset is the offset in the binary where the label is located.
+ binaryOffset int64
+ }
)
const (
- labelReturn = backend.LabelReturn
- labelInvalid = backend.LabelInvalid
+ labelReturn label = math.MaxUint32
+ labelInvalid = labelReturn - 1
)
+// String implements backend.Machine.
+func (l label) String() string {
+ return fmt.Sprintf("L%d", l)
+}
+
+func resetLabelPosition(l *labelPosition) {
+ *l = labelPosition{}
+}
+
// NewBackend returns a new backend for arm64.
func NewBackend() backend.Machine {
m := &machine{
spillSlots: make(map[regalloc.VRegID]int64),
- executableContext: newExecutableContext(),
- regAlloc: regalloc.NewAllocator(regInfo),
+ regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo),
amodePool: wazevoapi.NewPool[addressMode](resetAddressMode),
+ instrPool: wazevoapi.NewPool[instruction](resetInstruction),
+ labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition),
}
+ m.regAllocFn.m = m
return m
}
-func newExecutableContext() *backend.ExecutableContextT[instruction] {
- return backend.NewExecutableContextT[instruction](resetInstruction, setNext, setPrev, asNop0)
+func ssaBlockLabel(sb ssa.BasicBlock) label {
+ if sb.ReturnBlock() {
+ return labelReturn
+ }
+ return label(sb.ID())
}
-// ExecutableContext implements backend.Machine.
-func (m *machine) ExecutableContext() backend.ExecutableContext {
- return m.executableContext
+// getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block.
+func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition {
+ if sb.ReturnBlock() {
+ m.returnLabelPos.sb = sb
+ return &m.returnLabelPos
+ }
+
+ l := ssaBlockLabel(sb)
+ pos := m.labelPositionPool.GetOrAllocate(int(l))
+ pos.sb = sb
+ return pos
}
-// RegAlloc implements backend.Machine Function.
-func (m *machine) RegAlloc() {
- rf := m.regAllocFn
- for _, pos := range m.executableContext.OrderedBlockLabels {
- rf.AddBlock(pos.SB, pos.L, pos.Begin, pos.End)
+// LinkAdjacentBlocks implements backend.Machine.
+func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) {
+ prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next)
+ prevPos.end.next = nextPos.begin
+}
+
+// StartBlock implements backend.Machine.
+func (m *machine) StartBlock(blk ssa.BasicBlock) {
+ m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk)
+ labelPos := m.currentLabelPos
+ end := m.allocateNop()
+ m.perBlockHead, m.perBlockEnd = end, end
+ labelPos.begin, labelPos.end = end, end
+ m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos)
+}
+
+// EndBlock implements ExecutableContext.
+func (m *machine) EndBlock() {
+ // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions.
+ m.insertAtPerBlockHead(m.allocateNop())
+
+ m.currentLabelPos.begin = m.perBlockHead
+
+ if m.currentLabelPos.sb.EntryBlock() {
+ m.rootInstr = m.perBlockHead
+ }
+}
+
+func (m *machine) insertAtPerBlockHead(i *instruction) {
+ if m.perBlockHead == nil {
+ m.perBlockHead = i
+ m.perBlockEnd = i
+ return
}
+ i.next = m.perBlockHead
+ m.perBlockHead.prev = i
+ m.perBlockHead = i
+}
+
+// FlushPendingInstructions implements backend.Machine.
+func (m *machine) FlushPendingInstructions() {
+ l := len(m.pendingInstructions)
+ if l == 0 {
+ return
+ }
+ for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order.
+ m.insertAtPerBlockHead(m.pendingInstructions[i])
+ }
+ m.pendingInstructions = m.pendingInstructions[:0]
+}
+
+// RegAlloc implements backend.Machine Function.
+func (m *machine) RegAlloc() {
m.regAllocStarted = true
- m.regAlloc.DoAllocation(rf)
+ m.regAlloc.DoAllocation(&m.regAllocFn)
// Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes.
m.spillSlotSize = (m.spillSlotSize + 15) &^ 15
}
@@ -146,13 +257,22 @@ func (m *machine) Reset() {
m.clobberedRegs = m.clobberedRegs[:0]
m.regAllocStarted = false
m.regAlloc.Reset()
- m.regAllocFn.Reset()
m.spillSlotSize = 0
m.unresolvedAddressModes = m.unresolvedAddressModes[:0]
m.maxRequiredStackSizeForCalls = 0
- m.executableContext.Reset()
- m.jmpTableTargets = m.jmpTableTargets[:0]
+ m.jmpTableTargetsNext = 0
m.amodePool.Reset()
+ m.instrPool.Reset()
+ m.labelPositionPool.Reset()
+ m.pendingInstructions = m.pendingInstructions[:0]
+ m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil
+ m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0]
+}
+
+// StartLoweringFunction implements backend.Machine StartLoweringFunction.
+func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) {
+ m.maxSSABlockID = label(maxBlockID)
+ m.nextLabel = label(maxBlockID) + 1
}
// SetCurrentABI implements backend.Machine SetCurrentABI.
@@ -168,12 +288,11 @@ func (m *machine) DisableStackCheck() {
// SetCompiler implements backend.Machine.
func (m *machine) SetCompiler(ctx backend.Compiler) {
m.compiler = ctx
- m.regAllocFn = backend.NewRegAllocFunction[*instruction, *machine](m, ctx.SSABuilder(), ctx)
+ m.regAllocFn.ssaB = ctx.SSABuilder()
}
func (m *machine) insert(i *instruction) {
- ectx := m.executableContext
- ectx.PendingInstructions = append(ectx.PendingInstructions, i)
+ m.pendingInstructions = append(m.pendingInstructions, i)
}
func (m *machine) insertBrTargetLabel() label {
@@ -183,18 +302,18 @@ func (m *machine) insertBrTargetLabel() label {
}
func (m *machine) allocateBrTarget() (nop *instruction, l label) {
- ectx := m.executableContext
- l = ectx.AllocateLabel()
+ l = m.nextLabel
+ m.nextLabel++
nop = m.allocateInstr()
nop.asNop0WithLabel(l)
- pos := ectx.GetOrAllocateLabelPosition(l)
- pos.Begin, pos.End = nop, nop
+ pos := m.labelPositionPool.GetOrAllocate(int(l))
+ pos.begin, pos.end = nop, nop
return
}
// allocateInstr allocates an instruction.
func (m *machine) allocateInstr() *instruction {
- instr := m.executableContext.InstructionPool.Allocate()
+ instr := m.instrPool.Allocate()
if !m.regAllocStarted {
instr.addedBeforeRegAlloc = true
}
@@ -251,7 +370,6 @@ func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruc
// resolveRelativeAddresses resolves the relative addresses before encoding.
func (m *machine) resolveRelativeAddresses(ctx context.Context) {
- ectx := m.executableContext
for {
if len(m.unresolvedAddressModes) > 0 {
arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP()
@@ -265,35 +383,36 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
var fn string
var fnIndex int
- var labelToSSABlockID map[label]ssa.BasicBlockID
+ var labelPosToLabel map[*labelPosition]label
if wazevoapi.PerfMapEnabled {
- fn = wazevoapi.GetCurrentFunctionName(ctx)
- labelToSSABlockID = make(map[label]ssa.BasicBlockID)
- for i, l := range ectx.SsaBlockIDToLabels {
- labelToSSABlockID[l] = ssa.BasicBlockID(i)
+ labelPosToLabel = make(map[*labelPosition]label)
+ for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ {
+ labelPosToLabel[m.labelPositionPool.Get(i)] = label(i)
}
+
+ fn = wazevoapi.GetCurrentFunctionName(ctx)
fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx)
}
// Next, in order to determine the offsets of relative jumps, we have to calculate the size of each label.
var offset int64
- for i, pos := range ectx.OrderedBlockLabels {
- pos.BinaryOffset = offset
+ for i, pos := range m.orderedSSABlockLabelPos {
+ pos.binaryOffset = offset
var size int64
- for cur := pos.Begin; ; cur = cur.next {
+ for cur := pos.begin; ; cur = cur.next {
switch cur.kind {
case nop0:
l := cur.nop0Label()
- if pos := ectx.LabelPositions[l]; pos != nil {
- pos.BinaryOffset = offset + size
+ if pos := m.labelPositionPool.Get(int(l)); pos != nil {
+ pos.binaryOffset = offset + size
}
case condBr:
if !cur.condBrOffsetResolved() {
var nextLabel label
- if i < len(ectx.OrderedBlockLabels)-1 {
+ if i < len(m.orderedSSABlockLabelPos)-1 {
// Note: this is only used when the block ends with fallthrough,
// therefore can be safely assumed that the next block exists when it's needed.
- nextLabel = ectx.OrderedBlockLabels[i+1].L
+ nextLabel = ssaBlockLabel(m.orderedSSABlockLabelPos[i+1].sb)
}
m.condBrRelocs = append(m.condBrRelocs, condBrReloc{
cbr: cur, currentLabelPos: pos, offset: offset + size,
@@ -302,21 +421,14 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
}
}
size += cur.size()
- if cur == pos.End {
+ if cur == pos.end {
break
}
}
if wazevoapi.PerfMapEnabled {
if size > 0 {
- l := pos.L
- var labelStr string
- if blkID, ok := labelToSSABlockID[l]; ok {
- labelStr = fmt.Sprintf("%s::SSA_Block[%s]", l, blkID)
- } else {
- labelStr = l.String()
- }
- wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelStr))
+ wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelPosToLabel[pos]))
}
}
offset += size
@@ -330,7 +442,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
offset := reloc.offset
target := cbr.condBrLabel()
- offsetOfTarget := ectx.LabelPositions[target].BinaryOffset
+ offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset
diff := offsetOfTarget - offset
if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 {
// This case the conditional branch is too huge. We place the trampoline instructions at the end of the current block,
@@ -351,11 +463,11 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
}
var currentOffset int64
- for cur := ectx.RootInstr; cur != nil; cur = cur.next {
+ for cur := m.rootInstr; cur != nil; cur = cur.next {
switch cur.kind {
case br:
target := cur.brLabel()
- offsetOfTarget := ectx.LabelPositions[target].BinaryOffset
+ offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset
diff := offsetOfTarget - currentOffset
divided := diff >> 2
if divided < minSignedInt26 || divided > maxSignedInt26 {
@@ -366,7 +478,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
case condBr:
if !cur.condBrOffsetResolved() {
target := cur.condBrLabel()
- offsetOfTarget := ectx.LabelPositions[target].BinaryOffset
+ offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset
diff := offsetOfTarget - currentOffset
if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 {
panic("BUG: branch relocation for large conditional branch larger than 19-bit range must be handled properly")
@@ -378,7 +490,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) {
targets := m.jmpTableTargets[tableIndex]
for i := range targets {
l := label(targets[i])
- offsetOfTarget := ectx.LabelPositions[l].BinaryOffset
+ offsetOfTarget := m.labelPositionPool.Get(int(l)).binaryOffset
diff := offsetOfTarget - (currentOffset + brTableSequenceOffsetTableBegin)
targets[i] = uint32(diff)
}
@@ -399,7 +511,7 @@ const (
)
func (m *machine) insertConditionalJumpTrampoline(cbr *instruction, currentBlk *labelPosition, nextLabel label) {
- cur := currentBlk.End
+ cur := currentBlk.end
originalTarget := cbr.condBrLabel()
endNext := cur.next
@@ -422,32 +534,27 @@ func (m *machine) insertConditionalJumpTrampoline(cbr *instruction, currentBlk *
cur = linkInstr(cur, br)
// Update the end of the current block.
- currentBlk.End = cur
+ currentBlk.end = cur
linkInstr(cur, endNext)
}
// Format implements backend.Machine.
func (m *machine) Format() string {
- ectx := m.executableContext
begins := map[*instruction]label{}
- for _, pos := range ectx.LabelPositions {
+ for l := label(0); l < m.nextLabel; l++ {
+ pos := m.labelPositionPool.Get(int(l))
if pos != nil {
- begins[pos.Begin] = pos.L
+ begins[pos.begin] = l
}
}
- irBlocks := map[label]ssa.BasicBlockID{}
- for i, l := range ectx.SsaBlockIDToLabels {
- irBlocks[l] = ssa.BasicBlockID(i)
- }
-
var lines []string
- for cur := ectx.RootInstr; cur != nil; cur = cur.next {
+ for cur := m.rootInstr; cur != nil; cur = cur.next {
if l, ok := begins[cur]; ok {
var labelStr string
- if blkID, ok := irBlocks[l]; ok {
- labelStr = fmt.Sprintf("%s (SSA Block: %s):", l, blkID)
+ if l <= m.maxSSABlockID {
+ labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, int(l))
} else {
labelStr = fmt.Sprintf("%s:", l)
}
@@ -508,13 +615,17 @@ func (m *machine) frameSize() int64 {
return s
}
-func (m *machine) addJmpTableTarget(targets []ssa.BasicBlock) (index int) {
- // TODO: reuse the slice!
- labels := make([]uint32, len(targets))
- for j, target := range targets {
- labels[j] = uint32(m.executableContext.GetOrAllocateSSABlockLabel(target))
+func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) {
+ if m.jmpTableTargetsNext == len(m.jmpTableTargets) {
+ m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View())))
+ }
+
+ index = m.jmpTableTargetsNext
+ m.jmpTableTargetsNext++
+ m.jmpTableTargets[index] = m.jmpTableTargets[index][:0]
+ for _, targetBlockID := range targets.View() {
+ target := m.compiler.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID))
+ m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(target.ID()))
}
- index = len(m.jmpTableTargets)
- m.jmpTableTargets = append(m.jmpTableTargets, labels)
return
}
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
index d9032f921..c646a8fab 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
@@ -15,9 +15,7 @@ func (m *machine) PostRegAlloc() {
// setupPrologue initializes the prologue of the function.
func (m *machine) setupPrologue() {
- ectx := m.executableContext
-
- cur := ectx.RootInstr
+ cur := m.rootInstr
prevInitInst := cur.next
//
@@ -196,21 +194,20 @@ func (m *machine) createFrameSizeSlot(cur *instruction, s int64) *instruction {
// 1. Removes the redundant copy instruction.
// 2. Inserts the epilogue.
func (m *machine) postRegAlloc() {
- ectx := m.executableContext
- for cur := ectx.RootInstr; cur != nil; cur = cur.next {
+ for cur := m.rootInstr; cur != nil; cur = cur.next {
switch cur.kind {
case ret:
m.setupEpilogueAfter(cur.prev)
case loadConstBlockArg:
lc := cur
next := lc.next
- m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
m.lowerLoadConstantBlockArgAfterRegAlloc(lc)
- for _, instr := range m.executableContext.PendingInstructions {
+ for _, instr := range m.pendingInstructions {
cur = linkInstr(cur, instr)
}
linkInstr(cur, next)
- m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
default:
// Removes the redundant copy instruction.
if cur.IsCopy() && cur.rn.realReg() == cur.rd.RealReg() {
@@ -432,11 +429,9 @@ func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instructi
// CompileStackGrowCallSequence implements backend.Machine.
func (m *machine) CompileStackGrowCallSequence() []byte {
- ectx := m.executableContext
-
cur := m.allocateInstr()
cur.asNop0()
- ectx.RootInstr = cur
+ m.rootInstr = cur
// Save the callee saved and argument registers.
cur = m.saveRegistersInExecutionContext(cur, saveRequiredRegs)
@@ -458,16 +453,14 @@ func (m *machine) CompileStackGrowCallSequence() []byte {
ret.asRet()
linkInstr(cur, ret)
- m.encode(ectx.RootInstr)
+ m.encode(m.rootInstr)
return m.compiler.Buf()
}
func (m *machine) addsAddOrSubStackPointer(cur *instruction, rd regalloc.VReg, diff int64, add bool) *instruction {
- ectx := m.executableContext
-
- ectx.PendingInstructions = ectx.PendingInstructions[:0]
+ m.pendingInstructions = m.pendingInstructions[:0]
m.insertAddOrSubStackPointer(rd, diff, add)
- for _, inserted := range ectx.PendingInstructions {
+ for _, inserted := range m.pendingInstructions {
cur = linkInstr(cur, inserted)
}
return cur
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
index c7eb92cc2..f2ed53ae5 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
@@ -3,18 +3,226 @@ package arm64
// This file implements the interfaces required for register allocations. See backend.RegAllocFunctionMachine.
import (
- "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
)
-// ClobberedRegisters implements backend.RegAllocFunctionMachine.
-func (m *machine) ClobberedRegisters(regs []regalloc.VReg) {
- m.clobberedRegs = append(m.clobberedRegs[:0], regs...)
+// regAllocFn implements regalloc.Function.
+type regAllocFn struct {
+ ssaB ssa.Builder
+ m *machine
+ loopNestingForestRoots []ssa.BasicBlock
+ blockIter int
}
-// Swap implements backend.RegAllocFunctionMachine.
-func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) {
+// PostOrderBlockIteratorBegin implements regalloc.Function.
+func (f *regAllocFn) PostOrderBlockIteratorBegin() *labelPosition {
+ f.blockIter = len(f.m.orderedSSABlockLabelPos) - 1
+ return f.PostOrderBlockIteratorNext()
+}
+
+// PostOrderBlockIteratorNext implements regalloc.Function.
+func (f *regAllocFn) PostOrderBlockIteratorNext() *labelPosition {
+ if f.blockIter < 0 {
+ return nil
+ }
+ b := f.m.orderedSSABlockLabelPos[f.blockIter]
+ f.blockIter--
+ return b
+}
+
+// ReversePostOrderBlockIteratorBegin implements regalloc.Function.
+func (f *regAllocFn) ReversePostOrderBlockIteratorBegin() *labelPosition {
+ f.blockIter = 0
+ return f.ReversePostOrderBlockIteratorNext()
+}
+
+// ReversePostOrderBlockIteratorNext implements regalloc.Function.
+func (f *regAllocFn) ReversePostOrderBlockIteratorNext() *labelPosition {
+ if f.blockIter >= len(f.m.orderedSSABlockLabelPos) {
+ return nil
+ }
+ b := f.m.orderedSSABlockLabelPos[f.blockIter]
+ f.blockIter++
+ return b
+}
+
+// ClobberedRegisters implements regalloc.Function.
+func (f *regAllocFn) ClobberedRegisters(regs []regalloc.VReg) {
+ f.m.clobberedRegs = append(f.m.clobberedRegs[:0], regs...)
+}
+
+// LoopNestingForestRoots implements regalloc.Function.
+func (f *regAllocFn) LoopNestingForestRoots() int {
+ f.loopNestingForestRoots = f.ssaB.LoopNestingForestRoots()
+ return len(f.loopNestingForestRoots)
+}
+
+// LoopNestingForestRoot implements regalloc.Function.
+func (f *regAllocFn) LoopNestingForestRoot(i int) *labelPosition {
+ root := f.loopNestingForestRoots[i]
+ pos := f.m.getOrAllocateSSABlockLabelPosition(root)
+ return pos
+}
+
+// LowestCommonAncestor implements regalloc.Function.
+func (f *regAllocFn) LowestCommonAncestor(blk1, blk2 *labelPosition) *labelPosition {
+ sb := f.ssaB.LowestCommonAncestor(blk1.sb, blk2.sb)
+ pos := f.m.getOrAllocateSSABlockLabelPosition(sb)
+ return pos
+}
+
+// Idom implements regalloc.Function.
+func (f *regAllocFn) Idom(blk *labelPosition) *labelPosition {
+ sb := f.ssaB.Idom(blk.sb)
+ pos := f.m.getOrAllocateSSABlockLabelPosition(sb)
+ return pos
+}
+
+// SwapBefore implements regalloc.Function.
+func (f *regAllocFn) SwapBefore(x1, x2, tmp regalloc.VReg, instr *instruction) {
+ f.m.swap(instr.prev, x1, x2, tmp)
+}
+
+// StoreRegisterBefore implements regalloc.Function.
+func (f *regAllocFn) StoreRegisterBefore(v regalloc.VReg, instr *instruction) {
+ m := f.m
+ m.insertStoreRegisterAt(v, instr, false)
+}
+
+// StoreRegisterAfter implements regalloc.Function.
+func (f *regAllocFn) StoreRegisterAfter(v regalloc.VReg, instr *instruction) {
+ m := f.m
+ m.insertStoreRegisterAt(v, instr, true)
+}
+
+// ReloadRegisterBefore implements regalloc.Function.
+func (f *regAllocFn) ReloadRegisterBefore(v regalloc.VReg, instr *instruction) {
+ m := f.m
+ m.insertReloadRegisterAt(v, instr, false)
+}
+
+// ReloadRegisterAfter implements regalloc.Function.
+func (f *regAllocFn) ReloadRegisterAfter(v regalloc.VReg, instr *instruction) {
+ m := f.m
+ m.insertReloadRegisterAt(v, instr, true)
+}
+
+// InsertMoveBefore implements regalloc.Function.
+func (f *regAllocFn) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) {
+ f.m.insertMoveBefore(dst, src, instr)
+}
+
+// LoopNestingForestChild implements regalloc.Function.
+func (f *regAllocFn) LoopNestingForestChild(pos *labelPosition, i int) *labelPosition {
+ childSB := pos.sb.LoopNestingForestChildren()[i]
+ return f.m.getOrAllocateSSABlockLabelPosition(childSB)
+}
+
+// Succ implements regalloc.Block.
+func (f *regAllocFn) Succ(pos *labelPosition, i int) *labelPosition {
+ succSB := pos.sb.Succ(i)
+ if succSB.ReturnBlock() {
+ return nil
+ }
+ return f.m.getOrAllocateSSABlockLabelPosition(succSB)
+}
+
+// Pred implements regalloc.Block.
+func (f *regAllocFn) Pred(pos *labelPosition, i int) *labelPosition {
+ predSB := pos.sb.Pred(i)
+ return f.m.getOrAllocateSSABlockLabelPosition(predSB)
+}
+
+// BlockParams implements regalloc.Function.
+func (f *regAllocFn) BlockParams(pos *labelPosition, regs *[]regalloc.VReg) []regalloc.VReg {
+ c := f.m.compiler
+ *regs = (*regs)[:0]
+ for i := 0; i < pos.sb.Params(); i++ {
+ v := c.VRegOf(pos.sb.Param(i))
+ *regs = append(*regs, v)
+ }
+ return *regs
+}
+
+// ID implements regalloc.Block.
+func (pos *labelPosition) ID() int32 {
+ return int32(pos.sb.ID())
+}
+
+// InstrIteratorBegin implements regalloc.Block.
+func (pos *labelPosition) InstrIteratorBegin() *instruction {
+ ret := pos.begin
+ pos.cur = ret
+ return ret
+}
+
+// InstrIteratorNext implements regalloc.Block.
+func (pos *labelPosition) InstrIteratorNext() *instruction {
+ for {
+ if pos.cur == pos.end {
+ return nil
+ }
+ instr := pos.cur.next
+ pos.cur = instr
+ if instr == nil {
+ return nil
+ } else if instr.addedBeforeRegAlloc {
+ // Only concerned about the instruction added before regalloc.
+ return instr
+ }
+ }
+}
+
+// InstrRevIteratorBegin implements regalloc.Block.
+func (pos *labelPosition) InstrRevIteratorBegin() *instruction {
+ pos.cur = pos.end
+ return pos.cur
+}
+
+// InstrRevIteratorNext implements regalloc.Block.
+func (pos *labelPosition) InstrRevIteratorNext() *instruction {
+ for {
+ if pos.cur == pos.begin {
+ return nil
+ }
+ instr := pos.cur.prev
+ pos.cur = instr
+ if instr == nil {
+ return nil
+ } else if instr.addedBeforeRegAlloc {
+ // Only concerned about the instruction added before regalloc.
+ return instr
+ }
+ }
+}
+
+// FirstInstr implements regalloc.Block.
+func (pos *labelPosition) FirstInstr() *instruction { return pos.begin }
+
+// LastInstrForInsertion implements regalloc.Block.
+func (pos *labelPosition) LastInstrForInsertion() *instruction {
+ return lastInstrForInsertion(pos.begin, pos.end)
+}
+
+// Preds implements regalloc.Block.
+func (pos *labelPosition) Preds() int { return pos.sb.Preds() }
+
+// Entry implements regalloc.Block.
+func (pos *labelPosition) Entry() bool { return pos.sb.EntryBlock() }
+
+// Succs implements regalloc.Block.
+func (pos *labelPosition) Succs() int { return pos.sb.Succs() }
+
+// LoopHeader implements regalloc.Block.
+func (pos *labelPosition) LoopHeader() bool { return pos.sb.LoopHeader() }
+
+// LoopNestingForestChildren implements regalloc.Block.
+func (pos *labelPosition) LoopNestingForestChildren() int {
+ return len(pos.sb.LoopNestingForestChildren())
+}
+
+func (m *machine) swap(cur *instruction, x1, x2, tmp regalloc.VReg) {
prevNext := cur.next
var mov1, mov2, mov3 *instruction
if x1.RegType() == regalloc.RegTypeInt {
@@ -32,12 +240,12 @@ func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) {
if !tmp.Valid() {
r2 := x2.RealReg()
// Temporarily spill x1 to stack.
- cur = m.InsertStoreRegisterAt(x1, cur, true).prev
+ cur = m.insertStoreRegisterAt(x1, cur, true).prev
// Then move x2 to x1.
cur = linkInstr(cur, m.allocateInstr().asFpuMov128(x1, x2))
linkInstr(cur, prevNext)
// Then reload the original value on x1 from stack to r2.
- m.InsertReloadRegisterAt(x1.SetRealReg(r2), cur, true)
+ m.insertReloadRegisterAt(x1.SetRealReg(r2), cur, true)
} else {
mov1 = m.allocateInstr().asFpuMov128(tmp, x1)
mov2 = m.allocateInstr().asFpuMov128(x1, x2)
@@ -50,8 +258,7 @@ func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) {
}
}
-// InsertMoveBefore implements backend.RegAllocFunctionMachine.
-func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) {
+func (m *machine) insertMoveBefore(dst, src regalloc.VReg, instr *instruction) {
typ := src.RegType()
if typ != dst.RegType() {
panic("BUG: src and dst must have the same type")
@@ -70,13 +277,7 @@ func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) {
linkInstr(cur, prevNext)
}
-// SSABlockLabel implements backend.RegAllocFunctionMachine.
-func (m *machine) SSABlockLabel(id ssa.BasicBlockID) backend.Label {
- return m.executableContext.SsaBlockIDToLabels[id]
-}
-
-// InsertStoreRegisterAt implements backend.RegAllocFunctionMachine.
-func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction {
+func (m *machine) insertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction {
if !v.IsRealReg() {
panic("BUG: VReg must be backed by real reg to be stored")
}
@@ -100,8 +301,7 @@ func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, aft
return linkInstr(cur, prevNext)
}
-// InsertReloadRegisterAt implements backend.RegAllocFunctionMachine.
-func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction {
+func (m *machine) insertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction {
if !v.IsRealReg() {
panic("BUG: VReg must be backed by real reg to be stored")
}
@@ -134,8 +334,7 @@ func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, af
return linkInstr(cur, prevNext)
}
-// LastInstrForInsertion implements backend.RegAllocFunctionMachine.
-func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction {
+func lastInstrForInsertion(begin, end *instruction) *instruction {
cur := end
for cur.kind == nop0 {
cur = cur.prev
diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go
index edb0e36e3..a72b86f6b 100644
--- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go
+++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go
@@ -14,7 +14,7 @@ func UnwindStack(sp, _, top uintptr, returnAddresses []uintptr) []uintptr {
var stackBuf []byte
{
- // TODO: use unsafe.Slice after floor version is set to Go 1.20.
+ //nolint:staticcheck
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&stackBuf))
hdr.Data = sp
hdr.Len = l
@@ -78,13 +78,7 @@ func GoCallStackView(stackPointerBeforeGoCall *uint64) []uint64 {
// +-----------------+ <---- stackPointerBeforeGoCall
// (low address)
ptr := unsafe.Pointer(stackPointerBeforeGoCall)
+ data := (*uint64)(unsafe.Add(ptr, 16)) // skips the (frame_size, sliceSize).
size := *(*uint64)(unsafe.Add(ptr, 8))
- var view []uint64
- {
- sh := (*reflect.SliceHeader)(unsafe.Pointer(&view))
- sh.Data = uintptr(unsafe.Add(ptr, 16)) // skips the (frame_size, sliceSize).
- sh.Len = int(size)
- sh.Cap = int(size)
- }
- return view
+ return unsafe.Slice(data, size)
}