diff options
Diffstat (limited to 'vendor/github.com/bytedance/sonic/internal/rt')
7 files changed, 723 insertions, 0 deletions
diff --git a/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s b/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s new file mode 100644 index 000000000..8250e38d3 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s @@ -0,0 +1,60 @@ +// +build !noasm !appengine +// Code generated by asm2asm, DO NOT EDIT· + +#include "go_asm.h" +#include "funcdata.h" +#include "textflag.h" + +TEXT ·MoreStack(SB), NOSPLIT, $0 - 8 + NO_LOCAL_POINTERS +_entry: + MOVQ (TLS), R14 + MOVQ size+0(FP), R12 + NOTQ R12 + LEAQ (SP)(R12*1), R12 + CMPQ R12, 16(R14) + JBE _stack_grow + RET +_stack_grow: + CALL runtime·morestack_noctxt<>(SB) + JMP _entry + + +TEXT ·StopProf(SB), NOSPLIT, $0-0 + NO_LOCAL_POINTERS + CMPB github·com∕bytedance∕sonic∕internal∕rt·StopProfiling(SB), $0 + JEQ _ret_1 + MOVL $1, AX + LEAQ github·com∕bytedance∕sonic∕internal∕rt·yieldCount(SB), CX + LOCK + XADDL AX, (CX) + MOVL runtime·prof+4(SB), AX + TESTL AX, AX + JEQ _ret_1 + MOVL AX, github·com∕bytedance∕sonic∕internal∕rt·oldHz(SB) + MOVL $0, runtime·prof+4(SB) +_ret_1: + RET + + +TEXT ·StartProf(SB), NOSPLIT, $0-0 + NO_LOCAL_POINTERS + CMPB github·com∕bytedance∕sonic∕internal∕rt·StopProfiling(SB), $0 + JEQ _ret_2 + MOVL $-1, AX + LEAQ github·com∕bytedance∕sonic∕internal∕rt·yieldCount(SB), CX + LOCK + XADDL AX, (CX) + CMPL github·com∕bytedance∕sonic∕internal∕rt·yieldCount(SB), $0 + JNE _ret_2 + CMPL runtime·prof+4(SB), $0 + JNE _ret_2 + CMPL github·com∕bytedance∕sonic∕internal∕rt·oldHz(SB), $0 + JNE _branch_1 + MOVL $100, github·com∕bytedance∕sonic∕internal∕rt·oldHz(SB) +_branch_1: + MOVL github·com∕bytedance∕sonic∕internal∕rt·oldHz(SB), AX + MOVL AX, runtime·prof+4(SB) +_ret_2: + RET +
\ No newline at end of file diff --git a/vendor/github.com/bytedance/sonic/internal/rt/asm_arm64.s b/vendor/github.com/bytedance/sonic/internal/rt/asm_arm64.s new file mode 100644 index 000000000..a168a8266 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/asm_arm64.s @@ -0,0 +1,10 @@ +// +build !noasm !appengine +// Code generated by asm2asm, DO NOT EDIT. + +#include "go_asm.h" +#include "funcdata.h" +#include "textflag.h" + +TEXT ·MoreStack(SB), NOSPLIT, $0 - 8 + NO_LOCAL_POINTERS + RET diff --git a/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go b/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go new file mode 100644 index 000000000..358ce80ce --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go @@ -0,0 +1,113 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +import ( + `unsafe` + `reflect` +) + +//go:nosplit +func Get16(v []byte) int16 { + return *(*int16)((*GoSlice)(unsafe.Pointer(&v)).Ptr) +} + +//go:nosplit +func Get32(v []byte) int32 { + return *(*int32)((*GoSlice)(unsafe.Pointer(&v)).Ptr) +} + +//go:nosplit +func Get64(v []byte) int64 { + return *(*int64)((*GoSlice)(unsafe.Pointer(&v)).Ptr) +} + +//go:nosplit +func Mem2Str(v []byte) (s string) { + (*GoString)(unsafe.Pointer(&s)).Len = (*GoSlice)(unsafe.Pointer(&v)).Len + (*GoString)(unsafe.Pointer(&s)).Ptr = (*GoSlice)(unsafe.Pointer(&v)).Ptr + return +} + +//go:nosplit +func Str2Mem(s string) (v []byte) { + (*GoSlice)(unsafe.Pointer(&v)).Cap = (*GoString)(unsafe.Pointer(&s)).Len + (*GoSlice)(unsafe.Pointer(&v)).Len = (*GoString)(unsafe.Pointer(&s)).Len + (*GoSlice)(unsafe.Pointer(&v)).Ptr = (*GoString)(unsafe.Pointer(&s)).Ptr + return +} + +func BytesFrom(p unsafe.Pointer, n int, c int) (r []byte) { + (*GoSlice)(unsafe.Pointer(&r)).Ptr = p + (*GoSlice)(unsafe.Pointer(&r)).Len = n + (*GoSlice)(unsafe.Pointer(&r)).Cap = c + return +} + +func FuncAddr(f interface{}) unsafe.Pointer { + if vv := UnpackEface(f); vv.Type.Kind() != reflect.Func { + panic("f is not a function") + } else { + return *(*unsafe.Pointer)(vv.Value) + } +} + +func IndexChar(src string, index int) unsafe.Pointer { + return unsafe.Pointer(uintptr((*GoString)(unsafe.Pointer(&src)).Ptr) + uintptr(index)) +} + +func IndexByte(ptr []byte, index int) unsafe.Pointer { + return unsafe.Pointer(uintptr((*GoSlice)(unsafe.Pointer(&ptr)).Ptr) + uintptr(index)) +} + +//go:nosplit +func GuardSlice(buf *[]byte, n int) { + c := cap(*buf) + l := len(*buf) + if c-l < n { + c = c>>1 + n + l + if c < 32 { + c = 32 + } + tmp := make([]byte, l, c) + copy(tmp, *buf) + *buf = tmp + } + return +} + +//go:nosplit +func Ptr2SlicePtr(s unsafe.Pointer, l int, c int) unsafe.Pointer { + slice := &GoSlice{ + Ptr: s, + Len: l, + Cap: c, + } + return unsafe.Pointer(slice) +} + +//go:nosplit +func StrPtr(s string) unsafe.Pointer { + return (*GoString)(unsafe.Pointer(&s)).Ptr +} + +//go:nosplit +func StrFrom(p unsafe.Pointer, n int64) (s string) { + (*GoString)(unsafe.Pointer(&s)).Ptr = p + (*GoString)(unsafe.Pointer(&s)).Len = int(n) + return +}
\ No newline at end of file diff --git a/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go b/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go new file mode 100644 index 000000000..87df6b94a --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go @@ -0,0 +1,200 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +import ( + `reflect` + `unsafe` +) + +var ( + reflectRtypeItab = findReflectRtypeItab() +) + +const ( + F_direct = 1 << 5 + F_kind_mask = (1 << 5) - 1 +) + +type GoType struct { + Size uintptr + PtrData uintptr + Hash uint32 + Flags uint8 + Align uint8 + FieldAlign uint8 + KindFlags uint8 + Traits unsafe.Pointer + GCData *byte + Str int32 + PtrToSelf int32 +} + +func (self *GoType) Kind() reflect.Kind { + return reflect.Kind(self.KindFlags & F_kind_mask) +} + +func (self *GoType) Pack() (t reflect.Type) { + (*GoIface)(unsafe.Pointer(&t)).Itab = reflectRtypeItab + (*GoIface)(unsafe.Pointer(&t)).Value = unsafe.Pointer(self) + return +} + +func (self *GoType) String() string { + return self.Pack().String() +} + +func (self *GoType) Indirect() bool { + return self.KindFlags & F_direct == 0 +} + +type GoMap struct { + Count int + Flags uint8 + B uint8 + Overflow uint16 + Hash0 uint32 + Buckets unsafe.Pointer + OldBuckets unsafe.Pointer + Evacuate uintptr + Extra unsafe.Pointer +} + +type GoMapIterator struct { + K unsafe.Pointer + V unsafe.Pointer + T *GoMapType + H *GoMap + Buckets unsafe.Pointer + Bptr *unsafe.Pointer + Overflow *[]unsafe.Pointer + OldOverflow *[]unsafe.Pointer + StartBucket uintptr + Offset uint8 + Wrapped bool + B uint8 + I uint8 + Bucket uintptr + CheckBucket uintptr +} + +type GoItab struct { + it unsafe.Pointer + Vt *GoType + hv uint32 + _ [4]byte + fn [1]uintptr +} + +type GoIface struct { + Itab *GoItab + Value unsafe.Pointer +} + +type GoEface struct { + Type *GoType + Value unsafe.Pointer +} + +func (self GoEface) Pack() (v interface{}) { + *(*GoEface)(unsafe.Pointer(&v)) = self + return +} + +type GoPtrType struct { + GoType + Elem *GoType +} + +type GoMapType struct { + GoType + Key *GoType + Elem *GoType + Bucket *GoType + Hasher func(unsafe.Pointer, uintptr) uintptr + KeySize uint8 + ElemSize uint8 + BucketSize uint16 + Flags uint32 +} + +func (self *GoMapType) IndirectElem() bool { + return self.Flags & 2 != 0 +} + +type GoStructType struct { + GoType + Pkg *byte + Fields []GoStructField +} + +type GoStructField struct { + Name *byte + Type *GoType + OffEmbed uintptr +} + +type GoInterfaceType struct { + GoType + PkgPath *byte + Methods []GoInterfaceMethod +} + +type GoInterfaceMethod struct { + Name int32 + Type int32 +} + +type GoSlice struct { + Ptr unsafe.Pointer + Len int + Cap int +} + +type GoString struct { + Ptr unsafe.Pointer + Len int +} + +func PtrElem(t *GoType) *GoType { + return (*GoPtrType)(unsafe.Pointer(t)).Elem +} + +func MapType(t *GoType) *GoMapType { + return (*GoMapType)(unsafe.Pointer(t)) +} + +func IfaceType(t *GoType) *GoInterfaceType { + return (*GoInterfaceType)(unsafe.Pointer(t)) +} + +func UnpackType(t reflect.Type) *GoType { + return (*GoType)((*GoIface)(unsafe.Pointer(&t)).Value) +} + +func UnpackEface(v interface{}) GoEface { + return *(*GoEface)(unsafe.Pointer(&v)) +} + +func UnpackIface(v interface{}) GoIface { + return *(*GoIface)(unsafe.Pointer(&v)) +} + +func findReflectRtypeItab() *GoItab { + v := reflect.TypeOf(struct{}{}) + return (*GoIface)(unsafe.Pointer(&v)).Itab +} diff --git a/vendor/github.com/bytedance/sonic/internal/rt/gcwb.go b/vendor/github.com/bytedance/sonic/internal/rt/gcwb.go new file mode 100644 index 000000000..c3217c899 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/gcwb.go @@ -0,0 +1,124 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +import ( + `os` + `sync/atomic` + `unsafe` + + `golang.org/x/arch/x86/x86asm` +) + +const ( + _MaxInstr = 15 +) + +func isvar(arg x86asm.Arg) bool { + v, ok := arg.(x86asm.Mem) + return ok && v.Base == x86asm.RIP +} + +func iszero(arg x86asm.Arg) bool { + v, ok := arg.(x86asm.Imm) + return ok && v == 0 +} + +func GcwbAddr() uintptr { + var err error + var off uintptr + var ins x86asm.Inst + + /* get the function address */ + pc := uintptr(0) + fp := FuncAddr(atomic.StorePointer) + + /* search within the first 16 instructions */ + for i := 0; i < 16; i++ { + mem := unsafe.Pointer(uintptr(fp) + pc) + buf := BytesFrom(mem, _MaxInstr, _MaxInstr) + + /* disassemble the instruction */ + if ins, err = x86asm.Decode(buf, 64); err != nil { + panic("gcwbaddr: " + err.Error()) + } + + /* check for a byte comparison with zero */ + if ins.Op == x86asm.CMP && ins.MemBytes == 1 && isvar(ins.Args[0]) && iszero(ins.Args[1]) { + off = pc + uintptr(ins.Len) + uintptr(ins.Args[0].(x86asm.Mem).Disp) + break + } + + /* move to next instruction */ + nb := ins.Len + pc += uintptr(nb) + } + + /* check for address */ + if off == 0 { + panic("gcwbaddr: could not locate the variable `writeBarrier`") + } else { + return uintptr(fp) + off + } +} + +// StopProfiling is used to stop traceback introduced by SIGPROF while native code is running. +// WARN: this option is only a workaround for traceback issue (https://github.com/bytedance/sonic/issues/310), +// and will be dropped when the issue is fixed. +var StopProfiling = os.Getenv("SONIC_STOP_PROFILING") != "" + +// WARN: must be aligned with runtime.Prof +// type Prof struct { +// signalLock uint32 +// hz int32 +// } + +var ( + // // go:linkname runtimeProf runtime.prof + // runtimeProf Prof + + // count of native-C calls + yieldCount uint32 + + // previous value of runtimeProf.hz + oldHz int32 +) + +//go:nosplit +func MoreStack(size uintptr) + +func StopProf() + +// func StopProf() { +// atomic.AddUint32(&yieldCount, 1) +// if runtimeProf.hz != 0 { +// oldHz = runtimeProf.hz +// runtimeProf.hz = 0 +// } +// } + +func StartProf() + +// func StartProf() { +// atomic.AddUint32(&yieldCount, ^uint32(0)) +// if yieldCount == 0 && runtimeProf.hz == 0 { +// if oldHz == 0 { +// oldHz = 100 +// } +// runtimeProf.hz = oldHz +// } +// } diff --git a/vendor/github.com/bytedance/sonic/internal/rt/int48.go b/vendor/github.com/bytedance/sonic/internal/rt/int48.go new file mode 100644 index 000000000..e9f82d731 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/int48.go @@ -0,0 +1,36 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +const ( + MinInt48 = -(1 << 47) + MaxInt48 = +(1 << 47) - 1 +) + +func PackInt(v int) uint64 { + if u := uint64(v); v < MinInt48 || v > MaxInt48 { + panic("int48 out of range") + } else { + return ((u >> 63) << 47) | (u & 0x00007fffffffffff) + } +} + +func UnpackInt(v uint64) int { + v &= 0x0000ffffffffffff + v |= (v >> 47) * (0xffff << 48) + return int(v) +} diff --git a/vendor/github.com/bytedance/sonic/internal/rt/stackmap.go b/vendor/github.com/bytedance/sonic/internal/rt/stackmap.go new file mode 100644 index 000000000..e2c28c598 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/rt/stackmap.go @@ -0,0 +1,180 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +import ( + `fmt` + `strings` + `unsafe` + +) + +type Bitmap struct { + N int + B []byte +} + +func (self *Bitmap) grow() { + if self.N >= len(self.B) * 8 { + self.B = append(self.B, 0) + } +} + +func (self *Bitmap) mark(i int, bv int) { + if bv != 0 { + self.B[i / 8] |= 1 << (i % 8) + } else { + self.B[i / 8] &^= 1 << (i % 8) + } +} + +func (self *Bitmap) Set(i int, bv int) { + if i >= self.N { + panic("bitmap: invalid bit position") + } else { + self.mark(i, bv) + } +} + +func (self *Bitmap) Append(bv int) { + self.grow() + self.mark(self.N, bv) + self.N++ +} + +func (self *Bitmap) AppendMany(n int, bv int) { + for i := 0; i < n; i++ { + self.Append(bv) + } +} + +// var ( +// _stackMapLock = sync.Mutex{} +// _stackMapCache = make(map[*StackMap]struct{}) +// ) + +type BitVec struct { + N uintptr + B unsafe.Pointer +} + +func (self BitVec) Bit(i uintptr) byte { + return (*(*byte)(unsafe.Pointer(uintptr(self.B) + i / 8)) >> (i % 8)) & 1 +} + +func (self BitVec) String() string { + var i uintptr + var v []string + + /* add each bit */ + for i = 0; i < self.N; i++ { + v = append(v, fmt.Sprintf("%d", self.Bit(i))) + } + + /* join them together */ + return fmt.Sprintf( + "BitVec { %s }", + strings.Join(v, ", "), + ) +} + +type StackMap struct { + N int32 + L int32 + B [1]byte +} + +// func (self *StackMap) add() { +// _stackMapLock.Lock() +// _stackMapCache[self] = struct{}{} +// _stackMapLock.Unlock() +// } + +func (self *StackMap) Pin() uintptr { + // self.add() + return uintptr(unsafe.Pointer(self)) +} + +func (self *StackMap) Get(i int32) BitVec { + return BitVec { + N: uintptr(self.L), + B: unsafe.Pointer(uintptr(unsafe.Pointer(&self.B)) + uintptr(i * ((self.L + 7) >> 3))), + } +} + +func (self *StackMap) String() string { + sb := strings.Builder{} + sb.WriteString("StackMap {") + + /* dump every stack map */ + for i := int32(0); i < self.N; i++ { + sb.WriteRune('\n') + sb.WriteString(" " + self.Get(i).String()) + } + + /* close the stackmap */ + sb.WriteString("\n}") + return sb.String() +} + +func (self *StackMap) MarshalBinary() ([]byte, error) { + size := int(self.N) * int(self.L) + int(unsafe.Sizeof(self.L)) + int(unsafe.Sizeof(self.N)) + return BytesFrom(unsafe.Pointer(self), size, size), nil +} + +var ( + byteType = UnpackEface(byte(0)).Type +) + +const ( + _StackMapSize = unsafe.Sizeof(StackMap{}) +) + +//go:linkname mallocgc runtime.mallocgc +//goland:noinspection GoUnusedParameter +func mallocgc(nb uintptr, vt *GoType, zero bool) unsafe.Pointer + +type StackMapBuilder struct { + b Bitmap +} + +func (self *StackMapBuilder) Build() (p *StackMap) { + nb := len(self.b.B) + bm := mallocgc(_StackMapSize + uintptr(nb) - 1, byteType, false) + + /* initialize as 1 bitmap of N bits */ + p = (*StackMap)(bm) + p.N, p.L = 1, int32(self.b.N) + copy(BytesFrom(unsafe.Pointer(&p.B), nb, nb), self.b.B) + return +} + +func (self *StackMapBuilder) AddField(ptr bool) { + if ptr { + self.b.Append(1) + } else { + self.b.Append(0) + } +} + +func (self *StackMapBuilder) AddFields(n int, ptr bool) { + if ptr { + self.b.AppendMany(n, 1) + } else { + self.b.AppendMany(n, 0) + } +}
\ No newline at end of file |