diff options
Diffstat (limited to 'vendor/github.com/bytedance/sonic/internal/jit')
5 files changed, 510 insertions, 0 deletions
diff --git a/vendor/github.com/bytedance/sonic/internal/jit/arch_amd64.go b/vendor/github.com/bytedance/sonic/internal/jit/arch_amd64.go new file mode 100644 index 000000000..7405052d6 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/jit/arch_amd64.go @@ -0,0 +1,67 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jit + +import ( + `github.com/twitchyliquid64/golang-asm/asm/arch` + `github.com/twitchyliquid64/golang-asm/obj` +) + +var ( + _AC = arch.Set("amd64") +) + +func As(op string) obj.As { + if ret, ok := _AC.Instructions[op]; ok { + return ret + } else { + panic("invalid instruction: " + op) + } +} + +func Imm(imm int64) obj.Addr { + return obj.Addr { + Type : obj.TYPE_CONST, + Offset : imm, + } +} + +func Reg(reg string) obj.Addr { + if ret, ok := _AC.Register[reg]; ok { + return obj.Addr{Reg: ret, Type: obj.TYPE_REG} + } else { + panic("invalid register name: " + reg) + } +} + +func Ptr(reg obj.Addr, offs int64) obj.Addr { + return obj.Addr { + Reg : reg.Reg, + Type : obj.TYPE_MEM, + Offset : offs, + } +} + +func Sib(reg obj.Addr, idx obj.Addr, scale int16, offs int64) obj.Addr { + return obj.Addr { + Reg : reg.Reg, + Index : idx.Reg, + Scale : scale, + Type : obj.TYPE_MEM, + Offset : offs, + } +} diff --git a/vendor/github.com/bytedance/sonic/internal/jit/asm.s b/vendor/github.com/bytedance/sonic/internal/jit/asm.s new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/jit/asm.s diff --git a/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go b/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go new file mode 100644 index 000000000..00e6009db --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go @@ -0,0 +1,269 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jit + +import ( + `encoding/binary` + `strconv` + `strings` + `sync` + + `github.com/bytedance/sonic/loader` + `github.com/bytedance/sonic/internal/rt` + `github.com/twitchyliquid64/golang-asm/obj` + `github.com/twitchyliquid64/golang-asm/obj/x86` +) + +const ( + _LB_jump_pc = "_jump_pc_" +) + +type BaseAssembler struct { + i int + f func() + c []byte + o sync.Once + pb *Backend + xrefs map[string][]*obj.Prog + labels map[string]*obj.Prog + pendings map[string][]*obj.Prog +} + +/** Instruction Encoders **/ + +var _NOPS = [][16]byte { + {0x90}, // NOP + {0x66, 0x90}, // 66 NOP + {0x0f, 0x1f, 0x00}, // NOP DWORD ptr [EAX] + {0x0f, 0x1f, 0x40, 0x00}, // NOP DWORD ptr [EAX + 00H] + {0x0f, 0x1f, 0x44, 0x00, 0x00}, // NOP DWORD ptr [EAX + EAX*1 + 00H] + {0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00}, // 66 NOP DWORD ptr [EAX + EAX*1 + 00H] + {0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00}, // NOP DWORD ptr [EAX + 00000000H] + {0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, // NOP DWORD ptr [EAX + EAX*1 + 00000000H] + {0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, // 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] +} + +func (self *BaseAssembler) NOP() *obj.Prog { + p := self.pb.New() + p.As = obj.ANOP + self.pb.Append(p) + return p +} + +func (self *BaseAssembler) NOPn(n int) { + for i := len(_NOPS); i > 0 && n > 0; i-- { + for ; n >= i; n -= i { + self.Byte(_NOPS[i - 1][:i]...) + } + } +} + +func (self *BaseAssembler) Byte(v ...byte) { + for ; len(v) >= 8; v = v[8:] { self.From("QUAD", Imm(rt.Get64(v))) } + for ; len(v) >= 4; v = v[4:] { self.From("LONG", Imm(int64(rt.Get32(v)))) } + for ; len(v) >= 2; v = v[2:] { self.From("WORD", Imm(int64(rt.Get16(v)))) } + for ; len(v) >= 1; v = v[1:] { self.From("BYTE", Imm(int64(v[0]))) } +} + +func (self *BaseAssembler) Mark(pc int) { + self.i++ + self.Link(_LB_jump_pc + strconv.Itoa(pc)) +} + +func (self *BaseAssembler) Link(to string) { + var p *obj.Prog + var v []*obj.Prog + + /* placeholder substitution */ + if strings.Contains(to, "{n}") { + to = strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + } + + /* check for duplications */ + if _, ok := self.labels[to]; ok { + panic("label " + to + " has already been linked") + } + + /* get the pending links */ + p = self.NOP() + v = self.pendings[to] + + /* patch all the pending jumps */ + for _, q := range v { + q.To.Val = p + } + + /* mark the label as resolved */ + self.labels[to] = p + delete(self.pendings, to) +} + +func (self *BaseAssembler) Xref(pc int, d int64) { + self.Sref(_LB_jump_pc + strconv.Itoa(pc), d) +} + +func (self *BaseAssembler) Sref(to string, d int64) { + p := self.pb.New() + p.As = x86.ALONG + p.From = Imm(-d) + + /* placeholder substitution */ + if strings.Contains(to, "{n}") { + to = strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + } + + /* record the patch point */ + self.pb.Append(p) + self.xrefs[to] = append(self.xrefs[to], p) +} + +func (self *BaseAssembler) Xjmp(op string, to int) { + self.Sjmp(op, _LB_jump_pc + strconv.Itoa(to)) +} + +func (self *BaseAssembler) Sjmp(op string, to string) { + p := self.pb.New() + p.As = As(op) + + /* placeholder substitution */ + if strings.Contains(to, "{n}") { + to = strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + } + + /* check for backward jumps */ + if v, ok := self.labels[to]; ok { + p.To.Val = v + } else { + self.pendings[to] = append(self.pendings[to], p) + } + + /* mark as a branch, and add to instruction buffer */ + p.To.Type = obj.TYPE_BRANCH + self.pb.Append(p) +} + +func (self *BaseAssembler) Rjmp(op string, to obj.Addr) { + p := self.pb.New() + p.To = to + p.As = As(op) + self.pb.Append(p) +} + +func (self *BaseAssembler) From(op string, val obj.Addr) { + p := self.pb.New() + p.As = As(op) + p.From = val + self.pb.Append(p) +} + +func (self *BaseAssembler) Emit(op string, args ...obj.Addr) { + p := self.pb.New() + p.As = As(op) + self.assignOperands(p, args) + self.pb.Append(p) +} + +func (self *BaseAssembler) assignOperands(p *obj.Prog, args []obj.Addr) { + switch len(args) { + case 0 : + case 1 : p.To = args[0] + case 2 : p.To, p.From = args[1], args[0] + case 3 : p.To, p.From, p.RestArgs = args[2], args[0], args[1:2] + case 4 : p.To, p.From, p.RestArgs = args[2], args[3], args[:2] + default : panic("invalid operands") + } +} + +/** Assembler Helpers **/ + +func (self *BaseAssembler) Size() int { + self.build() + return len(self.c) +} + +func (self *BaseAssembler) Init(f func()) { + self.i = 0 + self.f = f + self.c = nil + self.o = sync.Once{} +} + +var jitLoader = loader.Loader{ + Name: "sonic.jit.", + File: "github.com/bytedance/sonic/jit.go", + Options: loader.Options{ + NoPreempt: true, + }, +} + +func (self *BaseAssembler) Load(name string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) loader.Function { + self.build() + return jitLoader.LoadOne(self.c, name, frameSize, argSize, argStackmap, localStackmap) +} + +/** Assembler Stages **/ + +func (self *BaseAssembler) init() { + self.pb = newBackend("amd64") + self.xrefs = map[string][]*obj.Prog{} + self.labels = map[string]*obj.Prog{} + self.pendings = map[string][]*obj.Prog{} +} + +func (self *BaseAssembler) build() { + self.o.Do(func() { + self.init() + self.f() + self.validate() + self.assemble() + self.resolve() + self.release() + }) +} + +func (self *BaseAssembler) release() { + self.pb.Release() + self.pb = nil + self.xrefs = nil + self.labels = nil + self.pendings = nil +} + +func (self *BaseAssembler) resolve() { + for s, v := range self.xrefs { + for _, prog := range v { + if prog.As != x86.ALONG { + panic("invalid RIP relative reference") + } else if p, ok := self.labels[s]; !ok { + panic("links are not fully resolved: " + s) + } else { + off := prog.From.Offset + p.Pc - prog.Pc + binary.LittleEndian.PutUint32(self.c[prog.Pc:], uint32(off)) + } + } + } +} + +func (self *BaseAssembler) validate() { + for key := range self.pendings { + panic("links are not fully resolved: " + key) + } +} + +func (self *BaseAssembler) assemble() { + self.c = self.pb.Assemble() +} diff --git a/vendor/github.com/bytedance/sonic/internal/jit/backend.go b/vendor/github.com/bytedance/sonic/internal/jit/backend.go new file mode 100644 index 000000000..75e180415 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/jit/backend.go @@ -0,0 +1,120 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jit + +import ( + `fmt` + `sync` + _ `unsafe` + + `github.com/twitchyliquid64/golang-asm/asm/arch` + `github.com/twitchyliquid64/golang-asm/obj` + `github.com/twitchyliquid64/golang-asm/objabi` +) + +type Backend struct { + Ctxt *obj.Link + Arch *arch.Arch + Head *obj.Prog + Tail *obj.Prog + Prog []*obj.Prog +} + +var ( + _progPool sync.Pool +) + +//go:nosplit +//go:linkname throw runtime.throw +func throw(_ string) + +func newProg() *obj.Prog { + if val := _progPool.Get(); val == nil { + return new(obj.Prog) + } else { + return remProg(val.(*obj.Prog)) + } +} + +func remProg(p *obj.Prog) *obj.Prog { + *p = obj.Prog{} + return p +} + +func newBackend(name string) (ret *Backend) { + ret = new(Backend) + ret.Arch = arch.Set(name) + ret.Ctxt = newLinkContext(ret.Arch.LinkArch) + ret.Arch.Init(ret.Ctxt) + return +} + +func newLinkContext(arch *obj.LinkArch) (ret *obj.Link) { + ret = obj.Linknew(arch) + ret.Headtype = objabi.Hlinux + ret.DiagFunc = diagLinkContext + return +} + +func diagLinkContext(str string, args ...interface{}) { + throw(fmt.Sprintf(str, args...)) +} + +func (self *Backend) New() (ret *obj.Prog) { + ret = newProg() + ret.Ctxt = self.Ctxt + self.Prog = append(self.Prog, ret) + return +} + +func (self *Backend) Append(p *obj.Prog) { + if self.Head == nil { + self.Head = p + self.Tail = p + } else { + self.Tail.Link = p + self.Tail = p + } +} + +func (self *Backend) Release() { + self.Arch = nil + self.Ctxt = nil + + /* return all the progs into pool */ + for _, p := range self.Prog { + _progPool.Put(p) + } + + /* clear all the references */ + self.Head = nil + self.Tail = nil + self.Prog = nil +} + +func (self *Backend) Assemble() []byte { + var sym obj.LSym + var fnv obj.FuncInfo + + /* construct the function */ + sym.Func = &fnv + fnv.Text = self.Head + + /* call the assembler */ + self.Arch.Assemble(self.Ctxt, &sym, self.New) + return sym.P +} diff --git a/vendor/github.com/bytedance/sonic/internal/jit/runtime.go b/vendor/github.com/bytedance/sonic/internal/jit/runtime.go new file mode 100644 index 000000000..ec69d067a --- /dev/null +++ b/vendor/github.com/bytedance/sonic/internal/jit/runtime.go @@ -0,0 +1,54 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jit + +import ( + `reflect` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` + `github.com/twitchyliquid64/golang-asm/obj` +) + +//go:noescape +//go:linkname getitab runtime.getitab +//goland:noinspection ALL +func getitab(inter *rt.GoType, typ *rt.GoType, canfail bool) *rt.GoItab + +func Func(f interface{}) obj.Addr { + if p := rt.UnpackEface(f); p.Type.Kind() != reflect.Func { + panic("f is not a function") + } else { + return Imm(*(*int64)(p.Value)) + } +} + +func Type(t reflect.Type) obj.Addr { + return Gtype(rt.UnpackType(t)) +} + +func Itab(i *rt.GoType, t reflect.Type) obj.Addr { + return Imm(int64(uintptr(unsafe.Pointer(getitab(i, rt.UnpackType(t), false))))) +} + +func Gitab(i *rt.GoItab) obj.Addr { + return Imm(int64(uintptr(unsafe.Pointer(i)))) +} + +func Gtype(t *rt.GoType) obj.Addr { + return Imm(int64(uintptr(unsafe.Pointer(t)))) +} |