diff options
Diffstat (limited to 'vendor/github.com/bytedance/sonic/internal/decoder/optdec')
9 files changed, 170 insertions, 76 deletions
diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/compiler.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/compiler.go index 7d9d60a01..b3ca7a4ee 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/compiler.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/compiler.go @@ -1,7 +1,6 @@ package optdec import ( - "encoding/json" "fmt" "reflect" @@ -169,7 +168,9 @@ func (c *compiler) compileBasic(vt reflect.Type) decFunc { case reflect.Struct: return c.compileStruct(vt) default: - panic(&json.UnmarshalTypeError{Type: vt}) + return &unsupportedTypeDecoder{ + typ: rt.UnpackType(vt), + } } } @@ -263,7 +264,7 @@ func (c *compiler) compileSlice(vt reflect.Type) decFunc { if et.IsUint64() { return &sliceU64Decoder{} } - if et.Kind() == reflect.String { + if et.Kind() == reflect.String && et != rt.JsonNumberType { return &sliceStringDecoder{} } @@ -343,7 +344,7 @@ func (c *compiler) compileMap(vt reflect.Type) decFunc { // Some common integer map later mt := rt.MapType(rt.UnpackType(vt)) - if mt.Key.Kind() == reflect.String { + if mt.Key.Kind() == reflect.String && mt.Key != rt.JsonNumberType { return &mapStrKeyDecoder{ mapType: mt, assign: rt.GetMapStrAssign(vt), @@ -399,7 +400,7 @@ func tryCompileKeyUnmarshaler(vt reflect.Type) decKey { return decodeKeyTextUnmarshaler } - /* not support map key with `json.Unmarshaler` */ + /* NOTE: encoding/json not support map key with `json.Unmarshaler` */ return nil } @@ -413,8 +414,18 @@ func (c *compiler) compileMapKey(vt reflect.Type) decKey { return decodeKeyU8 case reflect.Uint16: return decodeKeyU16 + // NOTE: actually, encoding/json can't use float as map key + case reflect.Float32: + return decodeFloat32Key + case reflect.Float64: + return decodeFloat64Key + case reflect.String: + if rt.UnpackType(vt.Key()) == rt.JsonNumberType { + return decodeJsonNumberKey + } + fallthrough default: - panic(&json.UnmarshalTypeError{Type: vt}) + return nil } } diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/errors.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/errors.go index db0af547b..daa75c59c 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/errors.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/errors.go @@ -70,4 +70,10 @@ Msg: msg, } } + + func error_unsuppoted(typ *rt.GoType) error { + return &json.UnsupportedTypeError{ + Type: typ.Pack(), + } +}
\ No newline at end of file diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/functor.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/functor.go index 2a0523d5e..48a8953c1 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/functor.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/functor.go @@ -279,3 +279,16 @@ func (d *recuriveDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) er } return dec.FromDom(vp, node, ctx) } + +type unsupportedTypeDecoder struct { + typ *rt.GoType +} + + +func (d *unsupportedTypeDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error { + if node.IsNull() { + return nil + } + return error_unsuppoted(d.typ) +} + diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/interface.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/interface.go index 0c063d55f..0285f0f8f 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/interface.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/interface.go @@ -13,32 +13,44 @@ type efaceDecoder struct { } func (d *efaceDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error { - if node.IsNull() { - *(*interface{})(vp) = interface{}(nil) - return nil - } - - eface := *(*rt.GoEface)(vp) - - // not pointer type, or nil pointer, or *interface{} - if eface.Value == nil || eface.Type.Kind() != reflect.Ptr || rt.PtrElem(eface.Type) == anyType { + /* check the defined pointer type for issue 379 */ + eface := (*rt.GoEface)(vp) + + /* + not pointer type, or nil pointer, or self-pointed interface{}, such as + ```go + var v interface{} + v = &v + return v + ``` see `issue758_test.go`. + */ + if eface.Value == nil || eface.Type.Kind() != reflect.Ptr || eface.Value == vp { ret, err := node.AsEface(ctx) if err != nil { return err } - *(*interface{})(vp) = ret return nil } + if node.IsNull() { + if eface.Type.Indirect() || (!eface.Type.Indirect() && eface.Type.Pack().Elem().Kind() != reflect.Ptr) { + *(*interface{})(vp) = nil + return nil + } + } + etp := rt.PtrElem(eface.Type) vp = eface.Value - /* check the defined pointer type for issue 379 */ if eface.Type.IsNamed() { + // check named pointer type, avoid call its `Unmarshaler` newp := vp etp = eface.Type vp = unsafe.Pointer(&newp) + } else if !eface.Type.Indirect() { + // check direct value + etp = rt.UnpackType(eface.Type.Pack().Elem()) } dec, err := findOrCompile(etp) @@ -65,19 +77,10 @@ func (d *ifaceDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error } vt := iface.Itab.Vt - - // not pointer type, or nil pointer, or *interface{} - if vp == nil || vt.Kind() != reflect.Ptr || rt.PtrElem(vt) == anyType { - ret, err := node.AsEface(ctx) - if err != nil { - return err - } - - *(*interface{})(vp) = ret - return nil + if vt.Kind() != reflect.Ptr || iface.Value == nil { + return error_type(d.typ) } - etp := rt.PtrElem(vt) vp = iface.Value diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/map.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/map.go index 1a2bda8f3..958ebc4b0 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/map.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/map.go @@ -292,9 +292,9 @@ func (d *mapU64KeyDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) e /** Decoder for generic cases */ -type decKey func(dec *mapDecoder, raw string, ctx *context) (interface{}, error) +type decKey func(dec *mapDecoder, raw string) (interface{}, error) -func decodeKeyU8(dec *mapDecoder, raw string, ctx *context) (interface{}, error) { +func decodeKeyU8(dec *mapDecoder, raw string) (interface{}, error) { key, err := Unquote(raw) if err != nil { return nil, err @@ -309,7 +309,7 @@ func decodeKeyU8(dec *mapDecoder, raw string, ctx *context) (interface{}, error) return uint8(ret), nil } -func decodeKeyU16(dec *mapDecoder, raw string, ctx *context) (interface{}, error) { +func decodeKeyU16(dec *mapDecoder, raw string) (interface{}, error) { key, err := Unquote(raw) if err != nil { return nil, err @@ -324,7 +324,7 @@ func decodeKeyU16(dec *mapDecoder, raw string, ctx *context) (interface{}, error return uint16(ret), nil } -func decodeKeyI8(dec *mapDecoder, raw string, ctx *context) (interface{}, error) { +func decodeKeyI8(dec *mapDecoder, raw string) (interface{}, error) { key, err := Unquote(raw) if err != nil { return nil, err @@ -339,7 +339,7 @@ func decodeKeyI8(dec *mapDecoder, raw string, ctx *context) (interface{}, error) return int8(ret), nil } -func decodeKeyI16(dec *mapDecoder, raw string, ctx *context) (interface{}, error) { +func decodeKeyI16(dec *mapDecoder, raw string) (interface{}, error) { key, err := Unquote(raw) if err != nil { return nil, err @@ -354,26 +354,53 @@ func decodeKeyI16(dec *mapDecoder, raw string, ctx *context) (interface{}, error return int16(ret), nil } -func decodeKeyJSONUnmarshaler(dec *mapDecoder, raw string, _ *context) (interface{}, error) { +func decodeKeyTextUnmarshaler(dec *mapDecoder, raw string) (interface{}, error) { + key, err := Unquote(raw) + if err != nil { + return nil, err + } ret := reflect.New(dec.mapType.Key.Pack()).Interface() - err := ret.(json.Unmarshaler).UnmarshalJSON([]byte(raw)) + err = ret.(encoding.TextUnmarshaler).UnmarshalText(rt.Str2Mem(key)) if err != nil { return nil, err } return ret, nil } -func decodeKeyTextUnmarshaler(dec *mapDecoder, raw string, ctx *context) (interface{}, error) { +func decodeFloat32Key(dec *mapDecoder, raw string) (interface{}, error) { key, err := Unquote(raw) if err != nil { return nil, err } - ret := reflect.New(dec.mapType.Key.Pack()).Interface() - err = ret.(encoding.TextUnmarshaler).UnmarshalText([]byte(key)) + ret, err := ParseF64(key) if err != nil { return nil, err } - return ret, nil + if ret > math.MaxFloat32 || ret < -math.MaxFloat32 { + return nil, error_value(key, dec.mapType.Key.Pack()) + } + return float32(ret), nil +} + +func decodeFloat64Key(dec *mapDecoder, raw string) (interface{}, error) { + key, err := Unquote(raw) + if err != nil { + return nil, err + } + return ParseF64(key) +} + +func decodeJsonNumberKey(dec *mapDecoder, raw string) (interface{}, error) { + // skip the quote + raw = raw[1:len(raw)-1] + end, ok := SkipNumberFast(raw, 0) + + // check trailing chars + if !ok || end != len(raw) { + return nil, error_value(raw, rt.JsonNumberType.Pack()) + } + + return json.Number(raw[0:end]), nil } type mapDecoder struct { @@ -389,8 +416,8 @@ func (d *mapDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error { } obj, ok := node.AsObj() - if !ok { - return error_mismatch(node, ctx, d.mapType.Pack()) + if !ok || d.keyDec == nil { + return error_mismatch(node, ctx, d.mapType.Pack()) } // allocate map @@ -404,7 +431,8 @@ func (d *mapDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error { for i := 0; i < obj.Len(); i++ { keyn := NewNode(next) raw := keyn.AsRaw(ctx) - key, err := d.keyDec(d, raw, ctx) + + key, err := d.keyDec(d, raw) if err != nil { if gerr == nil { gerr = error_mismatch(keyn, ctx, d.mapType.Pack()) diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/native.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/native.go index 5dadec0b7..60f407d2e 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/native.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/native.go @@ -74,6 +74,7 @@ type nodeBuf struct { depth uint64 nstart uintptr nend uintptr + iskey bool stat jsonStat } @@ -196,14 +197,14 @@ func (p *Parser) parse() ErrorCode { // check OoB here offset := p.nbuf.ncur - p.nbuf.nstart - curLen := offset / unsafe.Sizeof(node{}) - if curLen != uintptr(len(p.nodes)) { + curLen := int(offset / unsafe.Sizeof(node{})) + if curLen != len(p.nodes) { panic(fmt.Sprintf("current len: %d, real len: %d cap: %d", curLen, len(p.nodes), cap(p.nodes))) } // node buf is not enough, continue parse // the maxCap is always meet all valid JSON - maxCap := calMaxNodeCap(len(p.Json)) + maxCap := curLen + calMaxNodeCap(len(p.Json) - int(p.cur - p.start)) slice := rt.GoSlice{ Ptr: rt.Mallocgc(uintptr(maxCap) * nodeType.Size, nodeType, false), Len: maxCap, diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/node.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/node.go index b23901e38..774b6eef7 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/node.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/node.go @@ -301,6 +301,17 @@ func (self Node) AsI64(ctx *Context) (int64, bool) { } } +func (self Node) AsByte(ctx *Context) (uint8, bool) { + typ := self.Type() + if typ == KUint && self.U64() <= math.MaxUint8 { + return uint8(self.U64()), true + } else if typ == KSint && self.I64() == 0 { + return 0, true + } else { + return 0, false + } +} + /********* Parse Node String into Value ***************/ func (val Node) ParseI64(ctx *Context) (int64, bool) { @@ -457,20 +468,6 @@ func (val Node) AsStrRef(ctx *Context) (string, bool) { } } -func (val Node) AsBytesRef(ctx *Context) ([]byte, bool) { - switch val.Type() { - case KStringEscaped: - node := ptrCast(val.cptr) - offset := val.Position() - len := int(node.val) - return ctx.Parser.JsonBytes()[offset : offset + len], true - case KStringCommon: - return rt.Str2Mem(val.StringRef(ctx)), true - default: - return nil, false - } -} - func (val Node) AsStringText(ctx *Context) ([]byte, bool) { if !val.IsStr() { return nil, false @@ -551,7 +548,7 @@ func (val Node) AsRaw(ctx *Context) string { node := ptrCast(val.cptr) len := int(node.val) offset := val.Position() - // add start abd end quote + // add start and end quote ref := rt.Str2Mem(ctx.Parser.Json)[offset-1 : offset+len+1] return rt.Mem2Str(ref) case KRawNumber: fallthrough @@ -867,15 +864,38 @@ func (node *Node) AsSliceString(ctx *Context, vp unsafe.Pointer) error { return gerr } -func (node *Node) AsSliceBytes(ctx *Context) ([]byte, error) { - b, ok := node.AsBytesRef(ctx) - if !ok { - return nil, newUnmatched(node.Position(), rt.BytesType) +func (val *Node) AsSliceBytes(ctx *Context) ([]byte, error) { + var origin []byte + switch val.Type() { + case KStringEscaped: + node := ptrCast(val.cptr) + offset := val.Position() + len := int(node.val) + origin = ctx.Parser.JsonBytes()[offset : offset + len] + case KStringCommon: + origin = rt.Str2Mem(val.StringRef(ctx)) + case KArray: + arr := val.Array() + size := arr.Len() + a := make([]byte, size) + elem := NewNode(arr.Children()) + var gerr error + var ok bool + for i := 0; i < size; i++ { + a[i], ok = elem.AsByte(ctx) + if !ok && gerr == nil { + gerr = newUnmatched(val.Position(), rt.BytesType) + } + elem = NewNode(PtrOffset(elem.cptr, 1)) + } + return a, gerr + default: + return nil, newUnmatched(val.Position(), rt.BytesType) } - - b64, err := rt.DecodeBase64(b) + + b64, err := rt.DecodeBase64(origin) if err != nil { - return nil, newUnmatched(node.Position(), rt.BytesType) + return nil, newUnmatched(val.Position(), rt.BytesType) } return b64, nil } diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/slice.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/slice.go index a94e422b3..da7b12fc5 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/slice.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/slice.go @@ -80,9 +80,13 @@ func (d *arrayDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) error } /* zero rest of array */ - ptr := unsafe.Pointer(uintptr(vp) + uintptr(i)*d.elemType.Size) + addr := uintptr(vp) + uintptr(i)*d.elemType.Size n := uintptr(d.len-i) * d.elemType.Size - rt.ClearMemory(d.elemType, ptr, n) + + /* the boundary pointer may points to another unknown object, so we need to avoid using it */ + if n != 0 { + rt.ClearMemory(d.elemType, unsafe.Pointer(addr), n) + } return gerr } @@ -95,7 +99,18 @@ func (d *sliceEfaceDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) return nil } - return node.AsSliceEface(ctx, vp) + /* if slice is empty, just call `AsSliceEface` */ + if ((*rt.GoSlice)(vp)).Len == 0 { + return node.AsSliceEface(ctx, vp) + } + + decoder := sliceDecoder{ + elemType: rt.AnyType, + elemDec: &efaceDecoder{}, + typ: rt.SliceEfaceType.Pack(), + } + + return decoder.FromDom(vp, node, ctx) } type sliceI32Decoder struct { @@ -168,12 +183,8 @@ func (d *sliceBytesDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) } s, err := node.AsSliceBytes(ctx) - if err != nil { - return err - } - *(*[]byte)(vp) = s - return nil + return err } type sliceBytesUnmarshalerDecoder struct { diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/structs.go b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/structs.go index bce2758f1..8b148eadf 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/optdec/structs.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/optdec/structs.go @@ -4,6 +4,7 @@ import ( "reflect" "unsafe" + "github.com/bytedance/sonic/internal/decoder/consts" caching "github.com/bytedance/sonic/internal/optcaching" "github.com/bytedance/sonic/internal/resolver" ) @@ -38,7 +39,7 @@ func (d *structDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) erro next = val.Next() // find field idx - idx := d.fieldMap.Get(key) + idx := d.fieldMap.Get(key, ctx.Options()&uint64(consts.OptionCaseSensitive) != 0) if idx == -1 { if Options(ctx.Options())&OptionDisableUnknown != 0 { return error_field(key) |