diff options
Diffstat (limited to 'vendor/github.com/bytedance')
47 files changed, 804 insertions, 389 deletions
diff --git a/vendor/github.com/bytedance/sonic/README.md b/vendor/github.com/bytedance/sonic/README.md index 3486b9c47..a74fc4968 100644 --- a/vendor/github.com/bytedance/sonic/README.md +++ b/vendor/github.com/bytedance/sonic/README.md @@ -5,18 +5,27 @@ English | [中文](README_ZH_CN.md) A blazingly fast JSON serializing & deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data). ## Requirement -- Go 1.16~1.21 + +- Go 1.16~1.22 - Linux / MacOS / Windows(need go1.17 above) - Amd64 ARCH ## Features + - Runtime object binding without code generation - Complete APIs for JSON value manipulation - Fast, fast, fast! +## APIs + +see [go.dev](https://pkg.go.dev/github.com/bytedance/sonic) + ## Benchmarks + For **all sizes** of json and **all scenarios** of usage, **Sonic performs best**. + - [Medium](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13KB, 300+ key, 6 layers) + ```powershell goversion: 1.17.1 goos: darwin @@ -81,6 +90,7 @@ BenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.6 BenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op ``` + - [Small](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 keys, 3 layers)  - [Large](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635KB, 10000+ key, 6 layers) @@ -89,6 +99,7 @@ BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.7 See [bench.sh](https://github.com/bytedance/sonic/blob/main/scripts/bench.sh) for benchmark codes. ## How it works + See [INTRODUCTION.md](./docs/INTRODUCTION.md). ## Usage @@ -96,6 +107,7 @@ See [INTRODUCTION.md](./docs/INTRODUCTION.md). ### Marshal/Unmarshal Default behaviors are mostly consistent with `encoding/json`, except HTML escaping form (see [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) and `SortKeys` feature (optional support see [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)) that is **NOT** in conformity to [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). + ```go import "github.com/bytedance/sonic" @@ -107,8 +119,11 @@ err := sonic.Unmarshal(output, &data) ``` ### Streaming IO -Sonic supports decoding json from `io.Reader` or encoding objects into `io.`Writer`, aims at handling multiple values as well as reducing memory consumption. + +Sonic supports decoding json from `io.Reader` or encoding objects into `io.Writer`, aims at handling multiple values as well as reducing memory consumption. + - encoder + ```go var o1 = map[string]interface{}{ "a": "b", @@ -123,7 +138,9 @@ fmt.Println(w.String()) // {"a":"b"} // 1 ``` + - decoder + ```go var o = map[string]interface{}{} var r = strings.NewReader(`{"a":"b"}{"1":"2"}`) @@ -136,6 +153,7 @@ fmt.Printf("%+v", o) ``` ### Use Number/Use Int64 + ```go import "github.com/bytedance/sonic/decoder" @@ -164,7 +182,9 @@ fm := root.Interface().(float64) // jn == jm ``` ### Sort Keys + On account of the performance loss from sorting (roughly 10%), sonic doesn't enable this feature by default. If your component depends on it to work (like [zstd](https://github.com/facebook/zstd)), Use it like this: + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/encoder" @@ -177,19 +197,26 @@ v, err := encoder.Encode(m, encoder.SortMapKeys) var root := sonic.Get(JSON) err := root.SortKeys() ``` + ### Escape HTML + On account of the performance loss (roughly 15%), sonic doesn't enable this feature by default. You can use `encoder.EscapeHTML` option to open this feature (align with `encoding/json.HTMLEscape`). + ```go import "github.com/bytedance/sonic" v := map[string]string{"&&":"<>"} ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e"}}` ``` + ### Compact Format + Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. ### Print Error + If there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -215,7 +242,9 @@ if err != nil { ``` #### Mismatched Types [Sonic v1.6.0] + If there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON. + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -228,10 +257,15 @@ err := UnmarshalString(`{"A":"1","B":1}`, &data) println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n" fmt.Printf("%+v", data) // {A:0 B:1} ``` + ### Ast.Node + Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data. + #### Get/Index + Search partial JSON by given paths, which must be non-negative integer or string, or nil + ```go import "github.com/bytedance/sonic" @@ -245,10 +279,13 @@ raw := root.Raw() // == string(input) root, err := sonic.Get(input, "key1", 1, "key2") sub := root.Get("key3").Index(2).Int64() // == 3 ``` + **Tip**: since `Index()` uses offset to locate data, which is much faster than scanning like `Get()`, we suggest you use it as much as possible. And sonic also provides another API `IndexOrGet()` to underlying use offset as well as ensure the key is matched. #### Set/Unset + Modify the json content by Set()/Unset() + ```go import "github.com/bytedance/sonic" @@ -265,7 +302,9 @@ println(root.Get("key4").Check()) // "value not exist" ``` #### Serialize + To encode `ast.Node` as json, use `MarshalJson()` or `json.Marshal()` (MUST pass the node's pointer) + ```go import ( "encoding/json" @@ -279,6 +318,7 @@ println(string(buf) == string(exp)) // true ``` #### APIs + - validation: `Check()`, `Error()`, `Valid()`, `Exist()` - searching: `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()` - go-type casting: `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()` @@ -287,7 +327,9 @@ println(string(buf) == string(exp)) // true - modification: `Set()`, `SetByIndex()`, `Add()` ### Ast.Visitor + Sonic provides an advanced API for fully parsing JSON into non-standard types (neither `struct` not `map[string]interface{}`) without using any intermediate representation (`ast.Node` or `interface{}`). For example, you might have the following types which are like `interface{}` but actually not `interface{}`: + ```go type UserNode interface {} @@ -302,7 +344,9 @@ type ( UserArray struct{ Value []UserNode } ) ``` + Sonic provides the following API to return **the preorder traversal of a JSON AST**. The `ast.Visitor` is a SAX style interface which is used in some C++ JSON library. You should implement `ast.Visitor` by yourself and pass it to `ast.Preorder()` method. In your visitor you can make your custom types to represent JSON values. There may be an O(n) space container (such as stack) in your visitor to record the object / array hierarchy. + ```go func Preorder(str string, visitor Visitor, opts *VisitorOptions) error @@ -323,12 +367,14 @@ type Visitor interface { See [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) for detailed usage. We also implement a demo visitor for `UserNode` in [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go). ## Compatibility + Sonic **DOES NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. For developers who use sonic to build their applications in different environments, we have the following suggestions: - Developing on **Mac M1**: Make sure you have Rosetta 2 installed on your machine, and set `GOARCH=amd64` when building your application. Rosetta 2 can automatically translate x86 binaries to arm64 binaries and run x86 applications on Mac M1. - Developing on **Linux arm64**: You can install qemu and use the `qemu-x86_64 -cpu max` command to convert x86 binaries to amr64 binaries for applications built with sonic. The qemu can achieve a similar transfer effect to Rosetta 2 on Mac M1. For developers who want to use sonic on Linux arm64 without qemu, or those who want to handle JSON strictly consistent with `encoding/json`, we provide some compatible APIs as `sonic.API` + - `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options like `SortKeys=false` will be invalid. - `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) to run on sonic-supporting environment. It will fall back to `encoding/json`. - `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options will be invalid. @@ -336,7 +382,9 @@ For developers who want to use sonic on Linux arm64 without qemu, or those who w ## Tips ### Pretouch + Since Sonic uses [golang-asm](https://github.com/twitchyliquid64/golang-asm) as a JIT assembler, which is NOT very suitable for runtime compiling, first-hit running of a huge schema may cause request-timeout or even process-OOM. For better stability, we advise **using `Pretouch()` for huge-schema or compact-memory applications** before `Marshal()/Unmarshal()`. + ```go import ( "reflect" @@ -362,17 +410,23 @@ func init() { ``` ### Copy string -When decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. We provide the option `decoder.CopyString()` for users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree. + +When decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. - `Config.CopyString`/`decoder.CopyString()`: We provide the option for `Decode()` / `Unmarshal()` users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree. + +- `GetFromStringNoCopy()`: For memory safty, `sonic.Get()` / `sonic.GetFromString()` now copies return JSON. If users want to get json more quickly and not care about memory usage, you can use `GetFromStringNoCopy()` to return a JSON direclty referenced from source. ### Pass string or []byte? + For alignment to `encoding/json`, we provide API to pass `[]byte` as an argument, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when the origin JSON is huge. Therefore, you can use `UnmarshalString()` and `GetFromString()` to pass a string, as long as your origin data is a string or **nocopy-cast** is safe for your []byte. We also provide API `MarshalString()` for convenient **nocopy-cast** of encoded JSON []byte, which is safe since sonic's output bytes is always duplicated and unique. ### Accelerate `encoding.TextMarshaler` -To ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). +To ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). ### Better performance for generic data + In **fully-parsed** scenario, `Unmarshal()` performs better than `Get()`+`Node.Interface()`. But if you only have a part of the schema for specific json, you can combine `Get()` and `Unmarshal()` together: + ```go import "github.com/bytedance/sonic" @@ -380,7 +434,9 @@ node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user") var user User // your partial schema... err = sonic.UnmarshalString(node.Raw(), &user) ``` + Even if you don't have any schema, use `ast.Node` as the container of generic values instead of `map` or `interface`: + ```go import "github.com/bytedance/sonic" @@ -391,7 +447,9 @@ err = user.Check() // err = user.LoadAll() // only call this when you want to use 'user' concurrently... go someFunc(user) ``` + Why? Because `ast.Node` stores its children using `array`: + - `Array`'s performance is **much better** than `Map` when Inserting (Deserialize) and Scanning (Serialize) data; - **Hashing** (`map[x]`) is not as efficient as **Indexing** (`array[x]`), which `ast.Node` can conduct on **both array and object**; - Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**. @@ -399,6 +457,7 @@ Why? Because `ast.Node` stores its children using `array`: **CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, you can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}` ### Ast.Node or Ast.Visitor? + For generic data, `ast.Node` should be enough for your needs in most cases. However, `ast.Node` is designed for partially processing JSON string. It has some special designs such as lazy-load which might not be suitable for directly parsing the whole JSON string like `Unmarshal()`. Although `ast.Node` is better then `map` or `interface{}`, it's also a kind of intermediate representation after all if your final types are customized and you have to convert the above types to your custom types after parsing. @@ -408,4 +467,5 @@ For better performance, in previous case the `ast.Visitor` will be the better ch But `ast.Visitor` is not a very handy API. You might need to write a lot of code to implement your visitor and carefully maintain the tree hierarchy during decoding. Please read the comments in [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) carefully if you decide to use this API. ## Community + Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem. diff --git a/vendor/github.com/bytedance/sonic/README_ZH_CN.md b/vendor/github.com/bytedance/sonic/README_ZH_CN.md index a18e99afe..d0341ab72 100644 --- a/vendor/github.com/bytedance/sonic/README_ZH_CN.md +++ b/vendor/github.com/bytedance/sonic/README_ZH_CN.md @@ -6,10 +6,14 @@ ## 依赖 -- Go 1.16~1.21 +- Go 1.16~1.22 - Linux / MacOS / Windows（需要 Go1.17 以上） - Amd64 架构 +## 接口 + +详见 [go.dev](https://pkg.go.dev/github.com/bytedance/sonic) + ## 特色 - 运行时对象绑定，无需代码生成 @@ -19,7 +23,9 @@ ## 基准测试 对于**所有大小**的 json 和**所有使用场景**， **Sonic 表现均为最佳**。 + - [中型](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13kB, 300+ 键, 6 层) + ```powershell goversion: 1.17.1 goos: darwin @@ -84,6 +90,7 @@ BenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.6 BenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op ``` + - [小型](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 个键, 3 层)  - [大型](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635kB, 10000+ 个键, 6 层) @@ -100,6 +107,7 @@ BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.7 ### 序列化/反序列化 默认的行为基本上与 `encoding/json` 相一致，除了 HTML 转义形式（参见 [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) 和 `SortKeys` 功能（参见 [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)）**没有**遵循 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 。 + ```go import "github.com/bytedance/sonic" @@ -113,7 +121,9 @@ err := sonic.Unmarshal(output, &data) ### 流式输入输出 Sonic 支持解码 `io.Reader` 中输入的 json，或将对象编码为 json 后输出至 `io.Writer`，以处理多个值并减少内存消耗。 + - 编码器 + ```go var o1 = map[string]interface{}{ "a": "b", @@ -128,7 +138,9 @@ fmt.Println(w.String()) // {"a":"b"} // 1 ``` + - 解码器 + ```go var o = map[string]interface{}{} var r = strings.NewReader(`{"a":"b"}{"1":"2"}`) @@ -172,6 +184,7 @@ fm := root.Interface().(float64) // jn == jm ### 对键排序 考虑到排序带来的性能损失（约 10% ）， sonic 默认不会启用这个功能。如果你的组件依赖这个行为（如 [zstd](https://github.com/facebook/zstd)) ，可以仿照下面的例子： + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/encoder" @@ -188,6 +201,7 @@ err := root.SortKeys() ### HTML 转义 考虑到性能损失（约15%）， sonic 默认不会启用这个功能。你可以使用 `encoder.EscapeHTML` 选项来开启（与 `encoding/json.HTMLEscape` 行为一致）。 + ```go import "github.com/bytedance/sonic" @@ -196,11 +210,13 @@ ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e" ``` ### 紧凑格式 + Sonic 默认将基本类型（ `struct` ， `map` 等）编码为紧凑格式的 JSON ，除非使用 `json.RawMessage` or `json.Marshaler` 进行编码： sonic 确保输出的 JSON 合法，但出于性能考虑，**不会**加工成紧凑格式。我们提供选项 `encoder.CompactMarshaler` 来添加此过程， ### 打印错误 如果输入的 JSON 存在无效的语法，sonic 将返回 `decoder.SyntaxError`，该错误支持错误位置的美化输出。 + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -228,6 +244,7 @@ if err != nil { #### 类型不匹配 [Sonic v1.6.0] 如果给定键中存在**类型不匹配**的值， sonic 会抛出 `decoder.MismatchTypeError` （如果有多个，只会报告最后一个），但仍会跳过错误的值并解码下一个 JSON 。 + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -240,6 +257,7 @@ err := UnmarshalString(`{"A":"1","B":1}`, &data) println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n" fmt.Printf("%+v", data) // {A:0 B:1} ``` + ### `Ast.Node` Sonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列化和反序列化，并提供了获取和修改通用数据的鲁棒的 API。 @@ -247,6 +265,7 @@ Sonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列 #### 查找/索引 通过给定的路径搜索 JSON 片段，路径必须为非负整数，字符串或 `nil` 。 + ```go import "github.com/bytedance/sonic" @@ -260,11 +279,13 @@ raw := root.Raw() // == string(input) root, err := sonic.Get(input, "key1", 1, "key2") sub := root.Get("key3").Index(2).Int64() // == 3 ``` + **注意**：由于 `Index()` 使用偏移量来定位数据，比使用扫描的 `Get()` 要快的多，建议尽可能的使用 `Index` 。 Sonic 也提供了另一个 API， `IndexOrGet()` ，以偏移量为基础并且也确保键的匹配。 #### 修改 -使用 ` Set()` / `Unset()` 修改 json 的内容 +使用 `Set()` / `Unset()` 修改 json 的内容 + ```go import "github.com/bytedance/sonic" @@ -281,7 +302,9 @@ println(root.Get("key4").Check()) // "value not exist" ``` #### 序列化 + 要将 `ast.Node` 编码为 json ，使用 `MarshalJson()` 或者 `json.Marshal()` （必须传递指向节点的指针） + ```go import ( "encoding/json" @@ -295,6 +318,7 @@ println(string(buf) == string(exp)) // true ``` #### APIs + - 合法性检查： `Check()`, `Error()`, `Valid()`, `Exist()` - 索引： `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()` - 转换至 go 内置类型： `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()` @@ -303,7 +327,9 @@ println(string(buf) == string(exp)) // true - 修改： `Set()`, `SetByIndex()`, `Add()` ### `Ast.Visitor` + Sonic 提供了一个高级的 API 用于直接全量解析 JSON 到非标准容器里 (既不是 `struct` 也不是 `map[string]interface{}`) 且不需要借助任何中间表示 (`ast.Node` 或 `interface{}`)。举个例子，你可能定义了下述的类型，它们看起来像 `interface{}`，但实际上并不是： + ```go type UserNode interface {} @@ -318,7 +344,9 @@ type ( UserArray struct{ Value []UserNode } ) ``` + Sonic 提供了下述的 API 来返回 **“对 JSON AST 的前序遍历”**。`ast.Visitor` 是一个 SAX 风格的接口，这在某些 C++ 的 JSON 解析库中被使用到。你需要自己实现一个 `ast.Visitor`，将它传递给 `ast.Preorder()` 方法。在你的实现中你可以使用自定义的类型来表示 JSON 的值。在你的 `ast.Visitor` 中，可能需要有一个 O(n) 空间复杂度的容器（比如说栈）来记录 object / array 的层级。 + ```go func Preorder(str string, visitor Visitor, opts *VisitorOptions) error @@ -335,15 +363,18 @@ type Visitor interface { OnArrayEnd() error } ``` + 详细用法参看 [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go)，我们还为 `UserNode` 实现了一个示例 `ast.Visitor`，你可以在 [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go) 中找到它。 ## 兼容性 + 由于开发高性能代码的困难性， Sonic **不**保证对所有环境的支持。对于在不同环境中使用 Sonic 构建应用程序的开发者，我们有以下建议： - 在 **Mac M1** 上开发：确保在您的计算机上安装了 Rosetta 2，并在构建时设置 `GOARCH=amd64` 。 Rosetta 2 可以自动将 x86 二进制文件转换为 arm64 二进制文件，并在 Mac M1 上运行 x86 应用程序。 - 在 **Linux arm64** 上开发：您可以安装 qemu 并使用 `qemu-x86_64 -cpu max` 命令来将 x86 二进制文件转换为 arm64 二进制文件。qemu可以实现与Mac M1上的Rosetta 2类似的转换效果。 对于希望在不使用 qemu 下使用 sonic 的开发者，或者希望处理 JSON 时与 `encoding/JSON` 严格保持一致的开发者，我们在 `sonic.API` 中提供了一些兼容性 API + - `ConfigDefault`: 在支持 sonic 的环境下 sonic 的默认配置（`EscapeHTML=false`，`SortKeys=false`等）。行为与具有相应配置的 `encoding/json` 一致，一些选项，如 `SortKeys=false` 将无效。 - `ConfigStd`: 在支持 sonic 的环境下与标准库兼容的配置（`EscapeHTML=true`，`SortKeys=true`等）。行为与 `encoding/json` 一致。 - `ConfigFastest`: 在支持 sonic 的环境下运行最快的配置（`NoQuoteTextMarshaler=true`）。行为与具有相应配置的 `encoding/json` 一致，某些选项将无效。 @@ -351,7 +382,9 @@ type Visitor interface { ## 注意事项 ### 预热 + 由于 Sonic 使用 [golang-asm](https://github.com/twitchyliquid64/golang-asm) 作为 JIT 汇编器，这个库并不适用于运行时编译，第一次运行一个大型模式可能会导致请求超时甚至进程内存溢出。为了更好地稳定性，我们建议在运行大型模式或在内存有限的应用中，在使用 `Marshal()/Unmarshal()` 前运行 `Pretouch()`。 + ```go import ( "reflect" @@ -381,16 +414,17 @@ func init() { 当解码 **没有转义字符的字符串**时， sonic 会从原始的 JSON 缓冲区内引用而不是复制到新的一个缓冲区中。这对 CPU 的性能方面很有帮助，但是可能因此在解码后对象仍在使用的时候将整个 JSON 缓冲区保留在内存中。实践中我们发现，通过引用 JSON 缓冲区引入的额外内存通常是解码后对象的 20% 至 80% ，一旦应用长期保留这些对象（如缓存以备重用），服务器所使用的内存可能会增加。我们提供了选项 `decoder.CopyString()` 供用户选择，不引用 JSON 缓冲区。这可能在一定程度上降低 CPU 性能。 ### 传递字符串还是字节数组？ + 为了和 `encoding/json` 保持一致，我们提供了传递 `[]byte` 作为参数的 API ，但考虑到安全性，字符串到字节的复制是同时进行的，这在原始 JSON 非常大时可能会导致性能损失。因此，你可以使用 `UnmarshalString()` 和 `GetFromString()` 来传递字符串，只要你的原始数据是字符串，或**零拷贝类型转换**对于你的字节数组是安全的。我们也提供了 `MarshalString()` 的 API ，以便对编码的 JSON 字节数组进行**零拷贝类型转换**，因为 sonic 输出的字节始终是重复并且唯一的，所以这样是安全的。 ### 加速 `encoding.TextMarshaler` 为了保证数据安全性， `sonic.Encoder` 默认会对来自 `encoding.TextMarshaler` 接口的字符串进行引用和转义，如果大部分数据都是这种形式那可能会导致很大的性能损失。我们提供了 `encoder.NoQuoteTextMarshaler` 选项来跳过这些操作，但你**必须**保证他们的输出字符串依照 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 进行了转义和引用。 - ### 泛型的性能优化 在 **完全解析**的场景下， `Unmarshal()` 表现得比 `Get()`+`Node.Interface()` 更好。但是如果你只有特定 JSON 的部分模式，你可以将 `Get()` 和 `Unmarshal()` 结合使用： + ```go import "github.com/bytedance/sonic" @@ -398,7 +432,9 @@ node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user") var user User // your partial schema... err = sonic.UnmarshalString(node.Raw(), &user) ``` + 甚至如果你没有任何模式，可以用 `ast.Node` 代替 `map` 或 `interface` 作为泛型的容器： + ```go import "github.com/bytedance/sonic" @@ -409,7 +445,9 @@ err = user.Check() // err = user.LoadAll() // only call this when you want to use 'user' concurrently... go someFunc(user) ``` + 为什么？因为 `ast.Node` 使用 `array` 来存储其子节点： + - 在插入（反序列化）和扫描（序列化）数据时，`Array` 的性能比 `Map` **好得多**； - **哈希**（`map[x]`）的效率不如**索引**（`array[x]`）高效，而 `ast.Node` 可以在数组和对象上使用索引； - 使用 `Interface()` / `Map()` 意味着 sonic 必须解析所有的底层值，而 `ast.Node` 可以**按需解析**它们。 @@ -417,6 +455,7 @@ go someFunc(user) **注意**：由于 `ast.Node` 的惰性加载设计，其**不能**直接保证并发安全性，但你可以调用 `Node.Load()` / `Node.LoadAll()` 来实现并发安全。尽管可能会带来性能损失，但仍比转换成 `map` 或 `interface{}` 更为高效。 ### 使用 `ast.Node` 还是 `ast.Visitor`？ + 对于泛型数据的解析，`ast.Node` 在大多数场景上应该能够满足你的需求。 然而，`ast.Node` 是一种针对部分解析 JSON 而设计的泛型容器，它包含一些特殊设计，比如惰性加载，如果你希望像 `Unmarshal()` 那样直接解析整个 JSON，这些设计可能并不合适。尽管 `ast.Node` 相较于 `map` 或 `interface{}` 来说是更好的一种泛型容器，但它毕竟也是一种中间表示，如果你的最终类型是自定义的，你还得在解析完成后将上述类型转化成你自定义的类型。 diff --git a/vendor/github.com/bytedance/sonic/api.go b/vendor/github.com/bytedance/sonic/api.go index 9525b7afd..54d9a2160 100644 --- a/vendor/github.com/bytedance/sonic/api.go +++ b/vendor/github.com/bytedance/sonic/api.go @@ -20,6 +20,7 @@ import ( `io` `github.com/bytedance/sonic/ast` + `github.com/bytedance/sonic/internal/rt` ) // Config is a combination of sonic/encoder.Options and sonic/decoder.Options @@ -73,6 +74,9 @@ type Config struct { // NoValidateJSONMarshaler indicates that the encoder should not validate the output string // after encoding the JSONMarshaler to JSON. NoValidateJSONMarshaler bool + + // NoEncoderNewline indicates that the encoder should not add a newline after every message + NoEncoderNewline bool } var ( @@ -170,27 +174,41 @@ func UnmarshalString(buf string, val interface{}) error { return ConfigDefault.UnmarshalFromString(buf, val) } -// Get searches the given path from json, -// and returns its representing ast.Node. +// Get searches and locates the given path from src json, +// and returns a ast.Node representing the partially json. // // Each path arg must be integer or string: // - Integer is target index(>=0), means searching current node as array. // - String is target key, means searching current node as object. // // -// Note, the api expects the json is well-formed at least, -// otherwise it may return unexpected result. +// Notice: It expects the src json is **Well-formed** and **Immutable** when calling, +// otherwise it may return unexpected result. +// Considering memory safty, the returned JSON is **Copied** from the input func Get(src []byte, path ...interface{}) (ast.Node, error) { - return GetFromString(string(src), path...) + return GetCopyFromString(rt.Mem2Str(src), path...) } -// GetFromString is same with Get except src is string, -// which can reduce unnecessary memory copy. +// GetFromString is same with Get except src is string. +// +// WARNING: The returned JSON is **Referenced** from the input. +// Caching or long-time holding the returned node may cause OOM. +// If your src is big, consider use GetFromStringCopy(). func GetFromString(src string, path ...interface{}) (ast.Node, error) { return ast.NewSearcher(src).GetByPath(path...) } +// GetCopyFromString is same with Get except src is string +func GetCopyFromString(src string, path ...interface{}) (ast.Node, error) { + return ast.NewSearcher(src).GetByPathCopy(path...) +} + // Valid reports whether data is a valid JSON encoding. func Valid(data []byte) bool { return ConfigDefault.Valid(data) } + +// Valid reports whether data is a valid JSON encoding. +func ValidString(data string) bool { + return ConfigDefault.Valid(rt.Str2Mem(data)) +} diff --git a/vendor/github.com/bytedance/sonic/ast/api_amd64.go b/vendor/github.com/bytedance/sonic/ast/api_amd64.go index da6738efd..fa4375630 100644 --- a/vendor/github.com/bytedance/sonic/ast/api_amd64.go +++ b/vendor/github.com/bytedance/sonic/ast/api_amd64.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16,!go1.22 +// +build amd64,go1.16,!go1.23 /* * Copyright 2022 ByteDance Inc. @@ -131,27 +131,3 @@ func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { } return start, 0 } - -func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { - var err types.ParsingError - var start int - - self.parser.p = 0 - start, err = self.parser.getByPath(path...) - if err != 0 { - // for compatibility with old version - if err == types.ERR_NOT_FOUND { - return Node{}, ErrNotExist - } - if err == types.ERR_UNSUPPORT_TYPE { - panic("path must be either int(>=0) or string") - } - return Node{}, self.parser.syntaxError(err) - } - - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) - } - return newRawNode(self.parser.s[start:self.parser.p], t), nil -} diff --git a/vendor/github.com/bytedance/sonic/ast/api_compat.go b/vendor/github.com/bytedance/sonic/ast/api_compat.go index 7b475eb61..9244e76e1 100644 --- a/vendor/github.com/bytedance/sonic/ast/api_compat.go +++ b/vendor/github.com/bytedance/sonic/ast/api_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64 !go1.16 go1.23 /* * Copyright 2022 ByteDance Inc. @@ -21,14 +21,13 @@ package ast import ( `encoding/base64` `encoding/json` - `fmt` `github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/rt` ) func init() { - println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable") } func quote(buf *[]byte, val string) { @@ -88,37 +87,25 @@ func (self *Node) encodeInterface(buf *[]byte) error { return nil } -func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { - self.parser.p = 0 - +func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { var err types.ParsingError for _, p := range path { if idx, ok := p.(int); ok && idx >= 0 { - if err = self.parser.searchIndex(idx); err != 0 { - return Node{}, self.parser.ExportError(err) + if err = self.searchIndex(idx); err != 0 { + return -1, err } } else if key, ok := p.(string); ok { - if err = self.parser.searchKey(key); err != 0 { - return Node{}, self.parser.ExportError(err) + if err = self.searchKey(key); err != 0 { + return -1, err } } else { panic("path must be either int(>=0) or string") } } - var start = self.parser.p - if start, err = self.parser.skip(); err != 0 { - return Node{}, self.parser.ExportError(err) - } - ns := len(self.parser.s) - if self.parser.p > ns || start >= ns || start>=self.parser.p { - return Node{}, fmt.Errorf("skip %d char out of json boundary", start) - } - - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) + var start int + if start, err = self.skip(); err != 0 { + return -1, err } - - return newRawNode(self.parser.s[start:self.parser.p], t), nil -}
\ No newline at end of file + return start, 0 +} diff --git a/vendor/github.com/bytedance/sonic/ast/buffer.go b/vendor/github.com/bytedance/sonic/ast/buffer.go index 93f4ff47a..bccbf4814 100644 --- a/vendor/github.com/bytedance/sonic/ast/buffer.go +++ b/vendor/github.com/bytedance/sonic/ast/buffer.go @@ -58,29 +58,89 @@ func (self *linkedNodes) At(i int) (*Node) { return nil } -func (self *linkedNodes) Add(v Node) { - if self.size < _DEFAULT_NODE_CAP { - self.head[self.size] = v - self.size++ +func (self *linkedNodes) MoveOne(source int, target int) { + if source == target { return } + if source < 0 || source >= self.size || target < 0 || target >= self.size { + return + } + // reserve source + n := *self.At(source) + if source < target { + // move every element (source,target] one step back + for i:=source; i<target; i++ { + *self.At(i) = *self.At(i+1) + } + } else { + // move every element [target,source) one step forward + for i:=source; i>target; i-- { + *self.At(i) = *self.At(i-1) + } + } + // set target + *self.At(target) = n +} + +func (self *linkedNodes) Pop() { + if self == nil || self.size == 0 { + return + } + self.Set(self.size-1, Node{}) + self.size-- +} + +func (self *linkedPairs) Pop() { + if self == nil || self.size == 0 { + return + } + self.Set(self.size-1, Pair{}) + self.size-- +} + +func (self *linkedNodes) Push(v Node) { + self.Set(self.size, v) +} - a, b, c := self.size/_DEFAULT_NODE_CAP-1 , self.size%_DEFAULT_NODE_CAP, cap(self.tail) - if a - c >= 0 { +func (self *linkedNodes) Set(i int, v Node) { + if i < _DEFAULT_NODE_CAP { + self.head[i] = v + if self.size <= i { + self.size = i+1 + } + return + } + a, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP + if a < 0 { + self.head[b] = v + } else { + self.growTailLength(a+1) + var n = &self.tail[a] + if *n == nil { + *n = new(nodeChunk) + } + (*n)[b] = v + } + if self.size <= i { + self.size = i+1 + } +} + +func (self *linkedNodes) growTailLength(l int) { + if l <= len(self.tail) { + return + } + c := cap(self.tail) + for c < l { c += 1 + c>>_APPEND_GROW_SHIFT - tmp := make([]*nodeChunk, a + 1, c) - copy(tmp, self.tail) - self.tail = tmp - } else if a >= len(self.tail) { - self.tail = self.tail[:a+1] } - - var n = &self.tail[a] - if *n == nil { - *n = new(nodeChunk) + if c == cap(self.tail) { + self.tail = self.tail[:l] + return } - (*n)[b] = v - self.size++ + tmp := make([]*nodeChunk, l, c) + copy(tmp, self.tail) + self.tail = tmp } func (self *linkedNodes) ToSlice(con []Node) { @@ -169,29 +229,49 @@ func (self *linkedPairs) At(i int) *Pair { return nil } -func (self *linkedPairs) Add(v Pair) { - if self.size < _DEFAULT_NODE_CAP { - self.head[self.size] = v - self.size++ +func (self *linkedPairs) Push(v Pair) { + self.Set(self.size, v) +} + +func (self *linkedPairs) Set(i int, v Pair) { + if i < _DEFAULT_NODE_CAP { + self.head[i] = v + if self.size <= i { + self.size = i+1 + } return } + a, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP + if a < 0 { + self.head[b] = v + } else { + self.growTailLength(a+1) + var n = &self.tail[a] + if *n == nil { + *n = new(pairChunk) + } + (*n)[b] = v + } + if self.size <= i { + self.size = i+1 + } +} - a, b, c := self.size/_DEFAULT_NODE_CAP-1 , self.size%_DEFAULT_NODE_CAP, cap(self.tail) - if a - c >= 0 { +func (self *linkedPairs) growTailLength(l int) { + if l <= len(self.tail) { + return + } + c := cap(self.tail) + for c < l { c += 1 + c>>_APPEND_GROW_SHIFT - tmp := make([]*pairChunk, a + 1, c) - copy(tmp, self.tail) - self.tail = tmp - } else if a >= len(self.tail) { - self.tail = self.tail[:a+1] } - - var n = &self.tail[a] - if *n == nil { - *n = new(pairChunk) + if c == cap(self.tail) { + self.tail = self.tail[:l] + return } - (*n)[b] = v - self.size++ + tmp := make([]*pairChunk, l, c) + copy(tmp, self.tail) + self.tail = tmp } // linear search @@ -271,7 +351,7 @@ func (self *linkedPairs) Swap(i, j int) { } func (self *linkedPairs) Sort() { - sort.Sort(self) + sort.Stable(self) } // Compare two strings from the pos d. diff --git a/vendor/github.com/bytedance/sonic/ast/encode.go b/vendor/github.com/bytedance/sonic/ast/encode.go index 263ae5a9d..956809c2c 100644 --- a/vendor/github.com/bytedance/sonic/ast/encode.go +++ b/vendor/github.com/bytedance/sonic/ast/encode.go @@ -193,20 +193,9 @@ func (self *Node) encodeArray(buf *[]byte) error { *buf = append(*buf, '[') - var s = (*linkedNodes)(self.p) var started bool - if nb > 0 { - n := s.At(0) - if n.Exists() { - if err := n.encode(buf); err != nil { - return err - } - started = true - } - } - - for i := 1; i < nb; i++ { - n := s.At(i) + for i := 0; i < nb; i++ { + n := self.nodeAt(i) if !n.Exists() { continue } @@ -250,21 +239,10 @@ func (self *Node) encodeObject(buf *[]byte) error { *buf = append(*buf, '{') - var s = (*linkedPairs)(self.p) var started bool - if nb > 0 { - n := s.At(0) - if n.Value.Exists() { - if err := n.encode(buf); err != nil { - return err - } - started = true - } - } - - for i := 1; i < nb; i++ { - n := s.At(i) - if !n.Value.Exists() { + for i := 0; i < nb; i++ { + n := self.pairAt(i) + if n == nil || !n.Value.Exists() { continue } if started { diff --git a/vendor/github.com/bytedance/sonic/ast/iterator.go b/vendor/github.com/bytedance/sonic/ast/iterator.go index 3c4187a9c..64e1e5a90 100644 --- a/vendor/github.com/bytedance/sonic/ast/iterator.go +++ b/vendor/github.com/bytedance/sonic/ast/iterator.go @@ -32,7 +32,11 @@ func (self *Node) Values() (ListIterator, error) { if err := self.should(types.V_ARRAY, "an array"); err != nil { return ListIterator{}, err } - return ListIterator{Iterator{p: self}}, nil + return self.values(), nil +} + +func (self *Node) values() ListIterator { + return ListIterator{Iterator{p: self}} } // Properties returns iterator for object's children traversal @@ -40,7 +44,11 @@ func (self *Node) Properties() (ObjectIterator, error) { if err := self.should(types.V_OBJECT, "an object"); err != nil { return ObjectIterator{}, err } - return ObjectIterator{Iterator{p: self}}, nil + return self.properties(), nil +} + +func (self *Node) properties() ObjectIterator { + return ObjectIterator{Iterator{p: self}} } type Iterator struct { @@ -114,7 +122,7 @@ next_start: } else { n := self.p.pairAt(self.i) self.i++ - if !n.Value.Exists() { + if n == nil || !n.Value.Exists() { goto next_start } return n diff --git a/vendor/github.com/bytedance/sonic/ast/node.go b/vendor/github.com/bytedance/sonic/ast/node.go index 990908ddd..9637659b1 100644 --- a/vendor/github.com/bytedance/sonic/ast/node.go +++ b/vendor/github.com/bytedance/sonic/ast/node.go @@ -491,7 +491,6 @@ func (self *Node) StrictFloat64() (float64, error) { // Len returns children count of a array|object|string node // WARN: For partially loaded node, it also works but only counts the parsed children -// WARN: For ARRAY|OBJECT nodes which has been conducted `UnsetXX()`, its length WON'T change func (self *Node) Len() (int, error) { if err := self.checkRaw(); err != nil { return 0, err @@ -534,10 +533,12 @@ func (self *Node) Set(key string, node Node) (bool, error) { if err := node.Check(); err != nil { return false, err } - + if self.t == _V_NONE || self.t == types.V_NULL { *self = NewObject([]Pair{{key, node}}) return false, nil + } else if self.itype() != types.V_OBJECT { + return false, ErrUnsupportType } p := self.Get(key) @@ -548,7 +549,7 @@ func (self *Node) Set(key string, node Node) (bool, error) { *self = newObject(new(linkedPairs)) } s := (*linkedPairs)(self.p) - s.Add(Pair{key, node}) + s.Push(Pair{key, node}) self.l++ return false, nil @@ -565,20 +566,22 @@ func (self *Node) SetAny(key string, val interface{}) (bool, error) { return self.Set(key, NewAny(val)) } -// Unset RESET the node of given key under object parent, and reports if the key has existed. -// WARN: After conducting `UnsetXX()`, the node's length WON'T change +// Unset REMOVE (soft) the node of given key under object parent, and reports if the key has existed. func (self *Node) Unset(key string) (bool, error) { if err := self.should(types.V_OBJECT, "an object"); err != nil { return false, err } + // NOTICE: must get acurate length before deduct + if err := self.skipAllKey(); err != nil { + return false, err + } p, i := self.skipKey(key) if !p.Exists() { return false, nil } else if err := p.Check(); err != nil { return false, err } - - self.removePair(i) + self.removePairAt(i) return true, nil } @@ -614,22 +617,28 @@ func (self *Node) SetAnyByIndex(index int, val interface{}) (bool, error) { return self.SetByIndex(index, NewAny(val)) } -// UnsetByIndex remove the node of given index -// WARN: After conducting `UnsetXX()`, the node's length WON'T change +// UnsetByIndex REOMVE (softly) the node of given index. +// +// WARN: this will change address of elements, which is a dangerous action. +// Use Unset() for object or Pop() for array instead. func (self *Node) UnsetByIndex(index int) (bool, error) { - if err := self.Check(); err != nil { + if err := self.checkRaw(); err != nil { return false, err } var p *Node it := self.itype() + if it == types.V_ARRAY { - p = self.Index(index) - }else if it == types.V_OBJECT { - if err := self.checkRaw(); err != nil { + if err := self.skipAllIndex(); err != nil { return false, err } - pr := self.skipIndexPair(index) + p = self.nodeAt(index) + } else if it == types.V_OBJECT { + if err := self.skipAllKey(); err != nil { + return false, err + } + pr := self.pairAt(index) if pr == nil { return false, ErrNotExist } @@ -642,6 +651,12 @@ func (self *Node) UnsetByIndex(index int) (bool, error) { return false, ErrNotExist } + // last elem + if index == self.len() - 1 { + return true, self.Pop() + } + + // not last elem, self.len() change but linked-chunk not change if it == types.V_ARRAY { self.removeNode(index) }else if it == types.V_OBJECT { @@ -665,16 +680,101 @@ func (self *Node) Add(node Node) error { if err := self.should(types.V_ARRAY, "an array"); err != nil { return err } + s, err := self.unsafeArray() if err != nil { return err } - s.Add(node) + // Notice: array won't have unset node in tail + s.Push(node) self.l++ return nil } +// Pop remove the last child of the V_Array or V_Object node. +func (self *Node) Pop() error { + if err := self.checkRaw(); err != nil { + return err + } + + if it := self.itype(); it == types.V_ARRAY { + s, err := self.unsafeArray() + if err != nil { + return err + } + // remove tail unset nodes + for i := s.Len()-1; i >= 0; i-- { + if s.At(i).Exists() { + s.Pop() + self.l-- + break + } + s.Pop() + } + + } else if it == types.V_OBJECT { + s, err := self.unsafeMap() + if err != nil { + return err + } + // remove tail unset nodes + for i := s.Len()-1; i >= 0; i-- { + if p := s.At(i); p != nil && p.Value.Exists() { + s.Pop() + self.l-- + break + } + s.Pop() + } + + } else { + return ErrUnsupportType + } + + return nil +} + +// Move moves the child at src index to dst index, +// meanwhile slides sliblings from src+1 to dst. +// +// WARN: this will change address of elements, which is a dangerous action. +func (self *Node) Move(dst, src int) error { + if err := self.should(types.V_ARRAY, "an array"); err != nil { + return err + } + + s, err := self.unsafeArray() + if err != nil { + return err + } + + // check if any unset node exists + if l := s.Len(); self.len() != l { + di, si := dst, src + // find real pos of src and dst + for i := 0; i < l; i++ { + if s.At(i).Exists() { + di-- + si-- + } + if di == -1 { + dst = i + di-- + } + if si == -1 { + src = i + si-- + } + if di == -2 && si == -2 { + break + } + } + } + + s.MoveOne(src, dst) + return nil +} // SetAny wraps val with V_ANY node, and Add() the node. func (self *Node) AddAny(val interface{}) error { @@ -721,8 +821,6 @@ func (self *Node) Get(key string) *Node { // Index indexies node at given idx, // node type CAN be either V_OBJECT or V_ARRAY -// WARN: After conducting `UnsetXX()`, the node's length WON'T change, -// thus its children's indexing WON'T change too func (self *Node) Index(idx int) *Node { if err := self.checkRaw(); err != nil { return unwrapError(err) @@ -746,8 +844,6 @@ func (self *Node) Index(idx int) *Node { // IndexPair indexies pair at given idx, // node type MUST be either V_OBJECT -// WARN: After conducting `UnsetXX()`, the node's length WON'T change, -// thus its children's indexing WON'T change too func (self *Node) IndexPair(idx int) *Pair { if err := self.should(types.V_OBJECT, "an object"); err != nil { return nil @@ -755,19 +851,30 @@ func (self *Node) IndexPair(idx int) *Pair { return self.skipIndexPair(idx) } +func (self *Node) indexOrGet(idx int, key string) (*Node, int) { + if err := self.should(types.V_OBJECT, "an object"); err != nil { + return unwrapError(err), idx + } + + pr := self.skipIndexPair(idx) + if pr != nil && pr.Key == key { + return &pr.Value, idx + } + + return self.skipKey(key) +} + // IndexOrGet firstly use idx to index a value and check if its key matches // If not, then use the key to search value func (self *Node) IndexOrGet(idx int, key string) *Node { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return unwrapError(err) - } + node, _ := self.indexOrGet(idx, key) + return node +} - pr := self.skipIndexPair(idx) - if pr != nil && pr.Key == key { - return &pr.Value - } - n, _ := self.skipKey(key) - return n +// IndexOrGetWithIdx attempts to retrieve a node by index and key, returning the node and its correct index. +// If the key does not match at the given index, it searches by key and returns the node with its updated index. +func (self *Node) IndexOrGetWithIdx(idx int, key string) (*Node, int) { + return self.indexOrGet(idx, key) } /** Generic Value Converters **/ @@ -864,7 +971,7 @@ func (self *Node) SortKeys(recurse bool) error { } if self.itype() == types.V_OBJECT { return self.sortKeys(recurse) - } else { + } else if self.itype() == types.V_ARRAY { var err error err2 := self.ForEach(func(path Sequence, node *Node) bool { it := node.itype() @@ -880,10 +987,16 @@ func (self *Node) SortKeys(recurse bool) error { return err } return err2 + } else { + return nil } } func (self *Node) sortKeys(recurse bool) (err error) { + // check raw node first + if err := self.checkRaw(); err != nil { + return err + } ps, err := self.unsafeMap() if err != nil { return err @@ -1172,6 +1285,19 @@ func (self *Node) nodeAt(i int) *Node { p = &stack.v } else { p = (*linkedNodes)(self.p) + if l := p.Len(); l != self.len() { + // some nodes got unset, iterate to skip them + for j:=0; j<l; j++ { + v := p.At(j) + if v.Exists() { + i-- + } + if i < 0 { + return v + } + } + return nil + } } return p.At(i) } @@ -1183,6 +1309,19 @@ func (self *Node) pairAt(i int) *Pair { p = &stack.v } else { p = (*linkedPairs)(self.p) + if l := p.Len(); l != self.len() { + // some nodes got unset, iterate to skip them + for j:=0; j<l; j++ { + v := p.At(j) + if v != nil && v.Value.Exists() { + i-- + } + if i < 0 { + return v + } + } + return nil + } } return p.At(i) } @@ -1334,8 +1473,8 @@ func (self *Node) removeNode(i int) { return } *node = Node{} - // NOTICE: for consistency with linkedNodes, we DOSEN'T reduce size here - // self.l-- + // NOTICE: not be consistent with linkedNode.Len() + self.l-- } func (self *Node) removePair(i int) { @@ -1344,8 +1483,18 @@ func (self *Node) removePair(i int) { return } *last = Pair{} - // NOTICE: for consistency with linkedNodes, we DOSEN'T reduce size here - // self.l-- + // NOTICE: should be consistent with linkedPair.Len() + self.l-- +} + +func (self *Node) removePairAt(i int) { + p := (*linkedPairs)(self.p).At(i) + if p == nil { + return + } + *p = Pair{} + // NOTICE: should be consistent with linkedPair.Len() + self.l-- } func (self *Node) toGenericArray() ([]interface{}, error) { @@ -1353,17 +1502,16 @@ func (self *Node) toGenericArray() ([]interface{}, error) { if nb == 0 { return []interface{}{}, nil } - ret := make([]interface{}, nb) + ret := make([]interface{}, 0, nb) /* convert each item */ - var s = (*linkedNodes)(self.p) - for i := 0; i < nb; i++ { - p := s.At(i) - x, err := p.Interface() + it := self.values() + for v := it.next(); v != nil; v = it.next() { + vv, err := v.Interface() if err != nil { return nil, err } - ret[i] = x + ret = append(ret, vv) } /* all done */ @@ -1375,17 +1523,16 @@ func (self *Node) toGenericArrayUseNumber() ([]interface{}, error) { if nb == 0 { return []interface{}{}, nil } - ret := make([]interface{}, nb) + ret := make([]interface{}, 0, nb) /* convert each item */ - var s = (*linkedNodes)(self.p) - for i := 0; i < nb; i++ { - p := s.At(i) - x, err := p.InterfaceUseNumber() + it := self.values() + for v := it.next(); v != nil; v = it.next() { + vv, err := v.InterfaceUseNumber() if err != nil { return nil, err } - ret[i] = x + ret = append(ret, vv) } /* all done */ @@ -1413,14 +1560,13 @@ func (self *Node) toGenericObject() (map[string]interface{}, error) { ret := make(map[string]interface{}, nb) /* convert each item */ - var s = (*linkedPairs)(self.p) - for i := 0; i < nb; i++ { - p := s.At(i) - x, err := p.Value.Interface() + it := self.properties() + for v := it.next(); v != nil; v = it.next() { + vv, err := v.Value.Interface() if err != nil { return nil, err } - ret[p.Key] = x + ret[v.Key] = vv } /* all done */ @@ -1436,14 +1582,13 @@ func (self *Node) toGenericObjectUseNumber() (map[string]interface{}, error) { ret := make(map[string]interface{}, nb) /* convert each item */ - var s = (*linkedPairs)(self.p) - for i := 0; i < nb; i++ { - p := s.At(i) - x, err := p.Value.InterfaceUseNumber() + it := self.properties() + for v := it.next(); v != nil; v = it.next() { + vv, err := v.Value.InterfaceUseNumber() if err != nil { return nil, err } - ret[p.Key] = x + ret[v.Key] = vv } /* all done */ diff --git a/vendor/github.com/bytedance/sonic/ast/parser.go b/vendor/github.com/bytedance/sonic/ast/parser.go index cb16f20bb..3e5309c19 100644 --- a/vendor/github.com/bytedance/sonic/ast/parser.go +++ b/vendor/github.com/bytedance/sonic/ast/parser.go @@ -157,7 +157,7 @@ func (self *Parser) decodeArray(ret *linkedNodes) (Node, types.ParsingError) { } /* add the value to result */ - ret.Add(val) + ret.Push(val) self.p = self.lspace(self.p) /* check for EOF */ @@ -244,7 +244,7 @@ func (self *Parser) decodeObject(ret *linkedPairs) (Node, types.ParsingError) { /* add the value to result */ // FIXME: ret's address may change here, thus previous referred node in ret may be invalid !! - ret.Add(Pair{Key: key, Value: val}) + ret.Push(Pair{Key: key, Value: val}) self.p = self.lspace(self.p) /* check for EOF */ @@ -475,7 +475,7 @@ func (self *Node) skipNextNode() *Node { } /* add the value to result */ - ret.Add(val) + ret.Push(val) self.l++ parser.p = parser.lspace(parser.p) @@ -558,7 +558,7 @@ func (self *Node) skipNextPair() (*Pair) { } /* add the value to result */ - ret.Add(Pair{Key: key, Value: val}) + ret.Push(Pair{Key: key, Value: val}) self.l++ parser.p = parser.lspace(parser.p) diff --git a/vendor/github.com/bytedance/sonic/ast/search.go b/vendor/github.com/bytedance/sonic/ast/search.go index bb6fceaa7..7108e7ea6 100644 --- a/vendor/github.com/bytedance/sonic/ast/search.go +++ b/vendor/github.com/bytedance/sonic/ast/search.go @@ -16,6 +16,11 @@ package ast +import ( + `github.com/bytedance/sonic/internal/rt` + `github.com/bytedance/sonic/internal/native/types` +) + type Searcher struct { parser Parser } @@ -28,3 +33,48 @@ func NewSearcher(str string) *Searcher { }, } } + +// GetByPathCopy search in depth from top json and returns a **Copied** json node at the path location +func (self *Searcher) GetByPathCopy(path ...interface{}) (Node, error) { + return self.getByPath(true, path...) +} + +// GetByPathNoCopy search in depth from top json and returns a **Referenced** json node at the path location +// +// WARN: this search directly refer partial json from top json, which has faster speed, +// may consumes more memory. +func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { + return self.getByPath(false, path...) +} + +func (self *Searcher) getByPath(copystring bool, path ...interface{}) (Node, error) { + var err types.ParsingError + var start int + + self.parser.p = 0 + start, err = self.parser.getByPath(path...) + if err != 0 { + // for compatibility with old version + if err == types.ERR_NOT_FOUND { + return Node{}, ErrNotExist + } + if err == types.ERR_UNSUPPORT_TYPE { + panic("path must be either int(>=0) or string") + } + return Node{}, self.parser.syntaxError(err) + } + + t := switchRawType(self.parser.s[start]) + if t == _V_NONE { + return Node{}, self.parser.ExportError(err) + } + + // copy string to reducing memory usage + var raw string + if copystring { + raw = rt.Mem2Str([]byte(self.parser.s[start:self.parser.p])) + } else { + raw = self.parser.s[start:self.parser.p] + } + return newRawNode(raw, t), nil +} diff --git a/vendor/github.com/bytedance/sonic/compat.go b/vendor/github.com/bytedance/sonic/compat.go index ec414c0cf..728bc1767 100644 --- a/vendor/github.com/bytedance/sonic/compat.go +++ b/vendor/github.com/bytedance/sonic/compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64 !go1.16 go1.23 /* * Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go b/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go index 7c2845514..346ebbce5 100644 --- a/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go +++ b/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16,!go1.22 +// +build amd64,go1.16,!go1.23 /* * Copyright 2023 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go b/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go index 84bae4387..7883862c3 100644 --- a/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go +++ b/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64 !go1.16 go1.23 /* * Copyright 2023 ByteDance Inc. @@ -30,7 +30,7 @@ import ( ) func init() { - println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable") } const ( diff --git a/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go b/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go index e93b09a25..b4f1b7b52 100644 --- a/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go +++ b/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16,!go1.22 +// +build amd64,go1.16,!go1.23 /* * Copyright 2023 ByteDance Inc. @@ -63,6 +63,9 @@ const ( // after encoding the JSONMarshaler to JSON. NoValidateJSONMarshaler Options = encoder.NoValidateJSONMarshaler + // NoEncoderNewline indicates that the encoder should not add a newline after every message + NoEncoderNewline Options = encoder.NoEncoderNewline + // CompatibleWithStd is used to be compatible with std encoder. CompatibleWithStd Options = encoder.CompatibleWithStd ) diff --git a/vendor/github.com/bytedance/sonic/encoder/encoder_compat.go b/vendor/github.com/bytedance/sonic/encoder/encoder_compat.go index 2e02b59cd..320dd9b5c 100644 --- a/vendor/github.com/bytedance/sonic/encoder/encoder_compat.go +++ b/vendor/github.com/bytedance/sonic/encoder/encoder_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64 !go1.16 go1.23 /* * Copyright 2023 ByteDance Inc. @@ -28,7 +28,7 @@ import ( ) func init() { - println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable") } // Options is a set of encoding options. @@ -42,6 +42,7 @@ const ( bitNoNullSliceOrMap bitValidateString bitNoValidateJSONMarshaler + bitNoEncoderNewline // used for recursive compile bitPointerValue = 63 @@ -77,6 +78,9 @@ const ( // NoValidateJSONMarshaler indicates that the encoder should not validate the output string // after encoding the JSONMarshaler to JSON. NoValidateJSONMarshaler Options = 1 << bitNoValidateJSONMarshaler + + // NoEncoderNewline indicates that the encoder should not add a newline after every message + NoEncoderNewline Options = 1 << bitNoEncoderNewline // CompatibleWithStd is used to be compatible with std encoder. CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler @@ -130,6 +134,15 @@ func (self *Encoder) SetNoValidateJSONMarshaler(f bool) { } } +// SetNoEncoderNewline specifies if option NoEncoderNewline opens +func (self *Encoder) SetNoEncoderNewline(f bool) { + if f { + self.Opts |= NoEncoderNewline + } else { + self.Opts &= ^NoEncoderNewline + } +} + // SetCompactMarshaler specifies if option CompactMarshaler opens func (self *Encoder) SetCompactMarshaler(f bool) { if f { diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/asm_stubs_amd64_go121.go b/vendor/github.com/bytedance/sonic/internal/decoder/asm_stubs_amd64_go121.go index 5db5b5cd0..6adeac0cf 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/asm_stubs_amd64_go121.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/asm_stubs_amd64_go121.go @@ -1,4 +1,4 @@ -// +build go1.21,!go1.22 +// +build go1.21,!go1.23 // Copyright 2023 CloudWeGo Authors // diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/assembler_regabi_amd64.go b/vendor/github.com/bytedance/sonic/internal/decoder/assembler_regabi_amd64.go index 3d223e14e..0defb75a5 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/assembler_regabi_amd64.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/assembler_regabi_amd64.go @@ -1,5 +1,4 @@ -//go:build go1.17 && !go1.22 -// +build go1.17,!go1.22 +// +build go1.17,!go1.23 /* * Copyright 2021 ByteDance Inc. @@ -420,9 +419,9 @@ func (self *_Assembler) call_go(fn obj.Addr) { } func (self *_Assembler) callc(fn obj.Addr) { - self.Emit("XCHGQ", _IP, _BP) + self.save(_IP) self.call(fn) - self.Emit("XCHGQ", _IP, _BP) + self.load(_IP) } func (self *_Assembler) call_c(fn obj.Addr) { @@ -1164,7 +1163,7 @@ var ( var ( _F_FieldMap_GetCaseInsensitive obj.Addr - _Empty_Slice = make([]byte, 0) + _Empty_Slice = []byte{} _Zero_Base = int64(uintptr(((*rt.GoSlice)(unsafe.Pointer(&_Empty_Slice))).Ptr)) ) @@ -1641,7 +1640,8 @@ func (self *_Assembler) _asm_OP_check_empty(p *_Instr) { self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(rbracket))) // CMPB (IP)(IC), ']' self.Sjmp("JNE" , "_not_empty_array_{n}") // JNE _not_empty_array_{n} self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC - self.StorePtr(_Zero_Base, jit.Ptr(_VP, 0), _AX) // MOVQ $zerobase, (VP) + self.Emit("MOVQ", jit.Imm(_Zero_Base), _AX) + self.WritePtrAX(9, jit.Ptr(_VP, 0), false) self.Emit("PXOR", _X0, _X0) // PXOR X0, X0 self.Emit("MOVOU", _X0, jit.Ptr(_VP, 8)) // MOVOU X0, 8(VP) self.Xjmp("JMP" , p.vi()) // JMP {p.vi()} diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/assembler_stkabi_amd64.go b/vendor/github.com/bytedance/sonic/internal/decoder/assembler_stkabi_amd64.go index 57a38b420..9e2acc23f 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/assembler_stkabi_amd64.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/assembler_stkabi_amd64.go @@ -1651,7 +1651,8 @@ func (self *_Assembler) _asm_OP_check_empty(p *_Instr) { self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(rbracket))) // CMPB (IP)(IC), ']' self.Sjmp("JNE" , "_not_empty_array_{n}") // JNE _not_empty_array_{n} self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC - self.StorePtr(_Zero_Base, jit.Ptr(_VP, 0), _AX) // MOVQ $zerobase, (VP) + self.Emit("MOVQ", jit.Imm(_Zero_Base), _AX) + self.WritePtrAX(9, jit.Ptr(_VP, 0), false) self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 self.Emit("MOVOU", _X0, jit.Ptr(_VP, 8)) // MOVOU X0, 8(VP) self.Xjmp("JMP" , p.vi()) // JMP {p.vi()} diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/compiler.go b/vendor/github.com/bytedance/sonic/internal/decoder/compiler.go index e9e2b77fc..b350c0461 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/compiler.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/compiler.go @@ -527,40 +527,47 @@ func (self *_Compiler) compile(vt reflect.Type) (ret _Program, err error) { return } -func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) { - /* check for recursive nesting */ - ok := self.tab[vt] - if ok { - p.rtt(_OP_recurse, vt) - return - } - +func (self *_Compiler) checkMarshaler(p *_Program, vt reflect.Type) bool { pt := reflect.PtrTo(vt) /* check for `json.Unmarshaler` with pointer receiver */ if pt.Implements(jsonUnmarshalerType) { p.rtt(_OP_unmarshal_p, pt) - return + return true } /* check for `json.Unmarshaler` */ if vt.Implements(jsonUnmarshalerType) { p.add(_OP_lspace) self.compileUnmarshalJson(p, vt) - return + return true } /* check for `encoding.TextMarshaler` with pointer receiver */ if pt.Implements(encodingTextUnmarshalerType) { p.add(_OP_lspace) self.compileUnmarshalTextPtr(p, pt) - return + return true } /* check for `encoding.TextUnmarshaler` */ if vt.Implements(encodingTextUnmarshalerType) { p.add(_OP_lspace) self.compileUnmarshalText(p, vt) + return true + } + return false +} + +func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) { + /* check for recursive nesting */ + ok := self.tab[vt] + if ok { + p.rtt(_OP_recurse, vt) + return + } + + if self.checkMarshaler(p, vt) { return } @@ -683,17 +690,9 @@ func (self *_Compiler) compilePtr(p *_Program, sp int, et reflect.Type) { /* dereference all the way down */ for et.Kind() == reflect.Ptr { - if et.Implements(jsonUnmarshalerType) { - p.rtt(_OP_unmarshal_p, et) + if self.checkMarshaler(p, et) { return } - - if et.Implements(encodingTextUnmarshalerType) { - p.add(_OP_lspace) - self.compileUnmarshalTextPtr(p, et) - return - } - et = et.Elem() p.rtt(_OP_deref, et) } @@ -706,7 +705,7 @@ func (self *_Compiler) compilePtr(p *_Program, sp int, et reflect.Type) { /* enter the recursion */ p.add(_OP_lspace) self.tab[et] = true - + /* not inline the pointer type * recursing the defined pointer type's elem will casue issue379. */ @@ -716,8 +715,12 @@ func (self *_Compiler) compilePtr(p *_Program, sp int, et reflect.Type) { j := p.pc() p.add(_OP_goto) + + // set val pointer as nil p.pin(i) p.add(_OP_nil_1) + + // nothing todo p.pin(j) } diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64.go b/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64.go index 337af054c..c7514cb41 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64.go @@ -1,4 +1,4 @@ -// +build go1.17,!go1.22 +// +build go1.17,!go1.23 /* * Copyright 2021 ByteDance Inc. @@ -119,9 +119,9 @@ func (self *_ValueDecoder) call_go(fn obj.Addr) { } func (self *_ValueDecoder) callc(fn obj.Addr) { - self.Emit("XCHGQ", _IP, _BP) + self.save(_IP) self.call(fn) - self.Emit("XCHGQ", _IP, _BP) + self.load(_IP) } func (self *_ValueDecoder) call_c(fn obj.Addr) { diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64_test.s b/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64_test.s index 1c46928de..b4b0de183 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64_test.s +++ b/vendor/github.com/bytedance/sonic/internal/decoder/generic_regabi_amd64_test.s @@ -1,4 +1,4 @@ -// +build go1.17,!go1.22 +// +build go1.17,!go1.23 // // Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/internal/decoder/stream.go b/vendor/github.com/bytedance/sonic/internal/decoder/stream.go index a3716435a..7eb8a6951 100644 --- a/vendor/github.com/bytedance/sonic/internal/decoder/stream.go +++ b/vendor/github.com/bytedance/sonic/internal/decoder/stream.go @@ -23,11 +23,12 @@ import ( `github.com/bytedance/sonic/internal/native` `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` `github.com/bytedance/sonic/option` ) var ( - minLeftBufferShift uint = 1 + minLeftBufferShift uint = 1 ) // StreamDecoder is the decoder context object for streaming input. @@ -58,95 +59,71 @@ func NewStreamDecoder(r io.Reader) *StreamDecoder { // Either io error from underlying io.Reader (except io.EOF) // or syntax error from data will be recorded and stop subsequently decoding. func (self *StreamDecoder) Decode(val interface{}) (err error) { - if self.err != nil { - return self.err - } - - var buf = self.buf[self.scanp:] - var p = 0 - var recycle bool - if cap(buf) == 0 { - buf = bufPool.Get().([]byte) - recycle = true - } - - var first = true - var repeat = true - -read_more: - for { - l := len(buf) - realloc(&buf) - n, err := self.r.Read(buf[l:cap(buf)]) - buf = buf[:l+n] - if err != nil { - repeat = false - if err == io.EOF { - if len(buf) == 0 { - return err - } - break - } - self.err = err - return err - } - if n > 0 || first { - break - } - } - first = false - - l := len(buf) - if l > 0 { - self.Decoder.Reset(string(buf)) - - var x int - if ret := native.SkipOneFast(&self.s, &x); ret < 0 { - if repeat { - goto read_more + // read more data into buf + if self.More() { + // println(string(self.buf)) + var s = self.scanp + try_skip: + var e = len(self.buf) + // println("s:", s, "e:", e, "scanned:",self.scanned, "scanp:",self.scanp, self.buf) + var src = rt.Mem2Str(self.buf[s:e]) + // if len(src) > 5 { + // println(src[:5], src[len(src)-5:]) + // } else { + // println(src) + // } + // try skip + var x = 0; + if y := native.SkipOneFast(&src, &x); y < 0 { + if self.readMore() { + // println("more") + goto try_skip } else { - err = SyntaxError{x, self.s, types.ParsingError(-ret), ""} - self.err = err + // println("no more") + err = SyntaxError{e, self.s, types.ParsingError(-s), ""} + self.setErr(err) return } + } else { + s = y + s + e = x + s } - + + // println("decode: ", s, e) + // must copy string here for safety + self.Decoder.Reset(string(self.buf[s:e])) err = self.Decoder.Decode(val) if err != nil { - self.err = err + self.setErr(err) + return } - p = self.Decoder.Pos() - self.scanned += int64(p) - self.scanp = 0 - } - - if l > p { - // remain undecoded bytes, so copy them into self.buf - self.buf = append(self.buf[:0], buf[p:]...) - } else { - self.buf = nil - recycle = true - } + self.scanp = e + _, empty := self.scan() + if empty { + // println("recycle") + // no remain valid bytes, thus we just recycle buffer + mem := self.buf + self.buf = nil + bufPool.Put(mem[:0]) + } else { + // println("keep") + // remain undecoded bytes, move them onto head + n := copy(self.buf, self.buf[self.scanp:]) + self.buf = self.buf[:n] + } - if recycle { - buf = buf[:0] - bufPool.Put(buf) - } - return err -} + self.scanned += int64(self.scanp) + self.scanp = 0 + } -func (self StreamDecoder) repeatable(err error) bool { - if ee, ok := err.(SyntaxError); ok && - (ee.Code == types.ERR_EOF || (ee.Code == types.ERR_INVALID_CHAR && self.i >= len(self.s)-1)) { - return true - } - return false + return self.err } // InputOffset returns the input stream byte offset of the current decoder position. // The offset gives the location of the end of the most recently returned token and the beginning of the next token. func (self *StreamDecoder) InputOffset() int64 { + // println("input offset",self.scanned, self.scanp) return self.scanned + int64(self.scanp) } @@ -166,28 +143,72 @@ func (self *StreamDecoder) More() bool { return err == nil && c != ']' && c != '}' } +// More reports whether there is another element in the +// current array or object being parsed. +func (self *StreamDecoder) readMore() bool { + if self.err != nil { + return false + } + + var err error + var n int + for { + // Grow buffer if not large enough. + l := len(self.buf) + realloc(&self.buf) + + n, err = self.r.Read(self.buf[l:cap(self.buf)]) + self.buf = self.buf[: l+n] + + self.scanp = l + _, empty := self.scan() + if !empty { + return true + } + + // buffer has been scanned, now report any error + if err != nil { + self.setErr(err) + return false + } + } +} + +func (self *StreamDecoder) setErr(err error) { + self.err = err + mem := self.buf[:0] + self.buf = nil + bufPool.Put(mem) +} + func (self *StreamDecoder) peek() (byte, error) { var err error for { - for i := self.scanp; i < len(self.buf); i++ { - c := self.buf[i] - if isSpace(c) { - continue - } - self.scanp = i - return c, nil + c, empty := self.scan() + if !empty { + return byte(c), nil } // buffer has been scanned, now report any error if err != nil { - if err != io.EOF { - self.err = err - } + self.setErr(err) return 0, err } err = self.refill() } } +func (self *StreamDecoder) scan() (byte, bool) { + for i := self.scanp; i < len(self.buf); i++ { + c := self.buf[i] + if isSpace(c) { + continue + } + self.scanp = i + return c, false + } + return 0, true +} + func isSpace(c byte) bool { return types.SPACE_MASK & (1 << c) != 0 } @@ -212,17 +233,25 @@ func (self *StreamDecoder) refill() error { return err } -func realloc(buf *[]byte) { +func realloc(buf *[]byte) bool { l := uint(len(*buf)) c := uint(cap(*buf)) + if c == 0 { + // println("use pool!") + *buf = bufPool.Get().([]byte) + return true + } if c - l <= c >> minLeftBufferShift { + // println("realloc!") e := l+(l>>minLeftBufferShift) - if e < option.DefaultDecoderBufferSize { - e = option.DefaultDecoderBufferSize + if e <= c { + e = c*2 } tmp := make([]byte, l, e) copy(tmp, *buf) *buf = tmp + return true } + return false } diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/asm_stubs_amd64_go121.go b/vendor/github.com/bytedance/sonic/internal/encoder/asm_stubs_amd64_go121.go index 2f9445ac4..9f7ff65e6 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/asm_stubs_amd64_go121.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/asm_stubs_amd64_go121.go @@ -1,4 +1,4 @@ -// +build go1.21,!go1.22 +// +build go1.21,!go1.23 // Copyright 2023 CloudWeGo Authors // diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/assembler_regabi_amd64.go b/vendor/github.com/bytedance/sonic/internal/encoder/assembler_regabi_amd64.go index a89364b14..4584c6d29 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/assembler_regabi_amd64.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/assembler_regabi_amd64.go @@ -1,5 +1,4 @@ -//go:build go1.17 && !go1.22 -// +build go1.17,!go1.22 +// +build go1.17,!go1.23 /* * Copyright 2021 ByteDance Inc. @@ -171,7 +170,7 @@ var ( ) var ( - _REG_ffi = []obj.Addr{ _RP, _RL, _RC} + _REG_ffi = []obj.Addr{ _RP, _RL, _RC, _SP_q} _REG_b64 = []obj.Addr{_SP_p, _SP_q} _REG_all = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC} @@ -510,11 +509,9 @@ func (self *_Assembler) call_b64(pc obj.Addr) { func (self *_Assembler) call_c(pc obj.Addr) { self.Emit("XCHGQ", _SP_p, _BX) - self.Emit("XCHGQ", _SP_q, _BP) self.call(pc) // CALL $pc self.xload(_REG_ffi...) // LOAD $REG_ffi self.Emit("XCHGQ", _SP_p, _BX) - self.Emit("XCHGQ", _SP_q, _BP) } func (self *_Assembler) call_go(pc obj.Addr) { diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/compiler.go b/vendor/github.com/bytedance/sonic/internal/encoder/compiler.go index a949c90f7..ca0be8f40 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/compiler.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/compiler.go @@ -831,7 +831,7 @@ func (self *_Compiler) compileStructFieldZero(p *_Program, vt reflect.Type) { case reflect.Float32 : p.add(_OP_is_zero_4) case reflect.Float64 : p.add(_OP_is_zero_8) case reflect.String : p.add(_OP_is_nil_p1) - case reflect.Interface : p.add(_OP_is_nil_p1) + case reflect.Interface : p.add(_OP_is_nil) case reflect.Map : p.add(_OP_is_zero_map) case reflect.Ptr : p.add(_OP_is_nil) case reflect.Slice : p.add(_OP_is_nil_p1) diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/debug_go117.go b/vendor/github.com/bytedance/sonic/internal/encoder/debug_go117.go index 56a6cbf5e..37e6f7d4f 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/debug_go117.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/debug_go117.go @@ -1,4 +1,4 @@ -// +build go1.17,!go1.22 +// +build go1.17,!go1.23 /* * Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/encoder.go b/vendor/github.com/bytedance/sonic/internal/encoder/encoder.go index bd8bae357..0a46455eb 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/encoder.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/encoder.go @@ -41,6 +41,7 @@ const ( bitNoNullSliceOrMap bitValidateString bitNoValidateJSONMarshaler + bitNoEncoderNewline // used for recursive compile bitPointerValue = 63 @@ -76,6 +77,9 @@ const ( // NoValidateJSONMarshaler indicates that the encoder should not validate the output string // after encoding the JSONMarshaler to JSON. NoValidateJSONMarshaler Options = 1 << bitNoValidateJSONMarshaler + + // NoEncoderNewline indicates that the encoder should not add a newline after every message + NoEncoderNewline Options = 1 << bitNoEncoderNewline // CompatibleWithStd is used to be compatible with std encoder. CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler @@ -129,6 +133,16 @@ func (self *Encoder) SetNoValidateJSONMarshaler(f bool) { } } +// SetNoEncoderNewline specifies if option NoEncoderNewline opens +func (self *Encoder) SetNoEncoderNewline(f bool) { + if f { + self.Opts |= NoEncoderNewline + } else { + self.Opts &= ^NoEncoderNewline + } +} + + // SetCompactMarshaler specifies if option CompactMarshaler opens func (self *Encoder) SetCompactMarshaler(f bool) { if f { diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/primitives.go b/vendor/github.com/bytedance/sonic/internal/encoder/primitives.go index 0e47987c7..0f3c40881 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/primitives.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/primitives.go @@ -154,17 +154,17 @@ var ( ) var ( - _F_assertI2I = jit.Func(assertI2I) + _F_assertI2I = jit.Func(rt.AssertI2I2) ) func asText(v unsafe.Pointer) (string, error) { - text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) + text := rt.AssertI2I2(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() return rt.Mem2Str(r), e } func asJson(v unsafe.Pointer) (string, error) { - text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v)) + text := rt.AssertI2I2(_T_json_Marshaler, *(*rt.GoIface)(v)) r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON() return rt.Mem2Str(r), e } diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/stream.go b/vendor/github.com/bytedance/sonic/internal/encoder/stream.go index b6f3ce5fb..d498f68fc 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/stream.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/stream.go @@ -36,7 +36,8 @@ func NewStreamEncoder(w io.Writer) *StreamEncoder { // Encode encodes interface{} as JSON to io.Writer func (enc *StreamEncoder) Encode(val interface{}) (err error) { - out := newBytes() + buf := newBytes() + out := buf /* encode into the buffer */ err = EncodeInto(&out, val, enc.Opts) @@ -54,7 +55,9 @@ func (enc *StreamEncoder) Encode(val interface{}) (err error) { } // according to standard library, terminate each value with a newline... - buf.WriteByte('\n') + if enc.Opts & NoEncoderNewline == 0 { + buf.WriteByte('\n') + } /* copy into io.Writer */ _, err = io.Copy(enc.w, buf) @@ -75,10 +78,12 @@ func (enc *StreamEncoder) Encode(val interface{}) (err error) { } // according to standard library, terminate each value with a newline... - enc.w.Write([]byte{'\n'}) + if enc.Opts & NoEncoderNewline == 0 { + enc.w.Write([]byte{'\n'}) + } } free_bytes: - freeBytes(out) + freeBytes(buf) return err } diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go116.go b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go116.go index 5577c58d7..a21b6b4a6 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go116.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go116.go @@ -38,10 +38,6 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) //goland:noinspection GoUnusedParameter func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice -//go:linkname assertI2I runtime.assertI2I -//goland:noinspection GoUnusedParameter -func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface - //go:linkname mapiternext runtime.mapiternext //goland:noinspection GoUnusedParameter func mapiternext(it *rt.GoMapIterator) diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go117.go b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go117.go index 6c8c6ec75..7d934be33 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go117.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go117.go @@ -38,10 +38,6 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) //goland:noinspection GoUnusedParameter func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice -//go:linkname assertI2I runtime.assertI2I2 -//goland:noinspection GoUnusedParameter -func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface - //go:linkname mapiternext runtime.mapiternext //goland:noinspection GoUnusedParameter func mapiternext(it *rt.GoMapIterator) diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go120.go b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go120.go index 94a2c0f56..ce3c88e9a 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go120.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go120.go @@ -38,10 +38,6 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) //goland:noinspection GoUnusedParameter func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice -//go:linkname assertI2I runtime.assertI2I2 -//goland:noinspection GoUnusedParameter -func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface - //go:linkname mapiternext runtime.mapiternext //goland:noinspection GoUnusedParameter func mapiternext(it *rt.GoMapIterator) diff --git a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go121.go b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go121.go index e194fbbf3..700ddf8b5 100644 --- a/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go121.go +++ b/vendor/github.com/bytedance/sonic/internal/encoder/stubs_go121.go @@ -38,10 +38,6 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) //goland:noinspection GoUnusedParameter func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice -//go:linkname assertI2I runtime.assertI2I2 -//goland:noinspection GoUnusedParameter -func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface - //go:linkname mapiternext runtime.mapiternext //goland:noinspection GoUnusedParameter func mapiternext(it *rt.GoMapIterator) diff --git a/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go b/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go index d7d1751e5..00e6009db 100644 --- a/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go +++ b/vendor/github.com/bytedance/sonic/internal/jit/assembler_amd64.go @@ -72,18 +72,6 @@ func (self *BaseAssembler) NOPn(n int) { } } -func (self *BaseAssembler) StorePtr(ptr int64, to obj.Addr, tmp obj.Addr) { - if (to.Type != obj.TYPE_MEM) || (tmp.Type != obj.TYPE_REG) { - panic("must store imm to memory, tmp must be register") - } - if (ptr >> 32) != 0 { - self.Emit("MOVQ", Imm(ptr), tmp) - self.Emit("MOVQ", tmp, to) - } else { - self.Emit("MOVQ", Imm(ptr), to); - } -} - func (self *BaseAssembler) Byte(v ...byte) { for ; len(v) >= 8; v = v[8:] { self.From("QUAD", Imm(rt.Get64(v))) } for ; len(v) >= 4; v = v[4:] { self.From("LONG", Imm(int64(rt.Get32(v)))) } diff --git a/vendor/github.com/bytedance/sonic/internal/jit/runtime.go b/vendor/github.com/bytedance/sonic/internal/jit/runtime.go index ec69d067a..e4bc829da 100644 --- a/vendor/github.com/bytedance/sonic/internal/jit/runtime.go +++ b/vendor/github.com/bytedance/sonic/internal/jit/runtime.go @@ -24,11 +24,6 @@ import ( `github.com/twitchyliquid64/golang-asm/obj` ) -//go:noescape -//go:linkname getitab runtime.getitab -//goland:noinspection ALL -func getitab(inter *rt.GoType, typ *rt.GoType, canfail bool) *rt.GoItab - func Func(f interface{}) obj.Addr { if p := rt.UnpackEface(f); p.Type.Kind() != reflect.Func { panic("f is not a function") @@ -42,7 +37,7 @@ func Type(t reflect.Type) obj.Addr { } func Itab(i *rt.GoType, t reflect.Type) obj.Addr { - return Imm(int64(uintptr(unsafe.Pointer(getitab(i, rt.UnpackType(t), false))))) + return Imm(int64(uintptr(unsafe.Pointer(rt.Getitab(rt.IfaceType(i), rt.UnpackType(t), false))))) } func Gitab(i *rt.GoItab) obj.Addr { diff --git a/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s b/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s index 8250e38d3..4998d5f79 100644 --- a/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s +++ b/vendor/github.com/bytedance/sonic/internal/rt/asm_amd64.s @@ -1,5 +1,4 @@ -// +build !noasm !appengine -// Code generated by asm2asm, DO NOT EDIT· +// +build !noasm,amd64 !appengine,amd64 #include "go_asm.h" #include "funcdata.h" diff --git a/vendor/github.com/bytedance/sonic/internal/rt/asm_arm64.s b/vendor/github.com/bytedance/sonic/internal/rt/asm_compat.s index a168a8266..7d50b6483 100644 --- a/vendor/github.com/bytedance/sonic/internal/rt/asm_arm64.s +++ b/vendor/github.com/bytedance/sonic/internal/rt/asm_compat.s @@ -1,5 +1,4 @@ -// +build !noasm !appengine -// Code generated by asm2asm, DO NOT EDIT. +// +build !noasm,!amd64 !appengine,!amd64 #include "go_asm.h" #include "funcdata.h" diff --git a/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go b/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go index e3bf0db91..a68d98aff 100644 --- a/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go +++ b/vendor/github.com/bytedance/sonic/internal/rt/fastmem.go @@ -66,15 +66,16 @@ func FuncAddr(f interface{}) unsafe.Pointer { } } +//go:nocheckptr func IndexChar(src string, index int) unsafe.Pointer { return unsafe.Pointer(uintptr((*GoString)(unsafe.Pointer(&src)).Ptr) + uintptr(index)) } +//go:nocheckptr func IndexByte(ptr []byte, index int) unsafe.Pointer { return unsafe.Pointer(uintptr((*GoSlice)(unsafe.Pointer(&ptr)).Ptr) + uintptr(index)) } -//go:nosplit func GuardSlice(buf *[]byte, n int) { c := cap(*buf) l := len(*buf) diff --git a/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go b/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go index 2b2757f5b..befaeb715 100644 --- a/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go +++ b/vendor/github.com/bytedance/sonic/internal/rt/fastvalue.go @@ -211,3 +211,24 @@ func findReflectRtypeItab() *GoItab { v := reflect.TypeOf(struct{}{}) return (*GoIface)(unsafe.Pointer(&v)).Itab } + +func AssertI2I2(t *GoType, i GoIface) (r GoIface) { + inter := IfaceType(t) + tab := i.Itab + if tab == nil { + return + } + if (*GoInterfaceType)(tab.it) != inter { + tab = Getitab(inter, tab.Vt, true) + if tab == nil { + return + } + } + r.Itab = tab + r.Value = i.Value + return +} + +//go:noescape +//go:linkname Getitab runtime.getitab +func Getitab(inter *GoInterfaceType, typ *GoType, canfail bool) *GoItab diff --git a/vendor/github.com/bytedance/sonic/internal/rt/int48.go b/vendor/github.com/bytedance/sonic/internal/rt/int48.go index e9f82d731..a0eb64ad7 100644 --- a/vendor/github.com/bytedance/sonic/internal/rt/int48.go +++ b/vendor/github.com/bytedance/sonic/internal/rt/int48.go @@ -17,12 +17,12 @@ package rt const ( - MinInt48 = -(1 << 47) - MaxInt48 = +(1 << 47) - 1 + MinInt48 int64 = -(1 << 47) + MaxInt48 int64 = +(1 << 47) - 1 ) func PackInt(v int) uint64 { - if u := uint64(v); v < MinInt48 || v > MaxInt48 { + if u := uint64(v); int64(v) < MinInt48 || int64(v) > MaxInt48 { panic("int48 out of range") } else { return ((u >> 63) << 47) | (u & 0x00007fffffffffff) diff --git a/vendor/github.com/bytedance/sonic/loader/funcdata_go121.go b/vendor/github.com/bytedance/sonic/loader/funcdata_go121.go index f50663e2a..ebeaca5a7 100644 --- a/vendor/github.com/bytedance/sonic/loader/funcdata_go121.go +++ b/vendor/github.com/bytedance/sonic/loader/funcdata_go121.go @@ -1,5 +1,5 @@ -//go:build go1.21 && !go1.22 -// +build go1.21,!go1.22 +//go:build go1.21 && !go1.23 +// +build go1.21,!go1.23 /* * Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/loader/funcdata_latest.go b/vendor/github.com/bytedance/sonic/loader/funcdata_latest.go index 52c6097fb..08618dca4 100644 --- a/vendor/github.com/bytedance/sonic/loader/funcdata_latest.go +++ b/vendor/github.com/bytedance/sonic/loader/funcdata_latest.go @@ -1,5 +1,5 @@ -// go:build go1.18 && !go1.22 -// +build go1.18,!go1.22 +// go:build go1.18 && !go1.23 +// +build go1.18,!go1.23 /* * Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/loader/loader_latest.go b/vendor/github.com/bytedance/sonic/loader/loader_latest.go index b6e3e75f8..3664c04ba 100644 --- a/vendor/github.com/bytedance/sonic/loader/loader_latest.go +++ b/vendor/github.com/bytedance/sonic/loader/loader_latest.go @@ -1,5 +1,4 @@ -//go:build go1.16 && !go1.22 -// +build go1.16,!go1.22 +// +build go1.16,!go1.23 /* * Copyright 2021 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/loader/mmap_unix.go b/vendor/github.com/bytedance/sonic/loader/mmap_unix.go index 3ea944ed6..26bdb6584 100644 --- a/vendor/github.com/bytedance/sonic/loader/mmap_unix.go +++ b/vendor/github.com/bytedance/sonic/loader/mmap_unix.go @@ -1,5 +1,5 @@ -//go:build darwin || linux -// +build darwin linux +//go:build !windows +// +build !windows /** * Copyright 2023 ByteDance Inc. diff --git a/vendor/github.com/bytedance/sonic/loader/stubs.go b/vendor/github.com/bytedance/sonic/loader/stubs.go index 8377649b7..80f8de836 100644 --- a/vendor/github.com/bytedance/sonic/loader/stubs.go +++ b/vendor/github.com/bytedance/sonic/loader/stubs.go @@ -17,7 +17,8 @@ package loader import ( - `sync` + "sync/atomic" + "unsafe" _ `unsafe` ) @@ -25,16 +26,35 @@ import ( //goland:noinspection GoUnusedGlobalVariable var lastmoduledatap *moduledata -var moduledataMux sync.Mutex - func registerModule(mod *moduledata) { - moduledataMux.Lock() - lastmoduledatap.next = mod - lastmoduledatap = mod - moduledataMux.Unlock() + registerModuleLockFree(&lastmoduledatap, mod) } //go:linkname moduledataverify1 runtime.moduledataverify1 func moduledataverify1(_ *moduledata) +func registerModuleLockFree(tail **moduledata, mod *moduledata) { + for { + oldTail := loadModule(tail) + if casModule(tail, oldTail, mod) { + storeModule(&oldTail.next, mod) + break + } + } +} + +func loadModule(p **moduledata) *moduledata { + return (*moduledata)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func storeModule(p **moduledata, value *moduledata) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(value)) +} + +func casModule(p **moduledata, oldValue *moduledata, newValue *moduledata) bool { + return atomic.CompareAndSwapPointer( + (*unsafe.Pointer)(unsafe.Pointer(p)), + unsafe.Pointer(oldValue), + unsafe.Pointer(newValue), + ) +} diff --git a/vendor/github.com/bytedance/sonic/sonic.go b/vendor/github.com/bytedance/sonic/sonic.go index 1da238895..145527fc0 100644 --- a/vendor/github.com/bytedance/sonic/sonic.go +++ b/vendor/github.com/bytedance/sonic/sonic.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16,!go1.22 +// +build amd64,go1.16,!go1.23 /* * Copyright 2021 ByteDance Inc. @@ -61,6 +61,9 @@ func (cfg Config) Froze() API { if cfg.NoValidateJSONMarshaler { api.encoderOpts |= encoder.NoValidateJSONMarshaler } + if cfg.NoEncoderNewline { + api.encoderOpts |= encoder.NoEncoderNewline + } // configure decoder options: if cfg.UseInt64 { |