summaryrefslogtreecommitdiff
path: root/vendor/github.com/bytedance/sonic/utf8/utf8.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/github.com/bytedance/sonic/utf8/utf8.go')
-rw-r--r--vendor/github.com/bytedance/sonic/utf8/utf8.go81
1 files changed, 0 insertions, 81 deletions
diff --git a/vendor/github.com/bytedance/sonic/utf8/utf8.go b/vendor/github.com/bytedance/sonic/utf8/utf8.go
deleted file mode 100644
index 9d8bcc958..000000000
--- a/vendor/github.com/bytedance/sonic/utf8/utf8.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright 2022 ByteDance Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package utf8
-
-import (
- `runtime`
-
- `github.com/bytedance/sonic/internal/rt`
- `github.com/bytedance/sonic/internal/native/types`
- `github.com/bytedance/sonic/internal/native`
-)
-
-// CorrectWith corrects the invalid utf8 byte with repl string.
-func CorrectWith(dst []byte, src []byte, repl string) []byte {
- sstr := rt.Mem2Str(src)
- sidx := 0
-
- /* state machine records the invalid positions */
- m := types.NewStateMachine()
- m.Sp = 0 // invalid utf8 numbers
-
- for sidx < len(sstr) {
- scur := sidx
- ecode := native.ValidateUTF8(&sstr, &sidx, m)
-
- if m.Sp != 0 {
- if m.Sp > len(sstr) {
- panic("numbers of invalid utf8 exceed the string len!")
- }
- }
-
- for i := 0; i < m.Sp; i++ {
- ipos := m.Vt[i] // invalid utf8 position
- dst = append(dst, sstr[scur:ipos]...)
- dst = append(dst, repl...)
- scur = m.Vt[i] + 1
- }
- /* append the remained valid utf8 bytes */
- dst = append(dst, sstr[scur:sidx]...)
-
- /* not enough space, reset and continue */
- if ecode != 0 {
- m.Sp = 0
- }
- }
-
- types.FreeStateMachine(m)
- return dst
-}
-
-// Validate is a simd-accelereated drop-in replacement for the standard library's utf8.Valid.
-func Validate(src []byte) bool {
- if src == nil {
- return true
- }
- return ValidateString(rt.Mem2Str(src))
-}
-
-// ValidateString as Validate, but for string.
-func ValidateString(src string) bool {
- if src == "" {
- return true
- }
- ret := native.ValidateUTF8Fast(&src) == 0
- runtime.KeepAlive(src)
- return ret
-}