diff options
| author |  Terin Stock <terinjokes@gmail.com> | 2025-03-09 17:47:56 +0100 |
|---|---|---|
| committer | Terin Stock <terinjokes@gmail.com> | 2025-03-10 01:59:49 +0100 |
| commit | 3ac1ee16f377d31a0fb80c8dae28b6239ac4229e (patch) | |
| tree | f61faa581feaaeaba2542b9f2b8234a590684413 /vendor/github.com/klauspost | |
| parent | [chore] update URLs to forked source (diff) | |
| download | gotosocial-3ac1ee16f377d31a0fb80c8dae28b6239ac4229e.tar.xz | |
[chore] remove vendor
Diffstat (limited to 'vendor/github.com/klauspost')
128 files changed, 0 insertions, 63395 deletions
diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes deleted file mode 100644 index 402433593..000000000 --- a/vendor/github.com/klauspost/compress/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -* -text -*.bin -text -diff diff --git a/vendor/github.com/klauspost/compress/.gitignore b/vendor/github.com/klauspost/compress/.gitignore deleted file mode 100644 index d31b37815..000000000 --- a/vendor/github.com/klauspost/compress/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof -/s2/cmd/_s2sx/sfx-exe - -# Linux perf files -perf.data -perf.data.old - -# gdb history -.gdb_history diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml deleted file mode 100644 index 4528059ca..000000000 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ /dev/null @@ -1,123 +0,0 @@ -version: 2 - -before: - hooks: - - ./gen.sh - -builds: - - - id: "s2c" - binary: s2c - main: ./s2/cmd/s2c/main.go - flags: - - -trimpath - env: - - CGO_ENABLED=0 - goos: - - aix - - linux - - freebsd - - netbsd - - windows - - darwin - goarch: - - 386 - - amd64 - - arm - - arm64 - - ppc64 - - ppc64le - - mips64 - - mips64le - goarm: - - 7 - - - id: "s2d" - binary: s2d - main: ./s2/cmd/s2d/main.go - flags: - - -trimpath - env: - - CGO_ENABLED=0 - goos: - - aix - - linux - - freebsd - - netbsd - - windows - - darwin - goarch: - - 386 - - amd64 - - arm - - arm64 - - ppc64 - - ppc64le - - mips64 - - mips64le - goarm: - - 7 - - - id: "s2sx" - binary: s2sx - main: ./s2/cmd/_s2sx/main.go - flags: - - -modfile=s2sx.mod - - -trimpath - env: - - CGO_ENABLED=0 - goos: - - aix - - linux - - freebsd - - netbsd - - windows - - darwin - goarch: - - 386 - - amd64 - - arm - - arm64 - - ppc64 - - ppc64le - - mips64 - - mips64le - goarm: - - 7 - -archives: - - - id: s2-binaries - name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" - format_overrides: - - goos: windows - format: zip - files: - - unpack/* - - s2/LICENSE - - s2/README.md -checksum: - name_template: 'checksums.txt' -snapshot: - version_template: "{{ .Tag }}-next" -changelog: - sort: asc - filters: - exclude: - - '^doc:' - - '^docs:' - - '^test:' - - '^tests:' - - '^Update\sREADME.md' - -nfpms: - - - file_name_template: "s2_package__{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" - vendor: Klaus Post - homepage: https://github.com/klauspost/compress - maintainer: Klaus Post <klauspost@gmail.com> - description: S2 Compression Tool - license: BSD 3-Clause - formats: - - deb - - rpm diff --git a/vendor/github.com/klauspost/compress/LICENSE b/vendor/github.com/klauspost/compress/LICENSE deleted file mode 100644 index 87d557477..000000000 --- a/vendor/github.com/klauspost/compress/LICENSE +++ /dev/null @@ -1,304 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2019 Klaus Post. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ------------------- - -Files: gzhttp/* - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016-2017 The New York Times Company - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------- - -Files: s2/cmd/internal/readahead/* - -The MIT License (MIT) - -Copyright (c) 2015 Klaus Post - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ---------------------- -Files: snappy/* -Files: internal/snapref/* - -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ------------------ - -Files: s2/cmd/internal/filepathx/* - -Copyright 2016 The filepathx Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md deleted file mode 100644 index de264c85a..000000000 --- a/vendor/github.com/klauspost/compress/README.md +++ /dev/null @@ -1,721 +0,0 @@ -# compress
-
-This package provides various compression algorithms.
-
-* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go.
-* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy.
-* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
-* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams.
-* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
-* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
-* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
-
-[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
-[](https://github.com/klauspost/compress/actions/workflows/go.yml)
-[](https://sourcegraph.com/github.com/klauspost/compress?badge)
-
-# changelog
-
-* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
- * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
- * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
- * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
- * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
- * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
-
-* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
- * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
- * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963
- * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971
- * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951
-
-* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8)
- * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885
- * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938
-
-* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7)
- * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927
- * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930
-
-* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6)
- * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923
- * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925
-
-* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5)
- * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912
- * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908
- * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913
- * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910
- * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917
-https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918
-
-* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4)
- * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887
- * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886
- * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892
- * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890
- * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891
-
-* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3)
- * fse: Fix max header size https://github.com/klauspost/compress/pull/881
- * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877
- * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883
-
-* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2)
- * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876
-
-* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1)
- * s2: Fix S2 "best" dictionary wrong encoding by @klauspost in https://github.com/klauspost/compress/pull/871
- * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869
- * s2: Fix EstimateBlockSize on 6&7 length input by @klauspost in https://github.com/klauspost/compress/pull/867
-
-* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
- * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853
- * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
- * flate: Add limited window compression https://github.com/klauspost/compress/pull/843
- * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
- * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
- * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
-
-<details>
- <summary>See changes to v1.16.x</summary>
-
-
-* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
- * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
- * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
-
-* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6)
- * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806
- * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824
- * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815
- * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663
-
-* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5)
- * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802
- * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804
-
-* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4)
- * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784
- * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792
- * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785
- * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795
- * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779
- * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780
- * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799
-
-* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1)
- * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776
- * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767
- * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766
- * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773
- * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774
-
-* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0)
- * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685
- * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752
- * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755
- * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748
- * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747
- * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746
-</details>
-
-<details>
- <summary>See changes to v1.15.x</summary>
-
-* Jan 21st, 2023 (v1.15.15)
- * deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739
- * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
- * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745
- * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740
-
-* Jan 3rd, 2023 (v1.15.14)
-
- * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718
- * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720
- * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722
- * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723
-
-* Dec 11, 2022 (v1.15.13)
- * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691
- * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708
-
-* Oct 26, 2022 (v1.15.12)
-
- * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680
- * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683
-
-* Sept 26, 2022 (v1.15.11)
-
- * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678
- * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677
- * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668
- * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667
-
-* Sept 16, 2022 (v1.15.10)
-
- * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649
- * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651
- * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656
- * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657
- * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658
- * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635
- * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646
- * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659
-
-* July 21, 2022 (v1.15.9)
-
- * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
- * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
- * zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643
-
-* July 13, 2022 (v1.15.8)
-
- * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641
- * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638
- * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636
- * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637
- * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634
- * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640
- * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639
-
-* June 29, 2022 (v1.15.7)
-
- * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633
- * zip: Merge upstream https://github.com/klauspost/compress/pull/631
- * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624
- * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598
- * flate: Faster histograms https://github.com/klauspost/compress/pull/620
- * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622
-
-* June 3, 2022 (v1.15.6)
- * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613
- * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611
- * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605
- * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606
- * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608
- * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612
- * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609
- * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607
- * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614
- * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610
-
-* May 25, 2022 (v1.15.5)
- * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602
- * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601
- * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596
- * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588
- * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592
- * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599
- * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593
- * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586
- * flate: Inplace hashing for level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/590
-
-
-* May 11, 2022 (v1.15.4)
- * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577)
- * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581)
- * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583)
- * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580)
-
-* May 5, 2022 (v1.15.3)
- * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572)
- * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575)
-
-* Apr 26, 2022 (v1.15.2)
- * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537)
- * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539)
- * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555)
- * Minimum version is Go 1.16, added CI test on 1.18.
-
-* Mar 11, 2022 (v1.15.1)
- * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512)
- * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514)
- * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520)
- * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521)
- * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523)
-
-* Mar 3, 2022 (v1.15.0)
- * zstd: Refactor decoder by @klauspost in [#498](https://github.com/klauspost/compress/pull/498)
- * zstd: Add stream encoding without goroutines by @klauspost in [#505](https://github.com/klauspost/compress/pull/505)
- * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507)
- * flate: Inline literal emission by @klauspost in [#509](https://github.com/klauspost/compress/pull/509)
- * gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400)
- * gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510)
-
-Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
-
-Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected.
-
-While the release has been extensively tested, it is recommended to testing when upgrading.
-
-</details>
-
-<details>
- <summary>See changes to v1.14.x</summary>
-
-* Feb 22, 2022 (v1.14.4)
- * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
- * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
- * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501
- * huff0: Use static decompression buffer up to 30% faster by @klauspost in [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
-
-* Feb 17, 2022 (v1.14.3)
- * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478)
- * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483)
- * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486)
-
-* Jan 25, 2022 (v1.14.2)
- * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476)
- * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469)
- * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470)
- * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472)
- * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473)
- * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475)
-
-* Jan 11, 2022 (v1.14.1)
- * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
- * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
- * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
- * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
- * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
-</details>
-
-<details>
- <summary>See changes to v1.13.x</summary>
-
-* Aug 30, 2021 (v1.13.5)
- * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
- * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
- * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426)
- * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421)
-
-* Aug 12, 2021 (v1.13.4)
- * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy).
- * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415)
-
-* Aug 3, 2021 (v1.13.3)
- * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404)
- * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411)
- * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406)
- * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399)
- * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401)
- * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410)
-
-* Jun 14, 2021 (v1.13.1)
- * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396)
- * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394)
- * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389)
- * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395)
-
-* Jun 3, 2021 (v1.13.0)
- * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
- * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
- * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
-</details>
-
-
-<details>
- <summary>See changes to v1.12.x</summary>
-
-* May 25, 2021 (v1.12.3)
- * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
- * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
- * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373)
-
-* Apr 27, 2021 (v1.12.2)
- * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365)
- * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363)
- * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367)
- * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358)
- * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362)
- * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368)
-
-* Apr 14, 2021 (v1.12.1)
- * snappy package removed. Upstream added as dependency.
- * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353)
- * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352)
- * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348)
- * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
- * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
- * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
-</details>
-
-<details>
- <summary>See changes to v1.11.x</summary>
-
-* Mar 26, 2021 (v1.11.13)
- * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
- * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336)
- * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338)
- * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341)
-
-* Mar 5, 2021 (v1.11.12)
- * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives).
- * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328)
-
-* Mar 1, 2021 (v1.11.9)
- * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324)
- * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325)
- * s2: Fix binaries.
-
-* Feb 25, 2021 (v1.11.8)
- * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended.
- * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
- * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
- * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
- * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313)
-
-* Jan 14, 2021 (v1.11.7)
- * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309)
- * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310)
- * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311)
- * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308)
- * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312)
-
-* Jan 7, 2021 (v1.11.6)
- * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306)
- * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305)
-
-* Dec 20, 2020 (v1.11.4)
- * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304)
- * Add header decoder [#299](https://github.com/klauspost/compress/pull/299)
- * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297)
- * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300)
- * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303)
-
-* Nov 15, 2020 (v1.11.3)
- * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293)
- * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295)
-
-* Oct 11, 2020 (v1.11.2)
- * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291)
-
-* Oct 1, 2020 (v1.11.1)
- * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286)
-
-* Sept 8, 2020 (v1.11.0)
- * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281)
- * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282)
- * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274)
-</details>
-
-<details>
- <summary>See changes to v1.10.x</summary>
-
-* July 8, 2020 (v1.10.11)
- * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
- * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275)
-
-* June 23, 2020 (v1.10.10)
- * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270)
-
-* June 16, 2020 (v1.10.9):
- * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268)
- * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266)
- * Fuzzit tests removed. The service has been purchased and is no longer available.
-
-* June 5, 2020 (v1.10.8):
- * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265)
-
-* June 1, 2020 (v1.10.7):
- * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries)
- * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259)
- * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263)
-
-* May 21, 2020: (v1.10.6)
- * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252)
- * zstd: Stricter decompression checks.
-
-* April 12, 2020: (v1.10.5)
- * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239)
-
-* Apr 8, 2020: (v1.10.4)
- * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247)
-* Mar 11, 2020: (v1.10.3)
- * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245)
- * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244)
- * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240)
- * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241)
- * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238)
-
-* Feb 27, 2020: (v1.10.2)
- * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232)
- * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227)
-
-* Feb 18, 2020: (v1.10.1)
- * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226)
- * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224)
- * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224)
-
-* Feb 4, 2020: (v1.10.0)
- * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216)
- * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218)
- * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214)
- * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186)
-
-</details>
-
-<details>
- <summary>See changes prior to v1.10.0</summary>
-
-* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206).
-* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204)
-* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed.
-* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases.
-* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192)
-* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder.
-* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199)
-* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features
-* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197)
-* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198)
-* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit.
-* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191)
-* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188)
-* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187)
-* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines.
-* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate.
-* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184)
-* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate.
-* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180)
-* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB.
-* Nov 11, 2019: Reduce inflate memory use by 1KB.
-* Nov 10, 2019: Less allocations in deflate bit writer.
-* Nov 10, 2019: Fix inconsistent error returned by zstd decoder.
-* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174)
-* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173)
-* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172)
-* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105)
-
-</details>
-
-<details>
- <summary>See changes prior to v1.9.0</summary>
-
-* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169)
-* Oct 3, 2019: Fix inconsistent results on broken zstd streams.
-* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools)
-* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools).
-* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip).
-* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes).
-* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option.
-* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables.
-* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode.
-* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding.
-* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy.
-* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing.
-* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing.
-* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147)
-* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146)
-* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144)
-* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142)
-* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder.
-* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder.
-* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content.
-* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix.
-* June 17, 2019: zstd decompression bugfix.
-* June 17, 2019: fix 32 bit builds.
-* June 17, 2019: Easier use in modules (less dependencies).
-* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio.
-* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression.
-* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels.
-* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression!
-* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels.
-* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added.
-* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
-* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
-* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
-* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
-* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
-* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
-* May 28, 2017: Reduce allocations when resetting decoder.
-* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
-* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
-* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
-* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
-* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level.
-* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression.
-* Mar 24, 2016: Small speedup for level 1-3.
-* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
-* Feb 19, 2016: Handle small payloads faster in level 1-3.
-* Feb 19, 2016: Added faster level 2 + 3 compression modes.
-* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5.
-* Feb 14, 2016: Snappy: Merge upstream changes.
-* Feb 14, 2016: Snappy: Fix aggressive skipping.
-* Feb 14, 2016: Snappy: Update benchmark.
-* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression.
-* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%.
-* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content.
-* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup.
-* Jan 16, 2016: Optimization on deflate level 1,2,3 compression.
-* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives.
-* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs.
-* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms.
-* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update!
-* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet).
-* Nov 20 2015: Small optimization to bit writer on 64 bit systems.
-* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15).
-* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate.
-* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file
-* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x.
-
-</details>
-
-# deflate usage
-
-The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
-
-| old import | new import | Documentation
-|--------------------|-----------------------------------------|--------------------|
-| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)
-| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)
-| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)
-| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc)
-
-* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
-
-You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
-
-The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
-
-Currently there is only minor speedup on decompression (mostly CRC32 calculation).
-
-Memory usage is typically 1MB for a Writer. stdlib is in the same range.
-If you expect to have a lot of concurrently allocated Writers consider using
-the stateless compress described below.
-
-For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
-
-To disable all assembly add `-tags=noasm`. This works across all packages.
-
-# Stateless compression
-
-This package offers stateless compression as a special option for gzip/deflate.
-It will do compression but without maintaining any state between Write calls.
-
-This means there will be no memory kept between Write calls, but compression and speed will be suboptimal.
-
-This is only relevant in cases where you expect to run many thousands of compressors concurrently,
-but with very little activity. This is *not* intended for regular web servers serving individual requests.
-
-Because of this, the size of actual Write calls will affect output size.
-
-In gzip, specify level `-3` / `gzip.StatelessCompression` to enable.
-
-For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter)
-
-A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
-
-```go
- // replace 'ioutil.Discard' with your output.
- gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
- if err != nil {
- return err
- }
- defer gzw.Close()
-
- w := bufio.NewWriterSize(gzw, 4096)
- defer w.Flush()
-
- // Write to 'w'
-```
-
-This will only use up to 4KB in memory when the writer is idle.
-
-Compression is almost always worse than the fastest compression level
-and each write will allocate (a little) memory.
-
-# Performance Update 2018
-
-It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD.
-
-The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet.
-
-The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard library at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input.
-
-The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/tree/master/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet).
-
-
-## Overall differences.
-
-There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels.
-
-The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library.
-
-This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression.
-
-There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab.
-
-## Web Content
-
-This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS.
-
-Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big.
-
-Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case.
-
-## Object files
-
-This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible.
-
-The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression.
-
-The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively.
-
-## Highly Compressible File
-
-This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc.
-
-It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression.
-
-So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground".
-
-## Medium-High Compressible
-
-This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams.
-
-We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both.
-
-## Medium Compressible
-
-I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario.
-
-The most notable thing is how quickly the standard library drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior.
-
-
-## Un-compressible Content
-
-This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections.
-
-
-## Huffman only compression
-
-This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
-
-This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM).
-
-Since this type of compression has much less variance, the compression speed is mostly unaffected by the input data, and is usually more than *180MB/s* for a single core.
-
-The downside is that the compression ratio is usually considerably worse than even the fastest conventional compression. The compression ratio can never be better than 8:1 (12.5%).
-
-The linear time compression can be used as a "better than nothing" mode, where you cannot risk the encoder to slow down on some content. For comparison, the size of the "Twain" text is *233460 bytes* (+29% vs. level 1) and encode speed is 144MB/s (4.5x level 1). So in this case you trade a 30% size increase for a 4 times speedup.
-
-For more information see my blog post on [Fast Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
-
-This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
-
-# Other packages
-
-Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code):
-
-* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression.
-* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression.
-* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
-* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
-* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
-* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
-* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
-
-# license
-
-This code is licensed under the same conditions as the original Go code. See LICENSE file.
diff --git a/vendor/github.com/klauspost/compress/SECURITY.md b/vendor/github.com/klauspost/compress/SECURITY.md deleted file mode 100644 index ca6685e2b..000000000 --- a/vendor/github.com/klauspost/compress/SECURITY.md +++ /dev/null @@ -1,25 +0,0 @@ -# Security Policy - -## Supported Versions - -Security updates are applied only to the latest release. - -## Vulnerability Definition - -A security vulnerability is a bug that with certain input triggers a crash or an infinite loop. Most calls will have varying execution time and only in rare cases will slow operation be considered a security vulnerability. - -Corrupted output generally is not considered a security vulnerability, unless independent operations are able to affect each other. Note that not all functionality is re-entrant and safe to use concurrently. - -Out-of-memory crashes only applies if the en/decoder uses an abnormal amount of memory, with appropriate options applied, to limit maximum window size, concurrency, etc. However, if you are in doubt you are welcome to file a security issue. - -It is assumed that all callers are trusted, meaning internal data exposed through reflection or inspection of returned data structures is not considered a vulnerability. - -Vulnerabilities resulting from compiler/assembler errors should be reported upstream. Depending on the severity this package may or may not implement a workaround. - -## Reporting a Vulnerability - -If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. - -Please disclose it at [security advisory](https://github.com/klauspost/compress/security/advisories/new). If possible please provide a minimal reproducer. If the issue only applies to a single platform, it would be helpful to provide access to that. - -This project is maintained by a team of volunteers on a reasonable-effort basis. As such, vulnerabilities will be disclosed in a best effort base. diff --git a/vendor/github.com/klauspost/compress/compressible.go b/vendor/github.com/klauspost/compress/compressible.go deleted file mode 100644 index ea5a692d5..000000000 --- a/vendor/github.com/klauspost/compress/compressible.go +++ /dev/null @@ -1,85 +0,0 @@ -package compress - -import "math" - -// Estimate returns a normalized compressibility estimate of block b. -// Values close to zero are likely uncompressible. -// Values above 0.1 are likely to be compressible. -// Values above 0.5 are very compressible. -// Very small lengths will return 0. -func Estimate(b []byte) float64 { - if len(b) < 16 { - return 0 - } - - // Correctly predicted order 1 - hits := 0 - lastMatch := false - var o1 [256]byte - var hist [256]int - c1 := byte(0) - for _, c := range b { - if c == o1[c1] { - // We only count a hit if there was two correct predictions in a row. - if lastMatch { - hits++ - } - lastMatch = true - } else { - lastMatch = false - } - o1[c1] = c - c1 = c - hist[c]++ - } - - // Use x^0.6 to give better spread - prediction := math.Pow(float64(hits)/float64(len(b)), 0.6) - - // Calculate histogram distribution - variance := float64(0) - avg := float64(len(b)) / 256 - - for _, v := range hist { - Δ := float64(v) - avg - variance += Δ * Δ - } - - stddev := math.Sqrt(float64(variance)) / float64(len(b)) - exp := math.Sqrt(1 / float64(len(b))) - - // Subtract expected stddev - stddev -= exp - if stddev < 0 { - stddev = 0 - } - stddev *= 1 + exp - - // Use x^0.4 to give better spread - entropy := math.Pow(stddev, 0.4) - - // 50/50 weight between prediction and histogram distribution - return math.Pow((prediction+entropy)/2, 0.9) -} - -// ShannonEntropyBits returns the number of bits minimum required to represent -// an entropy encoding of the input bytes. -// https://en.wiktionary.org/wiki/Shannon_entropy -func ShannonEntropyBits(b []byte) int { - if len(b) == 0 { - return 0 - } - var hist [256]int - for _, c := range b { - hist[c]++ - } - shannon := float64(0) - invTotal := 1.0 / float64(len(b)) - for _, v := range hist[:] { - if v > 0 { - n := float64(v) - shannon += math.Ceil(-math.Log2(n*invTotal) * n) - } - } - return int(math.Ceil(shannon)) -} diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md deleted file mode 100644 index ea7324da6..000000000 --- a/vendor/github.com/klauspost/compress/fse/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Finite State Entropy
-
-This package provides Finite State Entropy encoding and decoding.
-
-Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS))
-encoding provides a fast near-optimal symbol encoding/decoding
-for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
-
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
-
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
-
-## News
-
- * Feb 2018: First implementation released. Consider this beta software for now.
-
-# Usage
-
-This package provides a low level interface that allows to compress single independent blocks.
-
-Each block is separate, and there is no built in integrity checks.
-This means that the caller should keep track of block sizes and also do checksums if needed.
-
-Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
-You must provide input and will receive the output and maybe an error.
-
-These error values can be returned:
-
-| Error | Description |
-|---------------------|-----------------------------------------------------------------------------|
-| `<nil>` | Everything ok, output is returned |
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
-| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
-| `(error)` | An internal error occurred. |
-
-As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
-
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
-object can be used for both.
-
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
-
-Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
-your input was likely corrupted.
-
-It is important to note that a successful decoding does *not* mean your output matches your original input.
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
-
-For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
-
-# Performance
-
-A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.
-All compression functions are currently only running on the calling goroutine so only one core will be used per block.
-
-The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
-is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be
-beneficial to transpose all your input values down by 64.
-
-With moderate block sizes around 64k speed are typically 200MB/s per core for compression and
-around 300MB/s decompression speed.
-
-The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s.
-
-# Plans
-
-At one point, more internals will be exposed to facilitate more "expert" usage of the components.
-
-A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).
-
-# Contributing
-
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
-changes will likely not be accepted. If in doubt open an issue before writing the PR.
\ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/fse/bitreader.go b/vendor/github.com/klauspost/compress/fse/bitreader.go deleted file mode 100644 index f65eb3909..000000000 --- a/vendor/github.com/klauspost/compress/fse/bitreader.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package fse - -import ( - "encoding/binary" - "errors" - "io" -) - -// bitReader reads a bitstream in reverse. -// The last set bit indicates the start of the stream and is used -// for aligning the input. -type bitReader struct { - in []byte - off uint // next byte to read is at in[off - 1] - value uint64 - bitsRead uint8 -} - -// init initializes and resets the bit reader. -func (b *bitReader) init(in []byte) error { - if len(in) < 1 { - return errors.New("corrupt stream: too short") - } - b.in = in - b.off = uint(len(in)) - // The highest bit of the last byte indicates where to start - v := in[len(in)-1] - if v == 0 { - return errors.New("corrupt stream, did not find end of stream") - } - b.bitsRead = 64 - b.value = 0 - if len(in) >= 8 { - b.fillFastStart() - } else { - b.fill() - b.fill() - } - b.bitsRead += 8 - uint8(highBits(uint32(v))) - return nil -} - -// getBits will return n bits. n can be 0. -func (b *bitReader) getBits(n uint8) uint16 { - if n == 0 || b.bitsRead >= 64 { - return 0 - } - return b.getBitsFast(n) -} - -// getBitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReader) getBitsFast(n uint8) uint16 { - const regMask = 64 - 1 - v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) - b.bitsRead += n - return v -} - -// fillFast() will make sure at least 32 bits are available. -// There must be at least 4 bytes available. -func (b *bitReader) fillFast() { - if b.bitsRead < 32 { - return - } - // 2 bounds checks. - v := b.in[b.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 - b.off -= 4 -} - -// fill() will make sure at least 32 bits are available. -func (b *bitReader) fill() { - if b.bitsRead < 32 { - return - } - if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 - b.off -= 4 - return - } - for b.off > 0 { - b.value = (b.value << 8) | uint64(b.in[b.off-1]) - b.bitsRead -= 8 - b.off-- - } -} - -// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. -func (b *bitReader) fillFastStart() { - // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) - b.bitsRead = 0 - b.off -= 8 -} - -// finished returns true if all bits have been read from the bit stream. -func (b *bitReader) finished() bool { - return b.bitsRead >= 64 && b.off == 0 -} - -// close the bitstream and returns an error if out-of-buffer reads occurred. -func (b *bitReader) close() error { - // Release reference. - b.in = nil - if b.bitsRead > 64 { - return io.ErrUnexpectedEOF - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/fse/bitwriter.go b/vendor/github.com/klauspost/compress/fse/bitwriter.go deleted file mode 100644 index e82fa3bb7..000000000 --- a/vendor/github.com/klauspost/compress/fse/bitwriter.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package fse - -import "fmt" - -// bitWriter will write bits. -// First bit will be LSB of the first byte of output. -type bitWriter struct { - bitContainer uint64 - nBits uint8 - out []byte -} - -// bitMask16 is bitmasks. Has extra to avoid bounds check. -var bitMask16 = [32]uint16{ - 0, 1, 3, 7, 0xF, 0x1F, - 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, - 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF, - 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, - 0xFFFF, 0xFFFF} /* up to 16 bits */ - -// addBits16NC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16NC(value uint16, bits uint8) { - b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) - b.nBits += bits -} - -// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// addBits16ZeroNC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -// This is fastest if bits can be zero. -func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) { - if bits == 0 { - return - } - value <<= (16 - bits) & 15 - value >>= (16 - bits) & 15 - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// flush will flush all pending full bytes. -// There will be at least 56 bits available for writing when this has been called. -// Using flush32 is faster, but leaves less space for writing. -func (b *bitWriter) flush() { - v := b.nBits >> 3 - switch v { - case 0: - case 1: - b.out = append(b.out, - byte(b.bitContainer), - ) - case 2: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - ) - case 3: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - ) - case 4: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - ) - case 5: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - ) - case 6: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - ) - case 7: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - ) - case 8: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - byte(b.bitContainer>>56), - ) - default: - panic(fmt.Errorf("bits (%d) > 64", b.nBits)) - } - b.bitContainer >>= v << 3 - b.nBits &= 7 -} - -// flush32 will flush out, so there are at least 32 bits available for writing. -func (b *bitWriter) flush32() { - if b.nBits < 32 { - return - } - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24)) - b.nBits -= 32 - b.bitContainer >>= 32 -} - -// flushAlign will flush remaining full bytes and align to next byte boundary. -func (b *bitWriter) flushAlign() { - nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { - b.out = append(b.out, byte(b.bitContainer>>(i*8))) - } - b.nBits = 0 - b.bitContainer = 0 -} - -// close will write the alignment bit and write the final byte(s) -// to the output. -func (b *bitWriter) close() { - // End mark - b.addBits16Clean(1, 1) - // flush until next byte. - b.flushAlign() -} - -// reset and continue writing by appending to out. -func (b *bitWriter) reset(out []byte) { - b.bitContainer = 0 - b.nBits = 0 - b.out = out -} diff --git a/vendor/github.com/klauspost/compress/fse/bytereader.go b/vendor/github.com/klauspost/compress/fse/bytereader.go deleted file mode 100644 index abade2d60..000000000 --- a/vendor/github.com/klauspost/compress/fse/bytereader.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package fse - -// byteReader provides a byte reader that reads -// little endian values from a byte stream. -// The input stream is manually advanced. -// The reader performs no bounds checks. -type byteReader struct { - b []byte - off int -} - -// init will initialize the reader and set the input. -func (b *byteReader) init(in []byte) { - b.b = in - b.off = 0 -} - -// advance the stream b n bytes. -func (b *byteReader) advance(n uint) { - b.off += int(n) -} - -// Uint32 returns a little endian uint32 starting at current offset. -func (b byteReader) Uint32() uint32 { - b2 := b.b[b.off:] - b2 = b2[:4] - v3 := uint32(b2[3]) - v2 := uint32(b2[2]) - v1 := uint32(b2[1]) - v0 := uint32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) -} - -// unread returns the unread portion of the input. -func (b byteReader) unread() []byte { - return b.b[b.off:] -} - -// remain will return the number of bytes remaining. -func (b byteReader) remain() int { - return len(b.b) - b.off -} diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go deleted file mode 100644 index 074018d8f..000000000 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package fse - -import ( - "errors" - "fmt" -) - -// Compress the input bytes. Input must be < 2GB. -// Provide a Scratch buffer to avoid memory allocations. -// Note that the output is also kept in the scratch buffer. -// If input is too hard to compress, ErrIncompressible is returned. -// If input is a single byte value repeated ErrUseRLE is returned. -func Compress(in []byte, s *Scratch) ([]byte, error) { - if len(in) <= 1 { - return nil, ErrIncompressible - } - if len(in) > (2<<30)-1 { - return nil, errors.New("input too big, must be < 2GB") - } - s, err := s.prepare(in) - if err != nil { - return nil, err - } - - // Create histogram, if none was provided. - maxCount := s.maxCount - if maxCount == 0 { - maxCount = s.countSimple(in) - } - // Reset for next run. - s.clearCount = true - s.maxCount = 0 - if maxCount == len(in) { - // One symbol, use RLE - return nil, ErrUseRLE - } - if maxCount == 1 || maxCount < (len(in)>>7) { - // Each symbol present maximum once or too well distributed. - return nil, ErrIncompressible - } - s.optimalTableLog() - err = s.normalizeCount() - if err != nil { - return nil, err - } - err = s.writeCount() - if err != nil { - return nil, err - } - - if false { - err = s.validateNorm() - if err != nil { - return nil, err - } - } - - err = s.buildCTable() - if err != nil { - return nil, err - } - err = s.compress(in) - if err != nil { - return nil, err - } - s.Out = s.bw.out - // Check if we compressed. - if len(s.Out) >= len(in) { - return nil, ErrIncompressible - } - return s.Out, nil -} - -// cState contains the compression state of a stream. -type cState struct { - bw *bitWriter - stateTable []uint16 - state uint16 -} - -// init will initialize the compression state to the first symbol of the stream. -func (c *cState) init(bw *bitWriter, ct *cTable, tableLog uint8, first symbolTransform) { - c.bw = bw - c.stateTable = ct.stateTable - - nbBitsOut := (first.deltaNbBits + (1 << 15)) >> 16 - im := int32((nbBitsOut << 16) - first.deltaNbBits) - lu := (im >> nbBitsOut) + first.deltaFindState - c.state = c.stateTable[lu] -} - -// encode the output symbol provided and write it to the bitstream. -func (c *cState) encode(symbolTT symbolTransform) { - nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 - dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState - c.bw.addBits16NC(c.state, uint8(nbBitsOut)) - c.state = c.stateTable[dstState] -} - -// encode the output symbol provided and write it to the bitstream. -func (c *cState) encodeZero(symbolTT symbolTransform) { - nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 - dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState - c.bw.addBits16ZeroNC(c.state, uint8(nbBitsOut)) - c.state = c.stateTable[dstState] -} - -// flush will write the tablelog to the output and flush the remaining full bytes. -func (c *cState) flush(tableLog uint8) { - c.bw.flush32() - c.bw.addBits16NC(c.state, tableLog) - c.bw.flush() -} - -// compress is the main compression loop that will encode the input from the last byte to the first. -func (s *Scratch) compress(src []byte) error { - if len(src) <= 2 { - return errors.New("compress: src too small") - } - tt := s.ct.symbolTT[:256] - s.bw.reset(s.Out) - - // Our two states each encodes every second byte. - // Last byte encoded (first byte decoded) will always be encoded by c1. - var c1, c2 cState - - // Encode so remaining size is divisible by 4. - ip := len(src) - if ip&1 == 1 { - c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]]) - c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]]) - c1.encodeZero(tt[src[ip-3]]) - ip -= 3 - } else { - c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]]) - c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]]) - ip -= 2 - } - if ip&2 != 0 { - c2.encodeZero(tt[src[ip-1]]) - c1.encodeZero(tt[src[ip-2]]) - ip -= 2 - } - src = src[:ip] - - // Main compression loop. - switch { - case !s.zeroBits && s.actualTableLog <= 8: - // We can encode 4 symbols without requiring a flush. - // We do not need to check if any output is 0 bits. - for ; len(src) >= 4; src = src[:len(src)-4] { - s.bw.flush32() - v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] - c2.encode(tt[v0]) - c1.encode(tt[v1]) - c2.encode(tt[v2]) - c1.encode(tt[v3]) - } - case !s.zeroBits: - // We do not need to check if any output is 0 bits. - for ; len(src) >= 4; src = src[:len(src)-4] { - s.bw.flush32() - v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] - c2.encode(tt[v0]) - c1.encode(tt[v1]) - s.bw.flush32() - c2.encode(tt[v2]) - c1.encode(tt[v3]) - } - case s.actualTableLog <= 8: - // We can encode 4 symbols without requiring a flush - for ; len(src) >= 4; src = src[:len(src)-4] { - s.bw.flush32() - v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] - c2.encodeZero(tt[v0]) - c1.encodeZero(tt[v1]) - c2.encodeZero(tt[v2]) - c1.encodeZero(tt[v3]) - } - default: - for ; len(src) >= 4; src = src[:len(src)-4] { - s.bw.flush32() - v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] - c2.encodeZero(tt[v0]) - c1.encodeZero(tt[v1]) - s.bw.flush32() - c2.encodeZero(tt[v2]) - c1.encodeZero(tt[v3]) - } - } - - // Flush final state. - // Used to initialize state when decoding. - c2.flush(s.actualTableLog) - c1.flush(s.actualTableLog) - - s.bw.close() - return nil -} - -// writeCount will write the normalized histogram count to header. -// This is read back by readNCount. -func (s *Scratch) writeCount() error { - var ( - tableLog = s.actualTableLog - tableSize = 1 << tableLog - previous0 bool - charnum uint16 - - maxHeaderSize = ((int(s.symbolLen)*int(tableLog) + 4 + 2) >> 3) + 3 - - // Write Table Size - bitStream = uint32(tableLog - minTablelog) - bitCount = uint(4) - remaining = int16(tableSize + 1) /* +1 for extra accuracy */ - threshold = int16(tableSize) - nbBits = uint(tableLog + 1) - ) - if cap(s.Out) < maxHeaderSize { - s.Out = make([]byte, 0, s.br.remain()+maxHeaderSize) - } - outP := uint(0) - out := s.Out[:maxHeaderSize] - - // stops at 1 - for remaining > 1 { - if previous0 { - start := charnum - for s.norm[charnum] == 0 { - charnum++ - } - for charnum >= start+24 { - start += 24 - bitStream += uint32(0xFFFF) << bitCount - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - } - for charnum >= start+3 { - start += 3 - bitStream += 3 << bitCount - bitCount += 2 - } - bitStream += uint32(charnum-start) << bitCount - bitCount += 2 - if bitCount > 16 { - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - bitCount -= 16 - } - } - - count := s.norm[charnum] - charnum++ - max := (2*threshold - 1) - remaining - if count < 0 { - remaining += count - } else { - remaining -= count - } - count++ // +1 for extra accuracy - if count >= threshold { - count += max // [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ - } - bitStream += uint32(count) << bitCount - bitCount += nbBits - if count < max { - bitCount-- - } - - previous0 = count == 1 - if remaining < 1 { - return errors.New("internal error: remaining<1") - } - for remaining < threshold { - nbBits-- - threshold >>= 1 - } - - if bitCount > 16 { - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - bitCount -= 16 - } - } - - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += (bitCount + 7) / 8 - - if charnum > s.symbolLen { - return errors.New("internal error: charnum > s.symbolLen") - } - s.Out = out[:outP] - return nil -} - -// symbolTransform contains the state transform for a symbol. -type symbolTransform struct { - deltaFindState int32 - deltaNbBits uint32 -} - -// String prints values as a human readable string. -func (s symbolTransform) String() string { - return fmt.Sprintf("dnbits: %08x, fs:%d", s.deltaNbBits, s.deltaFindState) -} - -// cTable contains tables used for compression. -type cTable struct { - tableSymbol []byte - stateTable []uint16 - symbolTT []symbolTransform -} - -// allocCtable will allocate tables needed for compression. -// If existing tables a re big enough, they are simply re-used. -func (s *Scratch) allocCtable() { - tableSize := 1 << s.actualTableLog - // get tableSymbol that is big enough. - if cap(s.ct.tableSymbol) < tableSize { - s.ct.tableSymbol = make([]byte, tableSize) - } - s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] - - ctSize := tableSize - if cap(s.ct.stateTable) < ctSize { - s.ct.stateTable = make([]uint16, ctSize) - } - s.ct.stateTable = s.ct.stateTable[:ctSize] - - if cap(s.ct.symbolTT) < 256 { - s.ct.symbolTT = make([]symbolTransform, 256) - } - s.ct.symbolTT = s.ct.symbolTT[:256] -} - -// buildCTable will populate the compression table so it is ready to be used. -func (s *Scratch) buildCTable() error { - tableSize := uint32(1 << s.actualTableLog) - highThreshold := tableSize - 1 - var cumul [maxSymbolValue + 2]int16 - - s.allocCtable() - tableSymbol := s.ct.tableSymbol[:tableSize] - // symbol start positions - { - cumul[0] = 0 - for ui, v := range s.norm[:s.symbolLen-1] { - u := byte(ui) // one less than reference - if v == -1 { - // Low proba symbol - cumul[u+1] = cumul[u] + 1 - tableSymbol[highThreshold] = u - highThreshold-- - } else { - cumul[u+1] = cumul[u] + v - } - } - // Encode last symbol separately to avoid overflowing u - u := int(s.symbolLen - 1) - v := s.norm[s.symbolLen-1] - if v == -1 { - // Low proba symbol - cumul[u+1] = cumul[u] + 1 - tableSymbol[highThreshold] = byte(u) - highThreshold-- - } else { - cumul[u+1] = cumul[u] + v - } - if uint32(cumul[s.symbolLen]) != tableSize { - return fmt.Errorf("internal error: expected cumul[s.symbolLen] (%d) == tableSize (%d)", cumul[s.symbolLen], tableSize) - } - cumul[s.symbolLen] = int16(tableSize) + 1 - } - // Spread symbols - s.zeroBits = false - { - step := tableStep(tableSize) - tableMask := tableSize - 1 - var position uint32 - // if any symbol > largeLimit, we may have 0 bits output. - largeLimit := int16(1 << (s.actualTableLog - 1)) - for ui, v := range s.norm[:s.symbolLen] { - symbol := byte(ui) - if v > largeLimit { - s.zeroBits = true - } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { - tableSymbol[position] = symbol - position = (position + step) & tableMask - for position > highThreshold { - position = (position + step) & tableMask - } /* Low proba area */ - } - } - - // Check if we have gone through all positions - if position != 0 { - return errors.New("position!=0") - } - } - - // Build table - table := s.ct.stateTable - { - tsi := int(tableSize) - for u, v := range tableSymbol { - // TableU16 : sorted by symbol order; gives next state value - table[cumul[v]] = uint16(tsi + u) - cumul[v]++ - } - } - - // Build Symbol Transformation Table - { - total := int16(0) - symbolTT := s.ct.symbolTT[:s.symbolLen] - tableLog := s.actualTableLog - tl := (uint32(tableLog) << 16) - (1 << tableLog) - for i, v := range s.norm[:s.symbolLen] { - switch v { - case 0: - case -1, 1: - symbolTT[i].deltaNbBits = tl - symbolTT[i].deltaFindState = int32(total - 1) - total++ - default: - maxBitsOut := uint32(tableLog) - highBits(uint32(v-1)) - minStatePlus := uint32(v) << maxBitsOut - symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus - symbolTT[i].deltaFindState = int32(total - v) - total += v - } - } - if total != int16(tableSize) { - return fmt.Errorf("total mismatch %d (got) != %d (want)", total, tableSize) - } - } - return nil -} - -// countSimple will create a simple histogram in s.count. -// Returns the biggest count. -// Does not update s.clearCount. -func (s *Scratch) countSimple(in []byte) (max int) { - for _, v := range in { - s.count[v]++ - } - m, symlen := uint32(0), s.symbolLen - for i, v := range s.count[:] { - if v == 0 { - continue - } - if v > m { - m = v - } - symlen = uint16(i) + 1 - } - s.symbolLen = symlen - return int(m) -} - -// minTableLog provides the minimum logSize to safely represent a distribution. -func (s *Scratch) minTableLog() uint8 { - minBitsSrc := highBits(uint32(s.br.remain()-1)) + 1 - minBitsSymbols := highBits(uint32(s.symbolLen-1)) + 2 - if minBitsSrc < minBitsSymbols { - return uint8(minBitsSrc) - } - return uint8(minBitsSymbols) -} - -// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog -func (s *Scratch) optimalTableLog() { - tableLog := s.TableLog - minBits := s.minTableLog() - maxBitsSrc := uint8(highBits(uint32(s.br.remain()-1))) - 2 - if maxBitsSrc < tableLog { - // Accuracy can be reduced - tableLog = maxBitsSrc - } - if minBits > tableLog { - tableLog = minBits - } - // Need a minimum to safely represent all symbol values - if tableLog < minTablelog { - tableLog = minTablelog - } - if tableLog > maxTableLog { - tableLog = maxTableLog - } - s.actualTableLog = tableLog -} - -var rtbTable = [...]uint32{0, 473195, 504333, 520860, 550000, 700000, 750000, 830000} - -// normalizeCount will normalize the count of the symbols so -// the total is equal to the table size. -func (s *Scratch) normalizeCount() error { - var ( - tableLog = s.actualTableLog - scale = 62 - uint64(tableLog) - step = (1 << 62) / uint64(s.br.remain()) - vStep = uint64(1) << (scale - 20) - stillToDistribute = int16(1 << tableLog) - largest int - largestP int16 - lowThreshold = (uint32)(s.br.remain() >> tableLog) - ) - - for i, cnt := range s.count[:s.symbolLen] { - // already handled - // if (count[s] == s.length) return 0; /* rle special case */ - - if cnt == 0 { - s.norm[i] = 0 - continue - } - if cnt <= lowThreshold { - s.norm[i] = -1 - stillToDistribute-- - } else { - proba := (int16)((uint64(cnt) * step) >> scale) - if proba < 8 { - restToBeat := vStep * uint64(rtbTable[proba]) - v := uint64(cnt)*step - (uint64(proba) << scale) - if v > restToBeat { - proba++ - } - } - if proba > largestP { - largestP = proba - largest = i - } - s.norm[i] = proba - stillToDistribute -= proba - } - } - - if -stillToDistribute >= (s.norm[largest] >> 1) { - // corner case, need another normalization method - return s.normalizeCount2() - } - s.norm[largest] += stillToDistribute - return nil -} - -// Secondary normalization method. -// To be used when primary method fails. -func (s *Scratch) normalizeCount2() error { - const notYetAssigned = -2 - var ( - distributed uint32 - total = uint32(s.br.remain()) - tableLog = s.actualTableLog - lowThreshold = total >> tableLog - lowOne = (total * 3) >> (tableLog + 1) - ) - for i, cnt := range s.count[:s.symbolLen] { - if cnt == 0 { - s.norm[i] = 0 - continue - } - if cnt <= lowThreshold { - s.norm[i] = -1 - distributed++ - total -= cnt - continue - } - if cnt <= lowOne { - s.norm[i] = 1 - distributed++ - total -= cnt - continue - } - s.norm[i] = notYetAssigned - } - toDistribute := (1 << tableLog) - distributed - - if (total / toDistribute) > lowOne { - // risk of rounding to zero - lowOne = (total * 3) / (toDistribute * 2) - for i, cnt := range s.count[:s.symbolLen] { - if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { - s.norm[i] = 1 - distributed++ - total -= cnt - continue - } - } - toDistribute = (1 << tableLog) - distributed - } - if distributed == uint32(s.symbolLen)+1 { - // all values are pretty poor; - // probably incompressible data (should have already been detected); - // find max, then give all remaining points to max - var maxV int - var maxC uint32 - for i, cnt := range s.count[:s.symbolLen] { - if cnt > maxC { - maxV = i - maxC = cnt - } - } - s.norm[maxV] += int16(toDistribute) - return nil - } - - if total == 0 { - // all of the symbols were low enough for the lowOne or lowThreshold - for i := uint32(0); toDistribute > 0; i = (i + 1) % (uint32(s.symbolLen)) { - if s.norm[i] > 0 { - toDistribute-- - s.norm[i]++ - } - } - return nil - } - - var ( - vStepLog = 62 - uint64(tableLog) - mid = uint64((1 << (vStepLog - 1)) - 1) - rStep = (((1 << vStepLog) * uint64(toDistribute)) + mid) / uint64(total) // scale on remaining - tmpTotal = mid - ) - for i, cnt := range s.count[:s.symbolLen] { - if s.norm[i] == notYetAssigned { - var ( - end = tmpTotal + uint64(cnt)*rStep - sStart = uint32(tmpTotal >> vStepLog) - sEnd = uint32(end >> vStepLog) - weight = sEnd - sStart - ) - if weight < 1 { - return errors.New("weight < 1") - } - s.norm[i] = int16(weight) - tmpTotal = end - } - } - return nil -} - -// validateNorm validates the normalized histogram table. -func (s *Scratch) validateNorm() (err error) { - var total int - for _, v := range s.norm[:s.symbolLen] { - if v >= 0 { - total += int(v) - } else { - total -= int(v) - } - } - defer func() { - if err == nil { - return - } - fmt.Printf("selected TableLog: %d, Symbol length: %d\n", s.actualTableLog, s.symbolLen) - for i, v := range s.norm[:s.symbolLen] { - fmt.Printf("%3d: %5d -> %4d \n", i, s.count[i], v) - } - }() - if total != (1 << s.actualTableLog) { - return fmt.Errorf("warning: Total == %d != %d", total, 1<<s.actualTableLog) - } - for i, v := range s.count[s.symbolLen:] { - if v != 0 { - return fmt.Errorf("warning: Found symbol out of range, %d after cut", i) - } - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go deleted file mode 100644 index 0c7dd4ffe..000000000 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ /dev/null @@ -1,376 +0,0 @@ -package fse - -import ( - "errors" - "fmt" -) - -const ( - tablelogAbsoluteMax = 15 -) - -// Decompress a block of data. -// You can provide a scratch buffer to avoid allocations. -// If nil is provided a temporary one will be allocated. -// It is possible, but by no way guaranteed that corrupt data will -// return an error. -// It is up to the caller to verify integrity of the returned data. -// Use a predefined Scratch to set maximum acceptable output size. -func Decompress(b []byte, s *Scratch) ([]byte, error) { - s, err := s.prepare(b) - if err != nil { - return nil, err - } - s.Out = s.Out[:0] - err = s.readNCount() - if err != nil { - return nil, err - } - err = s.buildDtable() - if err != nil { - return nil, err - } - err = s.decompress() - if err != nil { - return nil, err - } - - return s.Out, nil -} - -// readNCount will read the symbol distribution so decoding tables can be constructed. -func (s *Scratch) readNCount() error { - var ( - charnum uint16 - previous0 bool - b = &s.br - ) - iend := b.remain() - if iend < 4 { - return errors.New("input too small") - } - bitStream := b.Uint32() - nbBits := uint((bitStream & 0xF) + minTablelog) // extract tableLog - if nbBits > tablelogAbsoluteMax { - return errors.New("tableLog too large") - } - bitStream >>= 4 - bitCount := uint(4) - - s.actualTableLog = uint8(nbBits) - remaining := int32((1 << nbBits) + 1) - threshold := int32(1 << nbBits) - gotTotal := int32(0) - nbBits++ - - for remaining > 1 { - if previous0 { - n0 := charnum - for (bitStream & 0xFFFF) == 0xFFFF { - n0 += 24 - if b.off < iend-5 { - b.advance(2) - bitStream = b.Uint32() >> bitCount - } else { - bitStream >>= 16 - bitCount += 16 - } - } - for (bitStream & 3) == 3 { - n0 += 3 - bitStream >>= 2 - bitCount += 2 - } - n0 += uint16(bitStream & 3) - bitCount += 2 - if n0 > maxSymbolValue { - return errors.New("maxSymbolValue too small") - } - for charnum < n0 { - s.norm[charnum&0xff] = 0 - charnum++ - } - - if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 { - b.advance(bitCount >> 3) - bitCount &= 7 - bitStream = b.Uint32() >> bitCount - } else { - bitStream >>= 2 - } - } - - max := (2*(threshold) - 1) - (remaining) - var count int32 - - if (int32(bitStream) & (threshold - 1)) < max { - count = int32(bitStream) & (threshold - 1) - bitCount += nbBits - 1 - } else { - count = int32(bitStream) & (2*threshold - 1) - if count >= threshold { - count -= max - } - bitCount += nbBits - } - - count-- // extra accuracy - if count < 0 { - // -1 means +1 - remaining += count - gotTotal -= count - } else { - remaining -= count - gotTotal += count - } - s.norm[charnum&0xff] = int16(count) - charnum++ - previous0 = count == 0 - for remaining < threshold { - nbBits-- - threshold >>= 1 - } - if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 { - b.advance(bitCount >> 3) - bitCount &= 7 - } else { - bitCount -= (uint)(8 * (len(b.b) - 4 - b.off)) - b.off = len(b.b) - 4 - } - bitStream = b.Uint32() >> (bitCount & 31) - } - s.symbolLen = charnum - - if s.symbolLen <= 1 { - return fmt.Errorf("symbolLen (%d) too small", s.symbolLen) - } - if s.symbolLen > maxSymbolValue+1 { - return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) - } - if remaining != 1 { - return fmt.Errorf("corruption detected (remaining %d != 1)", remaining) - } - if bitCount > 32 { - return fmt.Errorf("corruption detected (bitCount %d > 32)", bitCount) - } - if gotTotal != 1<<s.actualTableLog { - return fmt.Errorf("corruption detected (total %d != %d)", gotTotal, 1<<s.actualTableLog) - } - b.advance((bitCount + 7) >> 3) - return nil -} - -// decSymbol contains information about a state entry, -// Including the state offset base, the output symbol and -// the number of bits to read for the low part of the destination state. -type decSymbol struct { - newState uint16 - symbol uint8 - nbBits uint8 -} - -// allocDtable will allocate decoding tables if they are not big enough. -func (s *Scratch) allocDtable() { - tableSize := 1 << s.actualTableLog - if cap(s.decTable) < tableSize { - s.decTable = make([]decSymbol, tableSize) - } - s.decTable = s.decTable[:tableSize] - - if cap(s.ct.tableSymbol) < 256 { - s.ct.tableSymbol = make([]byte, 256) - } - s.ct.tableSymbol = s.ct.tableSymbol[:256] - - if cap(s.ct.stateTable) < 256 { - s.ct.stateTable = make([]uint16, 256) - } - s.ct.stateTable = s.ct.stateTable[:256] -} - -// buildDtable will build the decoding table. -func (s *Scratch) buildDtable() error { - tableSize := uint32(1 << s.actualTableLog) - highThreshold := tableSize - 1 - s.allocDtable() - symbolNext := s.ct.stateTable[:256] - - // Init, lay down lowprob symbols - s.zeroBits = false - { - largeLimit := int16(1 << (s.actualTableLog - 1)) - for i, v := range s.norm[:s.symbolLen] { - if v == -1 { - s.decTable[highThreshold].symbol = uint8(i) - highThreshold-- - symbolNext[i] = 1 - } else { - if v >= largeLimit { - s.zeroBits = true - } - symbolNext[i] = uint16(v) - } - } - } - // Spread symbols - { - tableMask := tableSize - 1 - step := tableStep(tableSize) - position := uint32(0) - for ss, v := range s.norm[:s.symbolLen] { - for i := 0; i < int(v); i++ { - s.decTable[position].symbol = uint8(ss) - position = (position + step) & tableMask - for position > highThreshold { - // lowprob area - position = (position + step) & tableMask - } - } - } - if position != 0 { - // position must reach all cells once, otherwise normalizedCounter is incorrect - return errors.New("corrupted input (position != 0)") - } - } - - // Build Decoding table - { - tableSize := uint16(1 << s.actualTableLog) - for u, v := range s.decTable { - symbol := v.symbol - nextState := symbolNext[symbol] - symbolNext[symbol] = nextState + 1 - nBits := s.actualTableLog - byte(highBits(uint32(nextState))) - s.decTable[u].nbBits = nBits - newState := (nextState << nBits) - tableSize - if newState >= tableSize { - return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) - } - if newState == uint16(u) && nBits == 0 { - // Seems weird that this is possible with nbits > 0. - return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) - } - s.decTable[u].newState = newState - } - } - return nil -} - -// decompress will decompress the bitstream. -// If the buffer is over-read an error is returned. -func (s *Scratch) decompress() error { - br := &s.bits - if err := br.init(s.br.unread()); err != nil { - return err - } - - var s1, s2 decoder - // Initialize and decode first state and symbol. - s1.init(br, s.decTable, s.actualTableLog) - s2.init(br, s.decTable, s.actualTableLog) - - // Use temp table to avoid bound checks/append penalty. - var tmp = s.ct.tableSymbol[:256] - var off uint8 - - // Main part - if !s.zeroBits { - for br.off >= 8 { - br.fillFast() - tmp[off+0] = s1.nextFast() - tmp[off+1] = s2.nextFast() - br.fillFast() - tmp[off+2] = s1.nextFast() - tmp[off+3] = s2.nextFast() - off += 4 - // When off is 0, we have overflowed and should write. - if off == 0 { - s.Out = append(s.Out, tmp...) - if len(s.Out) >= s.DecompressLimit { - return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) - } - } - } - } else { - for br.off >= 8 { - br.fillFast() - tmp[off+0] = s1.next() - tmp[off+1] = s2.next() - br.fillFast() - tmp[off+2] = s1.next() - tmp[off+3] = s2.next() - off += 4 - if off == 0 { - s.Out = append(s.Out, tmp...) - // When off is 0, we have overflowed and should write. - if len(s.Out) >= s.DecompressLimit { - return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) - } - } - } - } - s.Out = append(s.Out, tmp[:off]...) - - // Final bits, a bit more expensive check - for { - if s1.finished() { - s.Out = append(s.Out, s1.final(), s2.final()) - break - } - br.fill() - s.Out = append(s.Out, s1.next()) - if s2.finished() { - s.Out = append(s.Out, s2.final(), s1.final()) - break - } - s.Out = append(s.Out, s2.next()) - if len(s.Out) >= s.DecompressLimit { - return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) - } - } - return br.close() -} - -// decoder keeps track of the current state and updates it from the bitstream. -type decoder struct { - state uint16 - br *bitReader - dt []decSymbol -} - -// init will initialize the decoder and read the first state from the stream. -func (d *decoder) init(in *bitReader, dt []decSymbol, tableLog uint8) { - d.dt = dt - d.br = in - d.state = in.getBits(tableLog) -} - -// next returns the next symbol and sets the next state. -// At least tablelog bits must be available in the bit reader. -func (d *decoder) next() uint8 { - n := &d.dt[d.state] - lowBits := d.br.getBits(n.nbBits) - d.state = n.newState + lowBits - return n.symbol -} - -// finished returns true if all bits have been read from the bitstream -// and the next state would require reading bits from the input. -func (d *decoder) finished() bool { - return d.br.finished() && d.dt[d.state].nbBits > 0 -} - -// final returns the current state symbol without decoding the next. -func (d *decoder) final() uint8 { - return d.dt[d.state].symbol -} - -// nextFast returns the next symbol and sets the next state. -// This can only be used if no symbols are 0 bits. -// At least tablelog bits must be available in the bit reader. -func (d *decoder) nextFast() uint8 { - n := d.dt[d.state] - lowBits := d.br.getBitsFast(n.nbBits) - d.state = n.newState + lowBits - return n.symbol -} diff --git a/vendor/github.com/klauspost/compress/fse/fse.go b/vendor/github.com/klauspost/compress/fse/fse.go deleted file mode 100644 index 535cbadfd..000000000 --- a/vendor/github.com/klauspost/compress/fse/fse.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -// Package fse provides Finite State Entropy encoding and decoding. -// -// Finite State Entropy encoding provides a fast near-optimal symbol encoding/decoding -// for byte blocks as implemented in zstd. -// -// See https://github.com/klauspost/compress/tree/master/fse for more information. -package fse - -import ( - "errors" - "fmt" - "math/bits" -) - -const ( - /*!MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ - maxMemoryUsage = 14 - defaultMemoryUsage = 13 - - maxTableLog = maxMemoryUsage - 2 - maxTablesize = 1 << maxTableLog - defaultTablelog = defaultMemoryUsage - 2 - minTablelog = 5 - maxSymbolValue = 255 -) - -var ( - // ErrIncompressible is returned when input is judged to be too hard to compress. - ErrIncompressible = errors.New("input is not compressible") - - // ErrUseRLE is returned from the compressor when the input is a single byte value repeated. - ErrUseRLE = errors.New("input is single value repeated") -) - -// Scratch provides temporary storage for compression and decompression. -type Scratch struct { - // Private - count [maxSymbolValue + 1]uint32 - norm [maxSymbolValue + 1]int16 - br byteReader - bits bitReader - bw bitWriter - ct cTable // Compression tables. - decTable []decSymbol // Decompression table. - maxCount int // count of the most probable symbol - - // Per block parameters. - // These can be used to override compression parameters of the block. - // Do not touch, unless you know what you are doing. - - // Out is output buffer. - // If the scratch is re-used before the caller is done processing the output, - // set this field to nil. - // Otherwise the output buffer will be re-used for next Compression/Decompression step - // and allocation will be avoided. - Out []byte - - // DecompressLimit limits the maximum decoded size acceptable. - // If > 0 decompression will stop when approximately this many bytes - // has been decoded. - // If 0, maximum size will be 2GB. - DecompressLimit int - - symbolLen uint16 // Length of active part of the symbol table. - actualTableLog uint8 // Selected tablelog. - zeroBits bool // no bits has prob > 50%. - clearCount bool // clear count - - // MaxSymbolValue will override the maximum symbol value of the next block. - MaxSymbolValue uint8 - - // TableLog will attempt to override the tablelog for the next block. - TableLog uint8 -} - -// Histogram allows to populate the histogram and skip that step in the compression, -// It otherwise allows to inspect the histogram when compression is done. -// To indicate that you have populated the histogram call HistogramFinished -// with the value of the highest populated symbol, as well as the number of entries -// in the most populated entry. These are accepted at face value. -// The returned slice will always be length 256. -func (s *Scratch) Histogram() []uint32 { - return s.count[:] -} - -// HistogramFinished can be called to indicate that the histogram has been populated. -// maxSymbol is the index of the highest set symbol of the next data segment. -// maxCount is the number of entries in the most populated entry. -// These are accepted at face value. -func (s *Scratch) HistogramFinished(maxSymbol uint8, maxCount int) { - s.maxCount = maxCount - s.symbolLen = uint16(maxSymbol) + 1 - s.clearCount = maxCount != 0 -} - -// prepare will prepare and allocate scratch tables used for both compression and decompression. -func (s *Scratch) prepare(in []byte) (*Scratch, error) { - if s == nil { - s = &Scratch{} - } - if s.MaxSymbolValue == 0 { - s.MaxSymbolValue = 255 - } - if s.TableLog == 0 { - s.TableLog = defaultTablelog - } - if s.TableLog > maxTableLog { - return nil, fmt.Errorf("tableLog (%d) > maxTableLog (%d)", s.TableLog, maxTableLog) - } - if cap(s.Out) == 0 { - s.Out = make([]byte, 0, len(in)) - } - if s.clearCount && s.maxCount == 0 { - for i := range s.count { - s.count[i] = 0 - } - s.clearCount = false - } - s.br.init(in) - if s.DecompressLimit == 0 { - // Max size 2GB. - s.DecompressLimit = (2 << 30) - 1 - } - - return s, nil -} - -// tableStep returns the next table index. -func tableStep(tableSize uint32) uint32 { - return (tableSize >> 1) + (tableSize >> 3) + 3 -} - -func highBits(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} diff --git a/vendor/github.com/klauspost/compress/gen.sh b/vendor/github.com/klauspost/compress/gen.sh deleted file mode 100644 index aff942205..000000000 --- a/vendor/github.com/klauspost/compress/gen.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd s2/cmd/_s2sx/ || exit 1 -go generate . diff --git a/vendor/github.com/klauspost/compress/huff0/.gitignore b/vendor/github.com/klauspost/compress/huff0/.gitignore deleted file mode 100644 index b3d262958..000000000 --- a/vendor/github.com/klauspost/compress/huff0/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/huff0-fuzz.zip diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md deleted file mode 100644 index 8b6e5c663..000000000 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Huff0 entropy compression
-
-This package provides Huff0 encoding and decoding as used in zstd.
-
-[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders),
-a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU
-(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
-
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding.
-
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
-
-## News
-
-This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
-
-This ensures that most functionality is well tested.
-
-# Usage
-
-This package provides a low level interface that allows to compress single independent blocks.
-
-Each block is separate, and there is no built in integrity checks.
-This means that the caller should keep track of block sizes and also do checksums if needed.
-
-Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and
-[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
-You must provide input and will receive the output and maybe an error.
-
-These error values can be returned:
-
-| Error | Description |
-|---------------------|-----------------------------------------------------------------------------|
-| `<nil>` | Everything ok, output is returned |
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress |
-| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated |
-| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) |
-| `(error)` | An internal error occurred. |
-
-
-As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
-
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same
-object can be used for both.
-
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
-
-The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.
-
-## Tables and re-use
-
-Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results.
-
-The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy)
-that controls this behaviour. See the documentation for details. This can be altered between each block.
-
-Do however note that this information is *not* stored in the output block and it is up to the users of the package to
-record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
-based on the boolean reported back from the CompressXX call.
-
-If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the
-[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
-
-## Decompressing
-
-The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
-This will initialize the decoding tables.
-You can supply the complete block to `ReadTable` and it will return the data part of the block
-which can be given to the decompressor.
-
-Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X)
-or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
-
-For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
-
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
-your input was likely corrupted.
-
-It is important to note that a successful decoding does *not* mean your output matches your original input.
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
-
-# Contributing
-
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking
-changes will likely not be accepted. If in doubt open an issue before writing the PR.
diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go deleted file mode 100644 index e36d9742f..000000000 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package huff0 - -import ( - "encoding/binary" - "errors" - "fmt" - "io" -) - -// bitReader reads a bitstream in reverse. -// The last set bit indicates the start of the stream and is used -// for aligning the input. -type bitReaderBytes struct { - in []byte - off uint // next byte to read is at in[off - 1] - value uint64 - bitsRead uint8 -} - -// init initializes and resets the bit reader. -func (b *bitReaderBytes) init(in []byte) error { - if len(in) < 1 { - return errors.New("corrupt stream: too short") - } - b.in = in - b.off = uint(len(in)) - // The highest bit of the last byte indicates where to start - v := in[len(in)-1] - if v == 0 { - return errors.New("corrupt stream, did not find end of stream") - } - b.bitsRead = 64 - b.value = 0 - if len(in) >= 8 { - b.fillFastStart() - } else { - b.fill() - b.fill() - } - b.advance(8 - uint8(highBit32(uint32(v)))) - return nil -} - -// peekBitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReaderBytes) peekByteFast() uint8 { - got := uint8(b.value >> 56) - return got -} - -func (b *bitReaderBytes) advance(n uint8) { - b.bitsRead += n - b.value <<= n & 63 -} - -// fillFast() will make sure at least 32 bits are available. -// There must be at least 4 bytes available. -func (b *bitReaderBytes) fillFast() { - if b.bitsRead < 32 { - return - } - - // 2 bounds checks. - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value |= uint64(low) << (b.bitsRead - 32) - b.bitsRead -= 32 - b.off -= 4 -} - -// fillFastStart() assumes the bitReaderBytes is empty and there is at least 8 bytes to read. -func (b *bitReaderBytes) fillFastStart() { - // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) - b.bitsRead = 0 - b.off -= 8 -} - -// fill() will make sure at least 32 bits are available. -func (b *bitReaderBytes) fill() { - if b.bitsRead < 32 { - return - } - if b.off > 4 { - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value |= uint64(low) << (b.bitsRead - 32) - b.bitsRead -= 32 - b.off -= 4 - return - } - for b.off > 0 { - b.value |= uint64(b.in[b.off-1]) << (b.bitsRead - 8) - b.bitsRead -= 8 - b.off-- - } -} - -// finished returns true if all bits have been read from the bit stream. -func (b *bitReaderBytes) finished() bool { - return b.off == 0 && b.bitsRead >= 64 -} - -func (b *bitReaderBytes) remaining() uint { - return b.off*8 + uint(64-b.bitsRead) -} - -// close the bitstream and returns an error if out-of-buffer reads occurred. -func (b *bitReaderBytes) close() error { - // Release reference. - b.in = nil - if b.remaining() > 0 { - return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining()) - } - if b.bitsRead > 64 { - return io.ErrUnexpectedEOF - } - return nil -} - -// bitReaderShifted reads a bitstream in reverse. -// The last set bit indicates the start of the stream and is used -// for aligning the input. -type bitReaderShifted struct { - in []byte - off uint // next byte to read is at in[off - 1] - value uint64 - bitsRead uint8 -} - -// init initializes and resets the bit reader. -func (b *bitReaderShifted) init(in []byte) error { - if len(in) < 1 { - return errors.New("corrupt stream: too short") - } - b.in = in - b.off = uint(len(in)) - // The highest bit of the last byte indicates where to start - v := in[len(in)-1] - if v == 0 { - return errors.New("corrupt stream, did not find end of stream") - } - b.bitsRead = 64 - b.value = 0 - if len(in) >= 8 { - b.fillFastStart() - } else { - b.fill() - b.fill() - } - b.advance(8 - uint8(highBit32(uint32(v)))) - return nil -} - -// peekBitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReaderShifted) peekBitsFast(n uint8) uint16 { - return uint16(b.value >> ((64 - n) & 63)) -} - -func (b *bitReaderShifted) advance(n uint8) { - b.bitsRead += n - b.value <<= n & 63 -} - -// fillFast() will make sure at least 32 bits are available. -// There must be at least 4 bytes available. -func (b *bitReaderShifted) fillFast() { - if b.bitsRead < 32 { - return - } - - // 2 bounds checks. - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value |= uint64(low) << ((b.bitsRead - 32) & 63) - b.bitsRead -= 32 - b.off -= 4 -} - -// fillFastStart() assumes the bitReaderShifted is empty and there is at least 8 bytes to read. -func (b *bitReaderShifted) fillFastStart() { - // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) - b.bitsRead = 0 - b.off -= 8 -} - -// fill() will make sure at least 32 bits are available. -func (b *bitReaderShifted) fill() { - if b.bitsRead < 32 { - return - } - if b.off > 4 { - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value |= uint64(low) << ((b.bitsRead - 32) & 63) - b.bitsRead -= 32 - b.off -= 4 - return - } - for b.off > 0 { - b.value |= uint64(b.in[b.off-1]) << ((b.bitsRead - 8) & 63) - b.bitsRead -= 8 - b.off-- - } -} - -func (b *bitReaderShifted) remaining() uint { - return b.off*8 + uint(64-b.bitsRead) -} - -// close the bitstream and returns an error if out-of-buffer reads occurred. -func (b *bitReaderShifted) close() error { - // Release reference. - b.in = nil - if b.remaining() > 0 { - return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining()) - } - if b.bitsRead > 64 { - return io.ErrUnexpectedEOF - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go deleted file mode 100644 index 0ebc9aaac..000000000 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package huff0 - -// bitWriter will write bits. -// First bit will be LSB of the first byte of output. -type bitWriter struct { - bitContainer uint64 - nBits uint8 - out []byte -} - -// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// encSymbol will add up to 16 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) encSymbol(ct cTable, symbol byte) { - enc := ct[symbol] - b.bitContainer |= uint64(enc.val) << (b.nBits & 63) - if false { - if enc.nBits == 0 { - panic("nbits 0") - } - } - b.nBits += enc.nBits -} - -// encTwoSymbols will add up to 32 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { - encA := ct[av] - encB := ct[bv] - sh := b.nBits & 63 - combined := uint64(encA.val) | (uint64(encB.val) << (encA.nBits & 63)) - b.bitContainer |= combined << sh - if false { - if encA.nBits == 0 { - panic("nbitsA 0") - } - if encB.nBits == 0 { - panic("nbitsB 0") - } - } - b.nBits += encA.nBits + encB.nBits -} - -// encFourSymbols adds up to 32 bits from four symbols. -// It will not check if there is space for them, -// so the caller must ensure that b has been flushed recently. -func (b *bitWriter) encFourSymbols(encA, encB, encC, encD cTableEntry) { - bitsA := encA.nBits - bitsB := bitsA + encB.nBits - bitsC := bitsB + encC.nBits - bitsD := bitsC + encD.nBits - combined := uint64(encA.val) | - (uint64(encB.val) << (bitsA & 63)) | - (uint64(encC.val) << (bitsB & 63)) | - (uint64(encD.val) << (bitsC & 63)) - b.bitContainer |= combined << (b.nBits & 63) - b.nBits += bitsD -} - -// flush32 will flush out, so there are at least 32 bits available for writing. -func (b *bitWriter) flush32() { - if b.nBits < 32 { - return - } - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24)) - b.nBits -= 32 - b.bitContainer >>= 32 -} - -// flushAlign will flush remaining full bytes and align to next byte boundary. -func (b *bitWriter) flushAlign() { - nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { - b.out = append(b.out, byte(b.bitContainer>>(i*8))) - } - b.nBits = 0 - b.bitContainer = 0 -} - -// close will write the alignment bit and write the final byte(s) -// to the output. -func (b *bitWriter) close() { - // End mark - b.addBits16Clean(1, 1) - // flush until next byte. - b.flushAlign() -} diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go deleted file mode 100644 index 84aa3d12f..000000000 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ /dev/null @@ -1,742 +0,0 @@ -package huff0 - -import ( - "fmt" - "math" - "runtime" - "sync" -) - -// Compress1X will compress the input. -// The output can be decoded using Decompress1X. -// Supply a Scratch object. The scratch object contains state about re-use, -// So when sharing across independent encodes, be sure to set the re-use policy. -func Compress1X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { - s, err = s.prepare(in) - if err != nil { - return nil, false, err - } - return compress(in, s, s.compress1X) -} - -// Compress4X will compress the input. The input is split into 4 independent blocks -// and compressed similar to Compress1X. -// The output can be decoded using Decompress4X. -// Supply a Scratch object. The scratch object contains state about re-use, -// So when sharing across independent encodes, be sure to set the re-use policy. -func Compress4X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { - s, err = s.prepare(in) - if err != nil { - return nil, false, err - } - if false { - // TODO: compress4Xp only slightly faster. - const parallelThreshold = 8 << 10 - if len(in) < parallelThreshold || runtime.GOMAXPROCS(0) == 1 { - return compress(in, s, s.compress4X) - } - return compress(in, s, s.compress4Xp) - } - return compress(in, s, s.compress4X) -} - -func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error)) (out []byte, reUsed bool, err error) { - // Nuke previous table if we cannot reuse anyway. - if s.Reuse == ReusePolicyNone { - s.prevTable = s.prevTable[:0] - } - - // Create histogram, if none was provided. - maxCount := s.maxCount - var canReuse = false - if maxCount == 0 { - maxCount, canReuse = s.countSimple(in) - } else { - canReuse = s.canUseTable(s.prevTable) - } - - // We want the output size to be less than this: - wantSize := len(in) - if s.WantLogLess > 0 { - wantSize -= wantSize >> s.WantLogLess - } - - // Reset for next run. - s.clearCount = true - s.maxCount = 0 - if maxCount >= len(in) { - if maxCount > len(in) { - return nil, false, fmt.Errorf("maxCount (%d) > length (%d)", maxCount, len(in)) - } - if len(in) == 1 { - return nil, false, ErrIncompressible - } - // One symbol, use RLE - return nil, false, ErrUseRLE - } - if maxCount == 1 || maxCount < (len(in)>>7) { - // Each symbol present maximum once or too well distributed. - return nil, false, ErrIncompressible - } - if s.Reuse == ReusePolicyMust && !canReuse { - // We must reuse, but we can't. - return nil, false, ErrIncompressible - } - if (s.Reuse == ReusePolicyPrefer || s.Reuse == ReusePolicyMust) && canReuse { - keepTable := s.cTable - keepTL := s.actualTableLog - s.cTable = s.prevTable - s.actualTableLog = s.prevTableLog - s.Out, err = compressor(in) - s.cTable = keepTable - s.actualTableLog = keepTL - if err == nil && len(s.Out) < wantSize { - s.OutData = s.Out - return s.Out, true, nil - } - if s.Reuse == ReusePolicyMust { - return nil, false, ErrIncompressible - } - // Do not attempt to re-use later. - s.prevTable = s.prevTable[:0] - } - - // Calculate new table. - err = s.buildCTable() - if err != nil { - return nil, false, err - } - - if false && !s.canUseTable(s.cTable) { - panic("invalid table generated") - } - - if s.Reuse == ReusePolicyAllow && canReuse { - hSize := len(s.Out) - oldSize := s.prevTable.estimateSize(s.count[:s.symbolLen]) - newSize := s.cTable.estimateSize(s.count[:s.symbolLen]) - if oldSize <= hSize+newSize || hSize+12 >= wantSize { - // Retain cTable even if we re-use. - keepTable := s.cTable - keepTL := s.actualTableLog - - s.cTable = s.prevTable - s.actualTableLog = s.prevTableLog - s.Out, err = compressor(in) - - // Restore ctable. - s.cTable = keepTable - s.actualTableLog = keepTL - if err != nil { - return nil, false, err - } - if len(s.Out) >= wantSize { - return nil, false, ErrIncompressible - } - s.OutData = s.Out - return s.Out, true, nil - } - } - - // Use new table - err = s.cTable.write(s) - if err != nil { - s.OutTable = nil - return nil, false, err - } - s.OutTable = s.Out - - // Compress using new table - s.Out, err = compressor(in) - if err != nil { - s.OutTable = nil - return nil, false, err - } - if len(s.Out) >= wantSize { - s.OutTable = nil - return nil, false, ErrIncompressible - } - // Move current table into previous. - s.prevTable, s.prevTableLog, s.cTable = s.cTable, s.actualTableLog, s.prevTable[:0] - s.OutData = s.Out[len(s.OutTable):] - return s.Out, false, nil -} - -// EstimateSizes will estimate the data sizes -func EstimateSizes(in []byte, s *Scratch) (tableSz, dataSz, reuseSz int, err error) { - s, err = s.prepare(in) - if err != nil { - return 0, 0, 0, err - } - - // Create histogram, if none was provided. - tableSz, dataSz, reuseSz = -1, -1, -1 - maxCount := s.maxCount - var canReuse = false - if maxCount == 0 { - maxCount, canReuse = s.countSimple(in) - } else { - canReuse = s.canUseTable(s.prevTable) - } - - // We want the output size to be less than this: - wantSize := len(in) - if s.WantLogLess > 0 { - wantSize -= wantSize >> s.WantLogLess - } - - // Reset for next run. - s.clearCount = true - s.maxCount = 0 - if maxCount >= len(in) { - if maxCount > len(in) { - return 0, 0, 0, fmt.Errorf("maxCount (%d) > length (%d)", maxCount, len(in)) - } - if len(in) == 1 { - return 0, 0, 0, ErrIncompressible - } - // One symbol, use RLE - return 0, 0, 0, ErrUseRLE - } - if maxCount == 1 || maxCount < (len(in)>>7) { - // Each symbol present maximum once or too well distributed. - return 0, 0, 0, ErrIncompressible - } - - // Calculate new table. - err = s.buildCTable() - if err != nil { - return 0, 0, 0, err - } - - if false && !s.canUseTable(s.cTable) { - panic("invalid table generated") - } - - tableSz, err = s.cTable.estTableSize(s) - if err != nil { - return 0, 0, 0, err - } - if canReuse { - reuseSz = s.prevTable.estimateSize(s.count[:s.symbolLen]) - } - dataSz = s.cTable.estimateSize(s.count[:s.symbolLen]) - - // Restore - return tableSz, dataSz, reuseSz, nil -} - -func (s *Scratch) compress1X(src []byte) ([]byte, error) { - return s.compress1xDo(s.Out, src), nil -} - -func (s *Scratch) compress1xDo(dst, src []byte) []byte { - var bw = bitWriter{out: dst} - - // N is length divisible by 4. - n := len(src) - n -= n & 3 - cTable := s.cTable[:256] - - // Encode last bytes. - for i := len(src) & 3; i > 0; i-- { - bw.encSymbol(cTable, src[n+i-1]) - } - n -= 4 - if s.actualTableLog <= 8 { - for ; n >= 0; n -= 4 { - tmp := src[n : n+4] - // tmp should be len 4 - bw.flush32() - bw.encFourSymbols(cTable[tmp[3]], cTable[tmp[2]], cTable[tmp[1]], cTable[tmp[0]]) - } - } else { - for ; n >= 0; n -= 4 { - tmp := src[n : n+4] - // tmp should be len 4 - bw.flush32() - bw.encTwoSymbols(cTable, tmp[3], tmp[2]) - bw.flush32() - bw.encTwoSymbols(cTable, tmp[1], tmp[0]) - } - } - bw.close() - return bw.out -} - -var sixZeros [6]byte - -func (s *Scratch) compress4X(src []byte) ([]byte, error) { - if len(src) < 12 { - return nil, ErrIncompressible - } - segmentSize := (len(src) + 3) / 4 - - // Add placeholder for output length - offsetIdx := len(s.Out) - s.Out = append(s.Out, sixZeros[:]...) - - for i := 0; i < 4; i++ { - toDo := src - if len(toDo) > segmentSize { - toDo = toDo[:segmentSize] - } - src = src[len(toDo):] - - idx := len(s.Out) - s.Out = s.compress1xDo(s.Out, toDo) - if len(s.Out)-idx > math.MaxUint16 { - // We cannot store the size in the jump table - return nil, ErrIncompressible - } - // Write compressed length as little endian before block. - if i < 3 { - // Last length is not written. - length := len(s.Out) - idx - s.Out[i*2+offsetIdx] = byte(length) - s.Out[i*2+offsetIdx+1] = byte(length >> 8) - } - } - - return s.Out, nil -} - -// compress4Xp will compress 4 streams using separate goroutines. -func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { - if len(src) < 12 { - return nil, ErrIncompressible - } - // Add placeholder for output length - s.Out = s.Out[:6] - - segmentSize := (len(src) + 3) / 4 - var wg sync.WaitGroup - wg.Add(4) - for i := 0; i < 4; i++ { - toDo := src - if len(toDo) > segmentSize { - toDo = toDo[:segmentSize] - } - src = src[len(toDo):] - - // Separate goroutine for each block. - go func(i int) { - s.tmpOut[i] = s.compress1xDo(s.tmpOut[i][:0], toDo) - wg.Done() - }(i) - } - wg.Wait() - for i := 0; i < 4; i++ { - o := s.tmpOut[i] - if len(o) > math.MaxUint16 { - // We cannot store the size in the jump table - return nil, ErrIncompressible - } - // Write compressed length as little endian before block. - if i < 3 { - // Last length is not written. - s.Out[i*2] = byte(len(o)) - s.Out[i*2+1] = byte(len(o) >> 8) - } - - // Write output. - s.Out = append(s.Out, o...) - } - return s.Out, nil -} - -// countSimple will create a simple histogram in s.count. -// Returns the biggest count. -// Does not update s.clearCount. -func (s *Scratch) countSimple(in []byte) (max int, reuse bool) { - reuse = true - _ = s.count // Assert that s != nil to speed up the following loop. - for _, v := range in { - s.count[v]++ - } - m := uint32(0) - if len(s.prevTable) > 0 { - for i, v := range s.count[:] { - if v == 0 { - continue - } - if v > m { - m = v - } - s.symbolLen = uint16(i) + 1 - if i >= len(s.prevTable) { - reuse = false - } else if s.prevTable[i].nBits == 0 { - reuse = false - } - } - return int(m), reuse - } - for i, v := range s.count[:] { - if v == 0 { - continue - } - if v > m { - m = v - } - s.symbolLen = uint16(i) + 1 - } - return int(m), false -} - -func (s *Scratch) canUseTable(c cTable) bool { - if len(c) < int(s.symbolLen) { - return false - } - for i, v := range s.count[:s.symbolLen] { - if v != 0 && c[i].nBits == 0 { - return false - } - } - return true -} - -//lint:ignore U1000 used for debugging -func (s *Scratch) validateTable(c cTable) bool { - if len(c) < int(s.symbolLen) { - return false - } - for i, v := range s.count[:s.symbolLen] { - if v != 0 { - if c[i].nBits == 0 { - return false - } - if c[i].nBits > s.actualTableLog { - return false - } - } - } - return true -} - -// minTableLog provides the minimum logSize to safely represent a distribution. -func (s *Scratch) minTableLog() uint8 { - minBitsSrc := highBit32(uint32(s.srcLen)) + 1 - minBitsSymbols := highBit32(uint32(s.symbolLen-1)) + 2 - if minBitsSrc < minBitsSymbols { - return uint8(minBitsSrc) - } - return uint8(minBitsSymbols) -} - -// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog -func (s *Scratch) optimalTableLog() { - tableLog := s.TableLog - minBits := s.minTableLog() - maxBitsSrc := uint8(highBit32(uint32(s.srcLen-1))) - 1 - if maxBitsSrc < tableLog { - // Accuracy can be reduced - tableLog = maxBitsSrc - } - if minBits > tableLog { - tableLog = minBits - } - // Need a minimum to safely represent all symbol values - if tableLog < minTablelog { - tableLog = minTablelog - } - if tableLog > tableLogMax { - tableLog = tableLogMax - } - s.actualTableLog = tableLog -} - -type cTableEntry struct { - val uint16 - nBits uint8 - // We have 8 bits extra -} - -const huffNodesMask = huffNodesLen - 1 - -func (s *Scratch) buildCTable() error { - s.optimalTableLog() - s.huffSort() - if cap(s.cTable) < maxSymbolValue+1 { - s.cTable = make([]cTableEntry, s.symbolLen, maxSymbolValue+1) - } else { - s.cTable = s.cTable[:s.symbolLen] - for i := range s.cTable { - s.cTable[i] = cTableEntry{} - } - } - - var startNode = int16(s.symbolLen) - nonNullRank := s.symbolLen - 1 - - nodeNb := startNode - huffNode := s.nodes[1 : huffNodesLen+1] - - // This overlays the slice above, but allows "-1" index lookups. - // Different from reference implementation. - huffNode0 := s.nodes[0 : huffNodesLen+1] - - for huffNode[nonNullRank].count() == 0 { - nonNullRank-- - } - - lowS := int16(nonNullRank) - nodeRoot := nodeNb + lowS - 1 - lowN := nodeNb - huffNode[nodeNb].setCount(huffNode[lowS].count() + huffNode[lowS-1].count()) - huffNode[lowS].setParent(nodeNb) - huffNode[lowS-1].setParent(nodeNb) - nodeNb++ - lowS -= 2 - for n := nodeNb; n <= nodeRoot; n++ { - huffNode[n].setCount(1 << 30) - } - // fake entry, strong barrier - huffNode0[0].setCount(1 << 31) - - // create parents - for nodeNb <= nodeRoot { - var n1, n2 int16 - if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { - n1 = lowS - lowS-- - } else { - n1 = lowN - lowN++ - } - if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { - n2 = lowS - lowS-- - } else { - n2 = lowN - lowN++ - } - - huffNode[nodeNb].setCount(huffNode0[n1+1].count() + huffNode0[n2+1].count()) - huffNode0[n1+1].setParent(nodeNb) - huffNode0[n2+1].setParent(nodeNb) - nodeNb++ - } - - // distribute weights (unlimited tree height) - huffNode[nodeRoot].setNbBits(0) - for n := nodeRoot - 1; n >= startNode; n-- { - huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) - } - for n := uint16(0); n <= nonNullRank; n++ { - huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) - } - s.actualTableLog = s.setMaxHeight(int(nonNullRank)) - maxNbBits := s.actualTableLog - - // fill result into tree (val, nbBits) - if maxNbBits > tableLogMax { - return fmt.Errorf("internal error: maxNbBits (%d) > tableLogMax (%d)", maxNbBits, tableLogMax) - } - var nbPerRank [tableLogMax + 1]uint16 - var valPerRank [16]uint16 - for _, v := range huffNode[:nonNullRank+1] { - nbPerRank[v.nbBits()]++ - } - // determine stating value per rank - { - min := uint16(0) - for n := maxNbBits; n > 0; n-- { - // get starting value within each rank - valPerRank[n] = min - min += nbPerRank[n] - min >>= 1 - } - } - - // push nbBits per symbol, symbol order - for _, v := range huffNode[:nonNullRank+1] { - s.cTable[v.symbol()].nBits = v.nbBits() - } - - // assign value within rank, symbol order - t := s.cTable[:s.symbolLen] - for n, val := range t { - nbits := val.nBits & 15 - v := valPerRank[nbits] - t[n].val = v - valPerRank[nbits] = v + 1 - } - - return nil -} - -// huffSort will sort symbols, decreasing order. -func (s *Scratch) huffSort() { - type rankPos struct { - base uint32 - current uint32 - } - - // Clear nodes - nodes := s.nodes[:huffNodesLen+1] - s.nodes = nodes - nodes = nodes[1 : huffNodesLen+1] - - // Sort into buckets based on length of symbol count. - var rank [32]rankPos - for _, v := range s.count[:s.symbolLen] { - r := highBit32(v+1) & 31 - rank[r].base++ - } - // maxBitLength is log2(BlockSizeMax) + 1 - const maxBitLength = 18 + 1 - for n := maxBitLength; n > 0; n-- { - rank[n-1].base += rank[n].base - } - for n := range rank[:maxBitLength] { - rank[n].current = rank[n].base - } - for n, c := range s.count[:s.symbolLen] { - r := (highBit32(c+1) + 1) & 31 - pos := rank[r].current - rank[r].current++ - prev := nodes[(pos-1)&huffNodesMask] - for pos > rank[r].base && c > prev.count() { - nodes[pos&huffNodesMask] = prev - pos-- - prev = nodes[(pos-1)&huffNodesMask] - } - nodes[pos&huffNodesMask] = makeNodeElt(c, byte(n)) - } -} - -func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { - maxNbBits := s.actualTableLog - huffNode := s.nodes[1 : huffNodesLen+1] - //huffNode = huffNode[: huffNodesLen] - - largestBits := huffNode[lastNonNull].nbBits() - - // early exit : no elt > maxNbBits - if largestBits <= maxNbBits { - return largestBits - } - totalCost := int(0) - baseCost := int(1) << (largestBits - maxNbBits) - n := uint32(lastNonNull) - - for huffNode[n].nbBits() > maxNbBits { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits())) - huffNode[n].setNbBits(maxNbBits) - n-- - } - // n stops at huffNode[n].nbBits <= maxNbBits - - for huffNode[n].nbBits() == maxNbBits { - n-- - } - // n end at index of smallest symbol using < maxNbBits - - // renorm totalCost - totalCost >>= largestBits - maxNbBits /* note : totalCost is necessarily a multiple of baseCost */ - - // repay normalized cost - { - const noSymbol = 0xF0F0F0F0 - var rankLast [tableLogMax + 2]uint32 - - for i := range rankLast[:] { - rankLast[i] = noSymbol - } - - // Get pos of last (smallest) symbol per rank - { - currentNbBits := maxNbBits - for pos := int(n); pos >= 0; pos-- { - if huffNode[pos].nbBits() >= currentNbBits { - continue - } - currentNbBits = huffNode[pos].nbBits() // < maxNbBits - rankLast[maxNbBits-currentNbBits] = uint32(pos) - } - } - - for totalCost > 0 { - nBitsToDecrease := uint8(highBit32(uint32(totalCost))) + 1 - - for ; nBitsToDecrease > 1; nBitsToDecrease-- { - highPos := rankLast[nBitsToDecrease] - lowPos := rankLast[nBitsToDecrease-1] - if highPos == noSymbol { - continue - } - if lowPos == noSymbol { - break - } - highTotal := huffNode[highPos].count() - lowTotal := 2 * huffNode[lowPos].count() - if highTotal <= lowTotal { - break - } - } - // only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) - // HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary - // FIXME: try to remove - for (nBitsToDecrease <= tableLogMax) && (rankLast[nBitsToDecrease] == noSymbol) { - nBitsToDecrease++ - } - totalCost -= 1 << (nBitsToDecrease - 1) - if rankLast[nBitsToDecrease-1] == noSymbol { - // this rank is no longer empty - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease] - } - huffNode[rankLast[nBitsToDecrease]].setNbBits(1 + - huffNode[rankLast[nBitsToDecrease]].nbBits()) - if rankLast[nBitsToDecrease] == 0 { - /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol - } else { - rankLast[nBitsToDecrease]-- - if huffNode[rankLast[nBitsToDecrease]].nbBits() != maxNbBits-nBitsToDecrease { - rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */ - } - } - } - - for totalCost < 0 { /* Sometimes, cost correction overshoot */ - if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - for huffNode[n].nbBits() == maxNbBits { - n-- - } - huffNode[n+1].setNbBits(huffNode[n+1].nbBits() - 1) - rankLast[1] = n + 1 - totalCost++ - continue - } - huffNode[rankLast[1]+1].setNbBits(huffNode[rankLast[1]+1].nbBits() - 1) - rankLast[1]++ - totalCost++ - } - } - return maxNbBits -} - -// A nodeElt is the fields -// -// count uint32 -// parent uint16 -// symbol byte -// nbBits uint8 -// -// in some order, all squashed into an integer so that the compiler -// always loads and stores entire nodeElts instead of separate fields. -type nodeElt uint64 - -func makeNodeElt(count uint32, symbol byte) nodeElt { - return nodeElt(count) | nodeElt(symbol)<<48 -} - -func (e *nodeElt) count() uint32 { return uint32(*e) } -func (e *nodeElt) parent() uint16 { return uint16(*e >> 32) } -func (e *nodeElt) symbol() byte { return byte(*e >> 48) } -func (e *nodeElt) nbBits() uint8 { return uint8(*e >> 56) } - -func (e *nodeElt) setCount(c uint32) { *e = (*e)&0xffffffff00000000 | nodeElt(c) } -func (e *nodeElt) setParent(p int16) { *e = (*e)&0xffff0000ffffffff | nodeElt(uint16(p))<<32 } -func (e *nodeElt) setNbBits(n uint8) { *e = (*e)&0x00ffffffffffffff | nodeElt(n)<<56 } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go deleted file mode 100644 index 0f56b02d7..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ /dev/null @@ -1,1167 +0,0 @@ -package huff0 - -import ( - "errors" - "fmt" - "io" - "sync" - - "github.com/klauspost/compress/fse" -) - -type dTable struct { - single []dEntrySingle -} - -// single-symbols decoding -type dEntrySingle struct { - entry uint16 -} - -// Uses special code for all tables that are < 8 bits. -const use8BitTables = true - -// ReadTable will read a table from the input. -// The size of the input may be larger than the table definition. -// Any content remaining after the table definition will be returned. -// If no Scratch is provided a new one is allocated. -// The returned Scratch can be used for encoding or decoding input using this table. -func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { - s, err = s.prepare(nil) - if err != nil { - return s, nil, err - } - if len(in) <= 1 { - return s, nil, errors.New("input too small for table") - } - iSize := in[0] - in = in[1:] - if iSize >= 128 { - // Uncompressed - oSize := iSize - 127 - iSize = (oSize + 1) / 2 - if int(iSize) > len(in) { - return s, nil, errors.New("input too small for table") - } - for n := uint8(0); n < oSize; n += 2 { - v := in[n/2] - s.huffWeight[n] = v >> 4 - s.huffWeight[n+1] = v & 15 - } - s.symbolLen = uint16(oSize) - in = in[iSize:] - } else { - if len(in) < int(iSize) { - return s, nil, fmt.Errorf("input too small for table, want %d bytes, have %d", iSize, len(in)) - } - // FSE compressed weights - s.fse.DecompressLimit = 255 - hw := s.huffWeight[:] - s.fse.Out = hw - b, err := fse.Decompress(in[:iSize], s.fse) - s.fse.Out = nil - if err != nil { - return s, nil, fmt.Errorf("fse decompress returned: %w", err) - } - if len(b) > 255 { - return s, nil, errors.New("corrupt input: output table too large") - } - s.symbolLen = uint16(len(b)) - in = in[iSize:] - } - - // collect weight stats - var rankStats [16]uint32 - weightTotal := uint32(0) - for _, v := range s.huffWeight[:s.symbolLen] { - if v > tableLogMax { - return s, nil, errors.New("corrupt input: weight too large") - } - v2 := v & 15 - rankStats[v2]++ - // (1 << (v2-1)) is slower since the compiler cannot prove that v2 isn't 0. - weightTotal += (1 << v2) >> 1 - } - if weightTotal == 0 { - return s, nil, errors.New("corrupt input: weights zero") - } - - // get last non-null symbol weight (implied, total must be 2^n) - { - tableLog := highBit32(weightTotal) + 1 - if tableLog > tableLogMax { - return s, nil, errors.New("corrupt input: tableLog too big") - } - s.actualTableLog = uint8(tableLog) - // determine last weight - { - total := uint32(1) << tableLog - rest := total - weightTotal - verif := uint32(1) << highBit32(rest) - lastWeight := highBit32(rest) + 1 - if verif != rest { - // last value must be a clean power of 2 - return s, nil, errors.New("corrupt input: last value not power of two") - } - s.huffWeight[s.symbolLen] = uint8(lastWeight) - s.symbolLen++ - rankStats[lastWeight]++ - } - } - - if (rankStats[1] < 2) || (rankStats[1]&1 != 0) { - // by construction : at least 2 elts of rank 1, must be even - return s, nil, errors.New("corrupt input: min elt size, even check failed ") - } - - // TODO: Choose between single/double symbol decoding - - // Calculate starting value for each rank - { - var nextRankStart uint32 - for n := uint8(1); n < s.actualTableLog+1; n++ { - current := nextRankStart - nextRankStart += rankStats[n] << (n - 1) - rankStats[n] = current - } - } - - // fill DTable (always full size) - tSize := 1 << tableLogMax - if len(s.dt.single) != tSize { - s.dt.single = make([]dEntrySingle, tSize) - } - cTable := s.prevTable - if cap(cTable) < maxSymbolValue+1 { - cTable = make([]cTableEntry, 0, maxSymbolValue+1) - } - cTable = cTable[:maxSymbolValue+1] - s.prevTable = cTable[:s.symbolLen] - s.prevTableLog = s.actualTableLog - - for n, w := range s.huffWeight[:s.symbolLen] { - if w == 0 { - cTable[n] = cTableEntry{ - val: 0, - nBits: 0, - } - continue - } - length := (uint32(1) << w) >> 1 - d := dEntrySingle{ - entry: uint16(s.actualTableLog+1-w) | (uint16(n) << 8), - } - - rank := &rankStats[w] - cTable[n] = cTableEntry{ - val: uint16(*rank >> (w - 1)), - nBits: uint8(d.entry), - } - - single := s.dt.single[*rank : *rank+length] - for i := range single { - single[i] = d - } - *rank += length - } - - return s, in, nil -} - -// Decompress1X will decompress a 1X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// Before this is called, the table must be initialized with ReadTable unless -// the encoder re-used the table. -// deprecated: Use the stateless Decoder() to get a concurrent version. -func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { - if cap(s.Out) < s.MaxDecodedSize { - s.Out = make([]byte, s.MaxDecodedSize) - } - s.Out = s.Out[:0:s.MaxDecodedSize] - s.Out, err = s.Decoder().Decompress1X(s.Out, in) - return s.Out, err -} - -// Decompress4X will decompress a 4X encoded stream. -// Before this is called, the table must be initialized with ReadTable unless -// the encoder re-used the table. -// The length of the supplied input must match the end of a block exactly. -// The destination size of the uncompressed data must be known and provided. -// deprecated: Use the stateless Decoder() to get a concurrent version. -func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { - if dstSize > s.MaxDecodedSize { - return nil, ErrMaxDecodedSizeExceeded - } - if cap(s.Out) < dstSize { - s.Out = make([]byte, s.MaxDecodedSize) - } - s.Out = s.Out[:0:dstSize] - s.Out, err = s.Decoder().Decompress4X(s.Out, in) - return s.Out, err -} - -// Decoder will return a stateless decoder that can be used by multiple -// decompressors concurrently. -// Before this is called, the table must be initialized with ReadTable. -// The Decoder is still linked to the scratch buffer so that cannot be reused. -// However, it is safe to discard the scratch. -func (s *Scratch) Decoder() *Decoder { - return &Decoder{ - dt: s.dt, - actualTableLog: s.actualTableLog, - bufs: &s.decPool, - } -} - -// Decoder provides stateless decoding. -type Decoder struct { - dt dTable - actualTableLog uint8 - bufs *sync.Pool -} - -func (d *Decoder) buffer() *[4][256]byte { - buf, ok := d.bufs.Get().(*[4][256]byte) - if ok { - return buf - } - return &[4][256]byte{} -} - -// decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { - if d.actualTableLog == 8 { - return d.decompress1X8BitExactly(dst, src) - } - var br bitReaderBytes - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:0] - - // Avoid bounds check by always having full sized table. - dt := d.dt.single[:256] - - // Use temp table to avoid bound checks/append penalty. - bufs := d.buffer() - buf := &bufs[0] - var off uint8 - - switch d.actualTableLog { - case 8: - const shift = 0 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - d.bufs.Put(bufs) - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 7: - const shift = 8 - 7 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - d.bufs.Put(bufs) - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 6: - const shift = 8 - 6 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 5: - const shift = 8 - 5 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 4: - const shift = 8 - 4 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 3: - const shift = 8 - 3 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 2: - const shift = 8 - 2 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - case 1: - const shift = 8 - 1 - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>(56+shift))] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - default: - d.bufs.Put(bufs) - return nil, fmt.Errorf("invalid tablelog: %d", d.actualTableLog) - } - - if len(dst)+int(off) > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:off]...) - - // br < 4, so uint8 is fine - bitsLeft := int8(uint8(br.off)*8 + (64 - br.bitsRead)) - shift := (8 - d.actualTableLog) & 7 - - for bitsLeft > 0 { - if br.bitsRead >= 64-8 { - for br.off > 0 { - br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) - br.bitsRead -= 8 - br.off-- - } - } - if len(dst) >= maxDecodedSize { - br.close() - d.bufs.Put(bufs) - return nil, ErrMaxDecodedSizeExceeded - } - v := dt[br.peekByteFast()>>shift] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= int8(nBits) - dst = append(dst, uint8(v.entry>>8)) - } - d.bufs.Put(bufs) - return dst, br.close() -} - -// decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) { - var br bitReaderBytes - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:0] - - // Avoid bounds check by always having full sized table. - dt := d.dt.single[:256] - - // Use temp table to avoid bound checks/append penalty. - bufs := d.buffer() - buf := &bufs[0] - var off uint8 - - const shift = 56 - - //fmt.Printf("mask: %b, tl:%d\n", mask, d.actualTableLog) - for br.off >= 4 { - br.fillFast() - v := dt[uint8(br.value>>shift)] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>shift)] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>shift)] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[uint8(br.value>>shift)] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - - if len(dst)+int(off) > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:off]...) - - // br < 4, so uint8 is fine - bitsLeft := int8(uint8(br.off)*8 + (64 - br.bitsRead)) - for bitsLeft > 0 { - if br.bitsRead >= 64-8 { - for br.off > 0 { - br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) - br.bitsRead -= 8 - br.off-- - } - } - if len(dst) >= maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := dt[br.peekByteFast()] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= int8(nBits) - dst = append(dst, uint8(v.entry>>8)) - } - d.bufs.Put(bufs) - return dst, br.close() -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { - if d.actualTableLog == 8 { - return d.decompress4X8bitExactly(dst, src) - } - - var br [4]bitReaderBytes - start := 6 - for i := 0; i < 3; i++ { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - shift := (56 + (8 - d.actualTableLog)) & 63 - - const tlSize = 1 << 8 - single := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - buf := d.buffer() - var off uint8 - var decoded int - - // Decode 4 values from each decoder/loop. - const bufoff = 256 - for { - if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { - break - } - - { - // Interleave 2 decodes. - const stream = 0 - const stream2 = 1 - br1 := &br[stream] - br2 := &br[stream2] - br1.fillFast() - br2.fillFast() - - v := single[uint8(br1.value>>shift)].entry - v2 := single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off] = uint8(v >> 8) - buf[stream2][off] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+1] = uint8(v >> 8) - buf[stream2][off+1] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+2] = uint8(v >> 8) - buf[stream2][off+2] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+3] = uint8(v >> 8) - buf[stream2][off+3] = uint8(v2 >> 8) - } - - { - const stream = 2 - const stream2 = 3 - br1 := &br[stream] - br2 := &br[stream2] - br1.fillFast() - br2.fillFast() - - v := single[uint8(br1.value>>shift)].entry - v2 := single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off] = uint8(v >> 8) - buf[stream2][off] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+1] = uint8(v >> 8) - buf[stream2][off+1] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+2] = uint8(v >> 8) - buf[stream2][off+2] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+3] = uint8(v >> 8) - buf[stream2][off+3] = uint8(v2 >> 8) - } - - off += 4 - - if off == 0 { - if bufoff > dstEvery { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 1") - } - // There must at least be 3 buffers left. - if len(out)-bufoff < dstEvery*3 { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 2") - } - //copy(out, buf[0][:]) - //copy(out[dstEvery:], buf[1][:]) - //copy(out[dstEvery*2:], buf[2][:]) - *(*[bufoff]byte)(out) = buf[0] - *(*[bufoff]byte)(out[dstEvery:]) = buf[1] - *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] - *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] - out = out[bufoff:] - decoded += bufoff * 4 - } - } - if off > 0 { - ioff := int(off) - if len(out) < dstEvery*3+ioff { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 3") - } - copy(out, buf[0][:off]) - copy(out[dstEvery:], buf[1][:off]) - copy(out[dstEvery*2:], buf[2][:off]) - copy(out[dstEvery*3:], buf[3][:off]) - decoded += int(off) * 4 - out = out[off:] - } - - // Decode remaining. - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - if br.finished() { - d.bufs.Put(buf) - return nil, io.ErrUnexpectedEOF - } - if br.bitsRead >= 56 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value |= uint64(low) << (br.bitsRead - 32) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) - br.bitsRead -= 8 - br.off-- - } - } - } - // end inline... - if offset >= endsAt { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - v := single[uint8(br.value>>shift)].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - d.bufs.Put(buf) - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - d.bufs.Put(buf) - return nil, err - } - } - d.bufs.Put(buf) - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { - var br [4]bitReaderBytes - start := 6 - for i := 0; i < 3; i++ { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const shift = 56 - const tlSize = 1 << 8 - single := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - buf := d.buffer() - var off uint8 - var decoded int - - // Decode 4 values from each decoder/loop. - const bufoff = 256 - for { - if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { - break - } - - { - // Interleave 2 decodes. - const stream = 0 - const stream2 = 1 - br1 := &br[stream] - br2 := &br[stream2] - br1.fillFast() - br2.fillFast() - - v := single[uint8(br1.value>>shift)].entry - v2 := single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off] = uint8(v >> 8) - buf[stream2][off] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+1] = uint8(v >> 8) - buf[stream2][off+1] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+2] = uint8(v >> 8) - buf[stream2][off+2] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+3] = uint8(v >> 8) - buf[stream2][off+3] = uint8(v2 >> 8) - } - - { - const stream = 2 - const stream2 = 3 - br1 := &br[stream] - br2 := &br[stream2] - br1.fillFast() - br2.fillFast() - - v := single[uint8(br1.value>>shift)].entry - v2 := single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off] = uint8(v >> 8) - buf[stream2][off] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+1] = uint8(v >> 8) - buf[stream2][off+1] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+2] = uint8(v >> 8) - buf[stream2][off+2] = uint8(v2 >> 8) - - v = single[uint8(br1.value>>shift)].entry - v2 = single[uint8(br2.value>>shift)].entry - br1.bitsRead += uint8(v) - br1.value <<= v & 63 - br2.bitsRead += uint8(v2) - br2.value <<= v2 & 63 - buf[stream][off+3] = uint8(v >> 8) - buf[stream2][off+3] = uint8(v2 >> 8) - } - - off += 4 - - if off == 0 { - if bufoff > dstEvery { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 1") - } - // There must at least be 3 buffers left. - if len(out)-bufoff < dstEvery*3 { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 2") - } - - //copy(out, buf[0][:]) - //copy(out[dstEvery:], buf[1][:]) - //copy(out[dstEvery*2:], buf[2][:]) - // copy(out[dstEvery*3:], buf[3][:]) - *(*[bufoff]byte)(out) = buf[0] - *(*[bufoff]byte)(out[dstEvery:]) = buf[1] - *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] - *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] - out = out[bufoff:] - decoded += bufoff * 4 - } - } - if off > 0 { - ioff := int(off) - if len(out) < dstEvery*3+ioff { - return nil, errors.New("corruption detected: stream overrun 3") - } - copy(out, buf[0][:off]) - copy(out[dstEvery:], buf[1][:off]) - copy(out[dstEvery*2:], buf[2][:off]) - copy(out[dstEvery*3:], buf[3][:off]) - decoded += int(off) * 4 - out = out[off:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - if br.finished() { - d.bufs.Put(buf) - return nil, io.ErrUnexpectedEOF - } - if br.bitsRead >= 56 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value |= uint64(low) << (br.bitsRead - 32) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) - br.bitsRead -= 8 - br.off-- - } - } - } - // end inline... - if offset >= endsAt { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - v := single[br.peekByteFast()].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - d.bufs.Put(buf) - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - d.bufs.Put(buf) - return nil, err - } - } - d.bufs.Put(buf) - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// matches will compare a decoding table to a coding table. -// Errors are written to the writer. -// Nothing will be written if table is ok. -func (s *Scratch) matches(ct cTable, w io.Writer) { - if s == nil || len(s.dt.single) == 0 { - return - } - dt := s.dt.single[:1<<s.actualTableLog] - tablelog := s.actualTableLog - ok := 0 - broken := 0 - for sym, enc := range ct { - errs := 0 - broken++ - if enc.nBits == 0 { - for _, dec := range dt { - if uint8(dec.entry>>8) == byte(sym) { - fmt.Fprintf(w, "symbol %x has decoder, but no encoder\n", sym) - errs++ - break - } - } - if errs == 0 { - broken-- - } - continue - } - // Unused bits in input - ub := tablelog - enc.nBits - top := enc.val << ub - // decoder looks at top bits. - dec := dt[top] - if uint8(dec.entry) != enc.nBits { - fmt.Fprintf(w, "symbol 0x%x bit size mismatch (enc: %d, dec:%d).\n", sym, enc.nBits, uint8(dec.entry)) - errs++ - } - if uint8(dec.entry>>8) != uint8(sym) { - fmt.Fprintf(w, "symbol 0x%x decoder output mismatch (enc: %d, dec:%d).\n", sym, sym, uint8(dec.entry>>8)) - errs++ - } - if errs > 0 { - fmt.Fprintf(w, "%d errors in base, stopping\n", errs) - continue - } - // Ensure that all combinations are covered. - for i := uint16(0); i < (1 << ub); i++ { - vval := top | i - dec := dt[vval] - if uint8(dec.entry) != enc.nBits { - fmt.Fprintf(w, "symbol 0x%x bit size mismatch (enc: %d, dec:%d).\n", vval, enc.nBits, uint8(dec.entry)) - errs++ - } - if uint8(dec.entry>>8) != uint8(sym) { - fmt.Fprintf(w, "symbol 0x%x decoder output mismatch (enc: %d, dec:%d).\n", vval, sym, uint8(dec.entry>>8)) - errs++ - } - if errs > 20 { - fmt.Fprintf(w, "%d errors, stopping\n", errs) - break - } - } - if errs == 0 { - ok++ - broken-- - } - } - if broken > 0 { - fmt.Fprintf(w, "%d broken, %d ok\n", broken, ok) - } -} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go deleted file mode 100644 index ba7e8e6b0..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ /dev/null @@ -1,226 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. -package huff0 - -import ( - "errors" - "fmt" - - "github.com/klauspost/compress/internal/cpuinfo" -) - -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog > 8. -// -//go:noescape -func decompress4x_main_loop_amd64(ctx *decompress4xContext) - -// decompress4x_8b_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. -// -//go:noescape -func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) - -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 - -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := 0; i < 3; i++ { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress1X when tablelog > 8. -// -//go:noescape -func decompress1x_main_loop_amd64(ctx *decompress1xContext) - -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation -// of Decompress1X when tablelog > 8. -// -//go:noescape -func decompress1x_main_loop_bmi2(ctx *decompress1xContext) - -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int -} - -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) - } - return dst, br.close() -} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s deleted file mode 100644 index c4c7ab2d1..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ /dev/null @@ -1,830 +0,0 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. - -//go:build amd64 && !appengine && !noasm && gc - -// func decompress4x_main_loop_amd64(ctx *decompress4xContext) -TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br0.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVW AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x02, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) - RET - -// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), CX - MOVBQZX 8(CX), DI - MOVQ 16(CX), BX - MOVQ 48(CX), SI - MOVQ 24(CX), R8 - MOVQ 32(CX), R9 - MOVQ (CX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ (R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 24(R10) - ORQ R14, R11 - - // exhausted += (br0.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // val3 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 48(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 72(R10) - ORQ R14, R11 - - // exhausted += (br1.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // val3 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 96(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 120(R10) - ORQ R14, R11 - - // exhausted += (br2.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // val3 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 144(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 168(R10) - ORQ R14, R11 - - // exhausted += (br3.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // val3 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVL AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x04, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) - RET - -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_amd64(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_1_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_2_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) - RET - - // Report error -error_max_decoded_size_exceeded: - MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET - -// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -// Requires: BMI2 -TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_1_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_2_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) - RET - - // Report error -error_max_decoded_size_exceeded: - MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go deleted file mode 100644 index 908c17de6..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ /dev/null @@ -1,299 +0,0 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - -// This file contains a generic implementation of Decoder.Decompress4X. -package huff0 - -import ( - "errors" - "fmt" -) - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - if use8BitTables && d.actualTableLog <= 8 { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := 0; i < 3; i++ { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - buf := d.buffer() - var off uint8 - var decoded int - - // Decode 2 values from each decoder/loop. - const bufoff = 256 - for { - if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { - break - } - - { - const stream = 0 - const stream2 = 1 - br[stream].fillFast() - br[stream2].fillFast() - - val := br[stream].peekBitsFast(d.actualTableLog) - val2 := br[stream2].peekBitsFast(d.actualTableLog) - v := single[val&tlMask] - v2 := single[val2&tlMask] - br[stream].advance(uint8(v.entry)) - br[stream2].advance(uint8(v2.entry)) - buf[stream][off] = uint8(v.entry >> 8) - buf[stream2][off] = uint8(v2.entry >> 8) - - val = br[stream].peekBitsFast(d.actualTableLog) - val2 = br[stream2].peekBitsFast(d.actualTableLog) - v = single[val&tlMask] - v2 = single[val2&tlMask] - br[stream].advance(uint8(v.entry)) - br[stream2].advance(uint8(v2.entry)) - buf[stream][off+1] = uint8(v.entry >> 8) - buf[stream2][off+1] = uint8(v2.entry >> 8) - } - - { - const stream = 2 - const stream2 = 3 - br[stream].fillFast() - br[stream2].fillFast() - - val := br[stream].peekBitsFast(d.actualTableLog) - val2 := br[stream2].peekBitsFast(d.actualTableLog) - v := single[val&tlMask] - v2 := single[val2&tlMask] - br[stream].advance(uint8(v.entry)) - br[stream2].advance(uint8(v2.entry)) - buf[stream][off] = uint8(v.entry >> 8) - buf[stream2][off] = uint8(v2.entry >> 8) - - val = br[stream].peekBitsFast(d.actualTableLog) - val2 = br[stream2].peekBitsFast(d.actualTableLog) - v = single[val&tlMask] - v2 = single[val2&tlMask] - br[stream].advance(uint8(v.entry)) - br[stream2].advance(uint8(v2.entry)) - buf[stream][off+1] = uint8(v.entry >> 8) - buf[stream2][off+1] = uint8(v2.entry >> 8) - } - - off += 2 - - if off == 0 { - if bufoff > dstEvery { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 1") - } - // There must at least be 3 buffers left. - if len(out)-bufoff < dstEvery*3 { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 2") - } - //copy(out, buf[0][:]) - //copy(out[dstEvery:], buf[1][:]) - //copy(out[dstEvery*2:], buf[2][:]) - //copy(out[dstEvery*3:], buf[3][:]) - *(*[bufoff]byte)(out) = buf[0] - *(*[bufoff]byte)(out[dstEvery:]) = buf[1] - *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] - *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] - out = out[bufoff:] - decoded += bufoff * 4 - } - } - if off > 0 { - ioff := int(off) - if len(out) < dstEvery*3+ioff { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 3") - } - copy(out, buf[0][:off]) - copy(out[dstEvery:], buf[1][:off]) - copy(out[dstEvery*2:], buf[2][:off]) - copy(out[dstEvery*3:], buf[3][:off]) - decoded += int(off) * 4 - out = out[off:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - d.bufs.Put(buf) - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - d.bufs.Put(buf) - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if use8BitTables && d.actualTableLog <= 8 { - return d.decompress1X8Bit(dst, src) - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:0] - - // Avoid bounds check by always having full sized table. - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - dt := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - bufs := d.buffer() - buf := &bufs[0] - var off uint8 - - for br.off >= 8 { - br.fillFast() - v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - // Refill - br.fillFast() - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - d.bufs.Put(bufs) - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - - if len(dst)+int(off) > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:off]...) - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if false && br.bitsRead >= 32 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value = (br.value << 32) | uint64(low) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value = (br.value << 8) | uint64(br.in[br.off-1]) - br.bitsRead -= 8 - br.off-- - } - } - } - if len(dst) >= maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) - } - d.bufs.Put(bufs) - return dst, br.close() -} diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go deleted file mode 100644 index 77ecd68e0..000000000 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ /dev/null @@ -1,337 +0,0 @@ -// Package huff0 provides fast huffman encoding as used in zstd. -// -// See README.md at https://github.com/klauspost/compress/tree/master/huff0 for details. -package huff0 - -import ( - "errors" - "fmt" - "math" - "math/bits" - "sync" - - "github.com/klauspost/compress/fse" -) - -const ( - maxSymbolValue = 255 - - // zstandard limits tablelog to 11, see: - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#huffman-tree-description - tableLogMax = 11 - tableLogDefault = 11 - minTablelog = 5 - huffNodesLen = 512 - - // BlockSizeMax is maximum input size for a single block uncompressed. - BlockSizeMax = 1<<18 - 1 -) - -var ( - // ErrIncompressible is returned when input is judged to be too hard to compress. - ErrIncompressible = errors.New("input is not compressible") - - // ErrUseRLE is returned from the compressor when the input is a single byte value repeated. - ErrUseRLE = errors.New("input is single value repeated") - - // ErrTooBig is return if input is too large for a single block. - ErrTooBig = errors.New("input too big") - - // ErrMaxDecodedSizeExceeded is return if input is too large for a single block. - ErrMaxDecodedSizeExceeded = errors.New("maximum output size exceeded") -) - -type ReusePolicy uint8 - -const ( - // ReusePolicyAllow will allow reuse if it produces smaller output. - ReusePolicyAllow ReusePolicy = iota - - // ReusePolicyPrefer will re-use aggressively if possible. - // This will not check if a new table will produce smaller output, - // except if the current table is impossible to use or - // compressed output is bigger than input. - ReusePolicyPrefer - - // ReusePolicyNone will disable re-use of tables. - // This is slightly faster than ReusePolicyAllow but may produce larger output. - ReusePolicyNone - - // ReusePolicyMust must allow reuse and produce smaller output. - ReusePolicyMust -) - -type Scratch struct { - count [maxSymbolValue + 1]uint32 - - // Per block parameters. - // These can be used to override compression parameters of the block. - // Do not touch, unless you know what you are doing. - - // Out is output buffer. - // If the scratch is re-used before the caller is done processing the output, - // set this field to nil. - // Otherwise the output buffer will be re-used for next Compression/Decompression step - // and allocation will be avoided. - Out []byte - - // OutTable will contain the table data only, if a new table has been generated. - // Slice of the returned data. - OutTable []byte - - // OutData will contain the compressed data. - // Slice of the returned data. - OutData []byte - - // MaxDecodedSize will set the maximum allowed output size. - // This value will automatically be set to BlockSizeMax if not set. - // Decoders will return ErrMaxDecodedSizeExceeded is this limit is exceeded. - MaxDecodedSize int - - srcLen int - - // MaxSymbolValue will override the maximum symbol value of the next block. - MaxSymbolValue uint8 - - // TableLog will attempt to override the tablelog for the next block. - // Must be <= 11 and >= 5. - TableLog uint8 - - // Reuse will specify the reuse policy - Reuse ReusePolicy - - // WantLogLess allows to specify a log 2 reduction that should at least be achieved, - // otherwise the block will be returned as incompressible. - // The reduction should then at least be (input size >> WantLogLess) - // If WantLogLess == 0 any improvement will do. - WantLogLess uint8 - - symbolLen uint16 // Length of active part of the symbol table. - maxCount int // count of the most probable symbol - clearCount bool // clear count - actualTableLog uint8 // Selected tablelog. - prevTableLog uint8 // Tablelog for previous table - prevTable cTable // Table used for previous compression. - cTable cTable // compression table - dt dTable // decompression table - nodes []nodeElt - tmpOut [4][]byte - fse *fse.Scratch - decPool sync.Pool // *[4][256]byte buffers. - huffWeight [maxSymbolValue + 1]byte -} - -// TransferCTable will transfer the previously used compression table. -func (s *Scratch) TransferCTable(src *Scratch) { - if cap(s.prevTable) < len(src.prevTable) { - s.prevTable = make(cTable, 0, maxSymbolValue+1) - } - s.prevTable = s.prevTable[:len(src.prevTable)] - copy(s.prevTable, src.prevTable) - s.prevTableLog = src.prevTableLog -} - -func (s *Scratch) prepare(in []byte) (*Scratch, error) { - if len(in) > BlockSizeMax { - return nil, ErrTooBig - } - if s == nil { - s = &Scratch{} - } - if s.MaxSymbolValue == 0 { - s.MaxSymbolValue = maxSymbolValue - } - if s.TableLog == 0 { - s.TableLog = tableLogDefault - } - if s.TableLog > tableLogMax || s.TableLog < minTablelog { - return nil, fmt.Errorf(" invalid tableLog %d (%d -> %d)", s.TableLog, minTablelog, tableLogMax) - } - if s.MaxDecodedSize <= 0 || s.MaxDecodedSize > BlockSizeMax { - s.MaxDecodedSize = BlockSizeMax - } - if s.clearCount && s.maxCount == 0 { - for i := range s.count { - s.count[i] = 0 - } - s.clearCount = false - } - if cap(s.Out) == 0 { - s.Out = make([]byte, 0, len(in)) - } - s.Out = s.Out[:0] - - s.OutTable = nil - s.OutData = nil - if cap(s.nodes) < huffNodesLen+1 { - s.nodes = make([]nodeElt, 0, huffNodesLen+1) - } - s.nodes = s.nodes[:0] - if s.fse == nil { - s.fse = &fse.Scratch{} - } - s.srcLen = len(in) - - return s, nil -} - -type cTable []cTableEntry - -func (c cTable) write(s *Scratch) error { - var ( - // precomputed conversion table - bitsToWeight [tableLogMax + 1]byte - huffLog = s.actualTableLog - // last weight is not saved. - maxSymbolValue = uint8(s.symbolLen - 1) - huffWeight = s.huffWeight[:256] - ) - const ( - maxFSETableLog = 6 - ) - // convert to weight - bitsToWeight[0] = 0 - for n := uint8(1); n < huffLog+1; n++ { - bitsToWeight[n] = huffLog + 1 - n - } - - // Acquire histogram for FSE. - hist := s.fse.Histogram() - hist = hist[:256] - for i := range hist[:16] { - hist[i] = 0 - } - for n := uint8(0); n < maxSymbolValue; n++ { - v := bitsToWeight[c[n].nBits] & 15 - huffWeight[n] = v - hist[v]++ - } - - // FSE compress if feasible. - if maxSymbolValue >= 2 { - huffMaxCnt := uint32(0) - huffMax := uint8(0) - for i, v := range hist[:16] { - if v == 0 { - continue - } - huffMax = byte(i) - if v > huffMaxCnt { - huffMaxCnt = v - } - } - s.fse.HistogramFinished(huffMax, int(huffMaxCnt)) - s.fse.TableLog = maxFSETableLog - b, err := fse.Compress(huffWeight[:maxSymbolValue], s.fse) - if err == nil && len(b) < int(s.symbolLen>>1) { - s.Out = append(s.Out, uint8(len(b))) - s.Out = append(s.Out, b...) - return nil - } - // Unable to compress (RLE/uncompressible) - } - // write raw values as 4-bits (max : 15) - if maxSymbolValue > (256 - 128) { - // should not happen : likely means source cannot be compressed - return ErrIncompressible - } - op := s.Out - // special case, pack weights 4 bits/weight. - op = append(op, 128|(maxSymbolValue-1)) - // be sure it doesn't cause msan issue in final combination - huffWeight[maxSymbolValue] = 0 - for n := uint16(0); n < uint16(maxSymbolValue); n += 2 { - op = append(op, (huffWeight[n]<<4)|huffWeight[n+1]) - } - s.Out = op - return nil -} - -func (c cTable) estTableSize(s *Scratch) (sz int, err error) { - var ( - // precomputed conversion table - bitsToWeight [tableLogMax + 1]byte - huffLog = s.actualTableLog - // last weight is not saved. - maxSymbolValue = uint8(s.symbolLen - 1) - huffWeight = s.huffWeight[:256] - ) - const ( - maxFSETableLog = 6 - ) - // convert to weight - bitsToWeight[0] = 0 - for n := uint8(1); n < huffLog+1; n++ { - bitsToWeight[n] = huffLog + 1 - n - } - - // Acquire histogram for FSE. - hist := s.fse.Histogram() - hist = hist[:256] - for i := range hist[:16] { - hist[i] = 0 - } - for n := uint8(0); n < maxSymbolValue; n++ { - v := bitsToWeight[c[n].nBits] & 15 - huffWeight[n] = v - hist[v]++ - } - - // FSE compress if feasible. - if maxSymbolValue >= 2 { - huffMaxCnt := uint32(0) - huffMax := uint8(0) - for i, v := range hist[:16] { - if v == 0 { - continue - } - huffMax = byte(i) - if v > huffMaxCnt { - huffMaxCnt = v - } - } - s.fse.HistogramFinished(huffMax, int(huffMaxCnt)) - s.fse.TableLog = maxFSETableLog - b, err := fse.Compress(huffWeight[:maxSymbolValue], s.fse) - if err == nil && len(b) < int(s.symbolLen>>1) { - sz += 1 + len(b) - return sz, nil - } - // Unable to compress (RLE/uncompressible) - } - // write raw values as 4-bits (max : 15) - if maxSymbolValue > (256 - 128) { - // should not happen : likely means source cannot be compressed - return 0, ErrIncompressible - } - // special case, pack weights 4 bits/weight. - sz += 1 + int(maxSymbolValue/2) - return sz, nil -} - -// estimateSize returns the estimated size in bytes of the input represented in the -// histogram supplied. -func (c cTable) estimateSize(hist []uint32) int { - nbBits := uint32(7) - for i, v := range c[:len(hist)] { - nbBits += uint32(v.nBits) * hist[i] - } - return int(nbBits >> 3) -} - -// minSize returns the minimum possible size considering the shannon limit. -func (s *Scratch) minSize(total int) int { - nbBits := float64(7) - fTotal := float64(total) - for _, v := range s.count[:s.symbolLen] { - n := float64(v) - if n > 0 { - nbBits += math.Log2(fTotal/n) * n - } - } - return int(nbBits) >> 3 -} - -func highBit32(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go deleted file mode 100644 index 3954c5121..000000000 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go +++ /dev/null @@ -1,34 +0,0 @@ -// Package cpuinfo gives runtime info about the current CPU. -// -// This is a very limited module meant for use internally -// in this project. For more versatile solution check -// https://github.com/klauspost/cpuid. -package cpuinfo - -// HasBMI1 checks whether an x86 CPU supports the BMI1 extension. -func HasBMI1() bool { - return hasBMI1 -} - -// HasBMI2 checks whether an x86 CPU supports the BMI2 extension. -func HasBMI2() bool { - return hasBMI2 -} - -// DisableBMI2 will disable BMI2, for testing purposes. -// Call returned function to restore previous state. -func DisableBMI2() func() { - old := hasBMI2 - hasBMI2 = false - return func() { - hasBMI2 = old - } -} - -// HasBMI checks whether an x86 CPU supports both BMI1 and BMI2 extensions. -func HasBMI() bool { - return HasBMI1() && HasBMI2() -} - -var hasBMI1 bool -var hasBMI2 bool diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go deleted file mode 100644 index e802579c4..000000000 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -package cpuinfo - -// go:noescape -func x86extensions() (bmi1, bmi2 bool) - -func init() { - hasBMI1, hasBMI2 = x86extensions() -} diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s deleted file mode 100644 index 4465fbe9e..000000000 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s +++ /dev/null @@ -1,36 +0,0 @@ -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" -#include "funcdata.h" -#include "go_asm.h" - -TEXT ·x86extensions(SB), NOSPLIT, $0 - // 1. determine max EAX value - XORQ AX, AX - CPUID - - CMPQ AX, $7 - JB unsupported - - // 2. EAX = 7, ECX = 0 --- see Table 3-8 "Information Returned by CPUID Instruction" - MOVQ $7, AX - MOVQ $0, CX - CPUID - - BTQ $3, BX // bit 3 = BMI1 - SETCS AL - - BTQ $8, BX // bit 8 = BMI2 - SETCS AH - - MOVB AL, bmi1+0(FP) - MOVB AH, bmi2+1(FP) - RET - -unsupported: - XORQ AX, AX - MOVB AL, bmi1+0(FP) - MOVB AL, bmi2+1(FP) - RET diff --git a/vendor/github.com/klauspost/compress/internal/race/norace.go b/vendor/github.com/klauspost/compress/internal/race/norace.go deleted file mode 100644 index affbbbb59..000000000 --- a/vendor/github.com/klauspost/compress/internal/race/norace.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !race - -package race - -func ReadSlice[T any](s []T) { -} - -func WriteSlice[T any](s []T) { -} diff --git a/vendor/github.com/klauspost/compress/internal/race/race.go b/vendor/github.com/klauspost/compress/internal/race/race.go deleted file mode 100644 index f5e240dcd..000000000 --- a/vendor/github.com/klauspost/compress/internal/race/race.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build race - -package race - -import ( - "runtime" - "unsafe" -) - -func ReadSlice[T any](s []T) { - if len(s) == 0 { - return - } - runtime.RaceReadRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0]))) -} - -func WriteSlice[T any](s []T) { - if len(s) == 0 { - return - } - runtime.RaceWriteRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0]))) -} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/LICENSE b/vendor/github.com/klauspost/compress/internal/snapref/LICENSE deleted file mode 100644 index 6050c10f4..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/internal/snapref/decode.go b/vendor/github.com/klauspost/compress/internal/snapref/decode.go deleted file mode 100644 index 40796a49d..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode.go +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snapref - -import ( - "encoding/binary" - "errors" - "io" -) - -var ( - // ErrCorrupt reports that the input is invalid. - ErrCorrupt = errors.New("snappy: corrupt input") - // ErrTooLarge reports that the uncompressed length is too large. - ErrTooLarge = errors.New("snappy: decoded block is too large") - // ErrUnsupported reports that the input isn't supported. - ErrUnsupported = errors.New("snappy: unsupported input") - - errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") -) - -// DecodedLen returns the length of the decoded block. -func DecodedLen(src []byte) (int, error) { - v, _, err := decodedLen(src) - return v, err -} - -// decodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func decodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { - return 0, 0, ErrCorrupt - } - - const wordSize = 32 << (^uint(0) >> 32 & 1) - if wordSize == 32 && v > 0x7fffffff { - return 0, 0, ErrTooLarge - } - return int(v), n, nil -} - -const ( - decodeErrCodeCorrupt = 1 - decodeErrCodeUnsupportedLiteralLength = 2 -) - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// Decode handles the Snappy block format, not the Snappy stream format. -func Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if dLen <= len(dst) { - dst = dst[:dLen] - } else { - dst = make([]byte, dLen) - } - switch decode(dst, src[s:]) { - case 0: - return dst, nil - case decodeErrCodeUnsupportedLiteralLength: - return nil, errUnsupportedLiteralLength - } - return nil, ErrCorrupt -} - -// NewReader returns a new Reader that decompresses from r, using the framing -// format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -func NewReader(r io.Reader) *Reader { - return &Reader{ - r: r, - decoded: make([]byte, maxBlockSize), - buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), - } -} - -// Reader is an io.Reader that can read Snappy-compressed bytes. -// -// Reader handles the Snappy stream format, not the Snappy block format. -type Reader struct { - r io.Reader - err error - decoded []byte - buf []byte - // decoded[i:j] contains decoded bytes that have not yet been passed on. - i, j int - readHeader bool -} - -// Reset discards any buffered data, resets all state, and switches the Snappy -// reader to read from r. This permits reusing a Reader rather than allocating -// a new one. -func (r *Reader) Reset(reader io.Reader) { - r.r = reader - r.err = nil - r.i = 0 - r.j = 0 - r.readHeader = false -} - -func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { - if _, r.err = io.ReadFull(r.r, p); r.err != nil { - if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - } - return false - } - return true -} - -func (r *Reader) fill() error { - for r.i >= r.j { - if !r.readFull(r.buf[:4], true) { - return r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - if chunkLen > len(r.buf) { - r.err = ErrUnsupported - return r.err - } - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf, false) { - return r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - n, err := DecodedLen(buf) - if err != nil { - r.err = err - return r.err - } - if n > len(r.decoded) { - r.err = ErrCorrupt - return r.err - } - if _, err := Decode(r.decoded, buf); err != nil { - r.err = err - return r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeUncompressedData: - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return r.err - } - buf := r.buf[:checksumSize] - if !r.readFull(buf, false) { - return r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n := chunkLen - checksumSize - if n > len(r.decoded) { - r.err = ErrCorrupt - return r.err - } - if !r.readFull(r.decoded[:n], false) { - return r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return r.err - } - if !r.readFull(r.buf[:len(magicBody)], false) { - return r.err - } - for i := 0; i < len(magicBody); i++ { - if r.buf[i] != magicBody[i] { - r.err = ErrCorrupt - return r.err - } - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - r.err = ErrUnsupported - return r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if !r.readFull(r.buf[:chunkLen], false) { - return r.err - } - } - - return nil -} - -// Read satisfies the io.Reader interface. -func (r *Reader) Read(p []byte) (int, error) { - if r.err != nil { - return 0, r.err - } - - if err := r.fill(); err != nil { - return 0, err - } - - n := copy(p, r.decoded[r.i:r.j]) - r.i += n - return n, nil -} - -// ReadByte satisfies the io.ByteReader interface. -func (r *Reader) ReadByte() (byte, error) { - if r.err != nil { - return 0, r.err - } - - if err := r.fill(); err != nil { - return 0, err - } - - c := r.decoded[r.i] - r.i++ - return c, nil -} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/decode_other.go b/vendor/github.com/klauspost/compress/internal/snapref/decode_other.go deleted file mode 100644 index 77395a6b8..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode_other.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snapref - -// decode writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read, and that len(dst) -// equals that length. -// -// It returns 0 on success or a decodeErrCodeXxx error code on failure. -func decode(dst, src []byte) int { - var d, s, offset, length int - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - length = int(x) + 1 - if length <= 0 { - return decodeErrCodeUnsupportedLiteralLength - } - if length > len(dst)-d || length > len(src)-s { - return decodeErrCodeCorrupt - } - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 4 + int(src[s-2])>>2&0x7 - offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - - case tagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - - case tagCopy4: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-5])>>2 - offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - } - - if offset <= 0 || d < offset || length > len(dst)-d { - return decodeErrCodeCorrupt - } - // Copy from an earlier sub-slice of dst to a later sub-slice. - // If no overlap, use the built-in copy: - if offset >= length { - copy(dst[d:d+length], dst[d-offset:]) - d += length - continue - } - - // Unlike the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - // - // We align the slices into a and b and show the compiler they are the same size. - // This allows the loop to run without bounds checks. - a := dst[d : d+length] - b := dst[d-offset:] - b = b[:len(a)] - for i := range a { - a[i] = b[i] - } - d += length - } - if d != len(dst) { - return decodeErrCodeCorrupt - } - return 0 -} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode.go b/vendor/github.com/klauspost/compress/internal/snapref/encode.go deleted file mode 100644 index 13c6040a5..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode.go +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snapref - -import ( - "encoding/binary" - "errors" - "io" -) - -// Encode returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// Encode handles the Snappy block format, not the Snappy stream format. -func Encode(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - for len(src) > 0 { - p := src - src = nil - if len(p) > maxBlockSize { - p, src = p[:maxBlockSize], p[maxBlockSize:] - } - if len(p) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], p) - } else { - d += encodeBlock(dst[d:], p) - } - } - return dst[:d] -} - -// inputMargin is the minimum number of extra input bytes to keep, inside -// encodeBlock's inner loop. On some architectures, this margin lets us -// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) -// literals can be implemented as a single load to and store from a 16-byte -// register. That literal's actual length can be as short as 1 byte, so this -// can copy up to 15 bytes too much, but that's OK as subsequent iterations of -// the encoding loop will fix up the copy overrun, and this inputMargin ensures -// that we don't overrun the dst and src buffers. -const inputMargin = 16 - 1 - -// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that -// could be encoded with a copy tag. This is the minimum with respect to the -// algorithm used by encodeBlock, not a minimum enforced by the file format. -// -// The encoded output must start with at least a 1 byte literal, as there are -// no previous bytes to copy. A minimal (1 byte) copy after that, generated -// from an emitCopy call in encodeBlock's main loop, would require at least -// another inputMargin bytes, for the reason above: we want any emitLiteral -// calls inside encodeBlock's main loop to use the fast path if possible, which -// requires being able to overrun by inputMargin bytes. Thus, -// minNonLiteralBlockSize equals 1 + 1 + inputMargin. -// -// The C++ code doesn't use this exact threshold, but it could, as discussed at -// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion -// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an -// optimization. It should not affect the encoded form. This is tested by -// TestSameEncodingAsCppShortCopies. -const minNonLiteralBlockSize = 1 + 1 + inputMargin - -// MaxEncodedLen returns the maximum length of a snappy block, given its -// uncompressed length. -// -// It will return a negative value if srcLen is too large to encode. -func MaxEncodedLen(srcLen int) int { - n := uint64(srcLen) - if n > 0xffffffff { - return -1 - } - // Compressed data can be defined as: - // compressed := item* literal* - // item := literal* copy - // - // The trailing literal sequence has a space blowup of at most 62/60 - // since a literal of length 60 needs one tag byte + one extra byte - // for length information. - // - // Item blowup is trickier to measure. Suppose the "copy" op copies - // 4 bytes of data. Because of a special check in the encoding code, - // we produce a 4-byte copy only if the offset is < 65536. Therefore - // the copy op takes 3 bytes to encode, and this type of item leads - // to at most the 62/60 blowup for representing literals. - // - // Suppose the "copy" op copies 5 bytes of data. If the offset is big - // enough, it will take 5 bytes to encode the copy op. Therefore the - // worst case here is a one-byte literal followed by a five-byte copy. - // That is, 6 bytes of input turn into 7 bytes of "compressed" data. - // - // This last factor dominates the blowup, so the final estimate is: - n = 32 + n + n/6 - if n > 0xffffffff { - return -1 - } - return int(n) -} - -var errClosed = errors.New("snappy: Writer is closed") - -// NewWriter returns a new Writer that compresses to w. -// -// The Writer returned does not buffer writes. There is no need to Flush or -// Close such a Writer. -// -// Deprecated: the Writer returned is not suitable for many small writes, only -// for few large writes. Use NewBufferedWriter instead, which is efficient -// regardless of the frequency and shape of the writes, and remember to Close -// that Writer when done. -func NewWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - obuf: make([]byte, obufLen), - } -} - -// NewBufferedWriter returns a new Writer that compresses to w, using the -// framing format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -// -// The Writer returned buffers writes. Users must call Close to guarantee all -// data has been forwarded to the underlying io.Writer. They may also call -// Flush zero or more times before calling Close. -func NewBufferedWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - ibuf: make([]byte, 0, maxBlockSize), - obuf: make([]byte, obufLen), - } -} - -// Writer is an io.Writer that can write Snappy-compressed bytes. -// -// Writer handles the Snappy stream format, not the Snappy block format. -type Writer struct { - w io.Writer - err error - - // ibuf is a buffer for the incoming (uncompressed) bytes. - // - // Its use is optional. For backwards compatibility, Writers created by the - // NewWriter function have ibuf == nil, do not buffer incoming bytes, and - // therefore do not need to be Flush'ed or Close'd. - ibuf []byte - - // obuf is a buffer for the outgoing (compressed) bytes. - obuf []byte - - // wroteStreamHeader is whether we have written the stream header. - wroteStreamHeader bool -} - -// Reset discards the writer's state and switches the Snappy writer to write to -// w. This permits reusing a Writer rather than allocating a new one. -func (w *Writer) Reset(writer io.Writer) { - w.w = writer - w.err = nil - if w.ibuf != nil { - w.ibuf = w.ibuf[:0] - } - w.wroteStreamHeader = false -} - -// Write satisfies the io.Writer interface. -func (w *Writer) Write(p []byte) (nRet int, errRet error) { - if w.ibuf == nil { - // Do not buffer incoming bytes. This does not perform or compress well - // if the caller of Writer.Write writes many small slices. This - // behavior is therefore deprecated, but still supported for backwards - // compatibility with code that doesn't explicitly Flush or Close. - return w.write(p) - } - - // The remainder of this method is based on bufio.Writer.Write from the - // standard library. - - for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { - var n int - if len(w.ibuf) == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, _ = w.write(p) - } else { - n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - w.Flush() - } - nRet += n - p = p[n:] - } - if w.err != nil { - return nRet, w.err - } - n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - nRet += n - return nRet, nil -} - -func (w *Writer) write(p []byte) (nRet int, errRet error) { - if w.err != nil { - return 0, w.err - } - for len(p) > 0 { - obufStart := len(magicChunk) - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - copy(w.obuf, magicChunk) - obufStart = 0 - } - - var uncompressed []byte - if len(p) > maxBlockSize { - uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] - } else { - uncompressed, p = p, nil - } - checksum := crc(uncompressed) - - // Compress the buffer, discarding the result if the improvement - // isn't at least 12.5%. - compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) - chunkType := uint8(chunkTypeCompressedData) - chunkLen := 4 + len(compressed) - obufEnd := obufHeaderLen + len(compressed) - if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { - chunkType = chunkTypeUncompressedData - chunkLen = 4 + len(uncompressed) - obufEnd = obufHeaderLen - } - - // Fill in the per-chunk header that comes before the body. - w.obuf[len(magicChunk)+0] = chunkType - w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) - w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) - w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) - w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) - w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) - w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) - w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) - - if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { - w.err = err - return nRet, err - } - if chunkType == chunkTypeUncompressedData { - if _, err := w.w.Write(uncompressed); err != nil { - w.err = err - return nRet, err - } - } - nRet += len(uncompressed) - } - return nRet, nil -} - -// Flush flushes the Writer to its underlying io.Writer. -func (w *Writer) Flush() error { - if w.err != nil { - return w.err - } - if len(w.ibuf) == 0 { - return nil - } - w.write(w.ibuf) - w.ibuf = w.ibuf[:0] - return w.err -} - -// Close calls Flush and then closes the Writer. -func (w *Writer) Close() error { - w.Flush() - ret := w.err - if w.err == nil { - w.err = errClosed - } - return ret -} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go deleted file mode 100644 index 2754bac6f..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snapref - -func load32(b []byte, i int) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load64(b []byte, i int) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= len(lit) && len(lit) <= 65536 -func emitLiteral(dst, lit []byte) int { - i, n := 0, uint(len(lit)-1) - switch { - case n < 60: - dst[0] = uint8(n)<<2 | tagLiteral - i = 1 - case n < 1<<8: - dst[0] = 60<<2 | tagLiteral - dst[1] = uint8(n) - i = 2 - default: - dst[0] = 61<<2 | tagLiteral - dst[1] = uint8(n) - dst[2] = uint8(n >> 8) - i = 3 - } - return i + copy(dst[i:], lit) -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= 65535 -// 4 <= length && length <= 65535 -func emitCopy(dst []byte, offset, length int) int { - i := 0 - // The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The - // threshold for this loop is a little higher (at 68 = 64 + 4), and the - // length emitted down below is a little lower (at 60 = 64 - 4), because - // it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed - // by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as - // a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as - // 3+3 bytes). The magic 4 in the 64±4 is because the minimum length for a - // tagCopy1 op is 4 bytes, which is why a length 3 copy has to be an - // encodes-as-3-bytes tagCopy2 instead of an encodes-as-2-bytes tagCopy1. - for length >= 68 { - // Emit a length 64 copy, encoded as 3 bytes. - dst[i+0] = 63<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - i += 3 - length -= 64 - } - if length > 64 { - // Emit a length 60 copy, encoded as 3 bytes. - dst[i+0] = 59<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - i += 3 - length -= 60 - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[i+0] = uint8(length-1)<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - return i + 3 - } - // Emit the remaining copy, encoded as 2 bytes. - dst[i+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - dst[i+1] = uint8(offset) - return i + 2 -} - -func hash(u, shift uint32) uint32 { - return (u * 0x1e35a7bd) >> shift -} - -// EncodeBlockInto exposes encodeBlock but checks dst size. -func EncodeBlockInto(dst, src []byte) (d int) { - if MaxEncodedLen(len(src)) > len(dst) { - return 0 - } - - // encodeBlock breaks on too big blocks, so split. - for len(src) > 0 { - p := src - src = nil - if len(p) > maxBlockSize { - p, src = p[:maxBlockSize], p[maxBlockSize:] - } - if len(p) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], p) - } else { - d += encodeBlock(dst[d:], p) - } - } - return d -} - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlock(dst, src []byte) (d int) { - // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. - // The table element type is uint16, as s < sLimit and sLimit < len(src) - // and len(src) <= maxBlockSize and maxBlockSize == 65536. - const ( - maxTableSize = 1 << 14 - // tableMask is redundant, but helps the compiler eliminate bounds - // checks. - tableMask = maxTableSize - 1 - ) - shift := uint32(32 - 8) - for tableSize := 1 << 8; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { - shift-- - } - // In Go, all array elements are zero-initialized, so there is no advantage - // to a smaller tableSize per se. However, it matches the C++ algorithm, - // and in the asm versions of this code, we can get away with zeroing only - // the first tableSize elements. - var table [maxTableSize]uint16 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - nextHash := hash(load32(src, s), shift) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := 32 - - nextS := s - candidate := 0 - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = int(table[nextHash&tableMask]) - table[nextHash&tableMask] = uint16(s) - nextHash = hash(load32(src, nextS), shift) - if load32(src, s) == load32(src, candidate) { - break - } - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - d += emitLiteral(dst[d:], src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - - // Extend the 4-byte match as long as possible. - // - // This is an inlined version of: - // s = extendMatch(src, candidate+4, s+4) - s += 4 - for i := candidate + 4; s < len(src) && src[i] == src[s]; i, s = i+1, s+1 { - } - - d += emitCopy(dst[d:], base-candidate, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load64(src, s-1) - prevHash := hash(uint32(x>>0), shift) - table[prevHash&tableMask] = uint16(s - 1) - currHash := hash(uint32(x>>8), shift) - candidate = int(table[currHash&tableMask]) - table[currHash&tableMask] = uint16(s) - if uint32(x>>8) != load32(src, candidate) { - nextHash = hash(uint32(x>>16), shift) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/snappy.go b/vendor/github.com/klauspost/compress/internal/snapref/snappy.go deleted file mode 100644 index 34d01f4aa..000000000 --- a/vendor/github.com/klauspost/compress/internal/snapref/snappy.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package snapref implements the Snappy compression format. It aims for very -// high speeds and reasonable compression. -// -// There are actually two Snappy formats: block and stream. They are related, -// but different: trying to decompress block-compressed data as a Snappy stream -// will fail, and vice versa. The block format is the Decode and Encode -// functions and the stream format is the Reader and Writer types. -// -// The block format, the more common case, is used when the complete size (the -// number of bytes) of the original data is known upfront, at the time -// compression starts. The stream format, also known as the framing format, is -// for when that isn't always true. -// -// The canonical, C++ implementation is at https://github.com/google/snappy and -// it only implements the block format. -package snapref - -import ( - "hash/crc32" -) - -/* -Each encoded block begins with the varint-encoded length of the decoded data, -followed by a sequence of chunks. Chunks begin and end on byte boundaries. The -first byte of each chunk is broken into its 2 least and 6 most significant bits -called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. -Zero means a literal tag. All other values mean a copy tag. - -For literal tags: - - If m < 60, the next 1 + m bytes are literal bytes. - - Otherwise, let n be the little-endian unsigned integer denoted by the next - m - 59 bytes. The next 1 + n bytes after that are literal bytes. - -For copy tags, length bytes are copied from offset bytes ago, in the style of -Lempel-Ziv compression algorithms. In particular: - - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). - The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 - of the offset. The next byte is bits 0-7 of the offset. - - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). - The length is 1 + m. The offset is the little-endian unsigned integer - denoted by the next 2 bytes. - - For l == 3, this tag is a legacy format that is no longer issued by most - encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in - [1, 65). The length is 1 + m. The offset is the little-endian unsigned - integer denoted by the next 4 bytes. -*/ -const ( - tagLiteral = 0x00 - tagCopy1 = 0x01 - tagCopy2 = 0x02 - tagCopy4 = 0x03 -) - -const ( - checksumSize = 4 - chunkHeaderSize = 4 - magicChunk = "\xff\x06\x00\x00" + magicBody - magicBody = "sNaPpY" - - // maxBlockSize is the maximum size of the input to encodeBlock. It is not - // part of the wire format per se, but some parts of the encoder assume - // that an offset fits into a uint16. - // - // Also, for the framing format (Writer type instead of Encode function), - // https://github.com/google/snappy/blob/master/framing_format.txt says - // that "the uncompressed data in a chunk must be no longer than 65536 - // bytes". - maxBlockSize = 65536 - - // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is - // hard coded to be a const instead of a variable, so that obufLen can also - // be a const. Their equivalence is confirmed by - // TestMaxEncodedLenOfMaxBlockSize. - maxEncodedLenOfMaxBlockSize = 76490 - - obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize - obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize -) - -const ( - chunkTypeCompressedData = 0x00 - chunkTypeUncompressedData = 0x01 - chunkTypePadding = 0xfe - chunkTypeStreamIdentifier = 0xff -) - -var crcTable = crc32.MakeTable(crc32.Castagnoli) - -// crc implements the checksum specified in section 3 of -// https://github.com/google/snappy/blob/master/framing_format.txt -func crc(b []byte) uint32 { - c := crc32.Update(0, crcTable, b) - return uint32(c>>15|c<<17) + 0xa282ead8 -} diff --git a/vendor/github.com/klauspost/compress/s2/.gitignore b/vendor/github.com/klauspost/compress/s2/.gitignore deleted file mode 100644 index 3a89c6e3e..000000000 --- a/vendor/github.com/klauspost/compress/s2/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -testdata/bench - -# These explicitly listed benchmark data files are for an obsolete version of -# snappy_test.go. -testdata/alice29.txt -testdata/asyoulik.txt -testdata/fireworks.jpeg -testdata/geo.protodata -testdata/html -testdata/html_x_4 -testdata/kppkn.gtb -testdata/lcet10.txt -testdata/paper-100k.pdf -testdata/plrabn12.txt -testdata/urls.10K diff --git a/vendor/github.com/klauspost/compress/s2/LICENSE b/vendor/github.com/klauspost/compress/s2/LICENSE deleted file mode 100644 index 1d2d645bd..000000000 --- a/vendor/github.com/klauspost/compress/s2/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. -Copyright (c) 2019 Klaus Post. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/s2/README.md b/vendor/github.com/klauspost/compress/s2/README.md deleted file mode 100644 index 8284bb081..000000000 --- a/vendor/github.com/klauspost/compress/s2/README.md +++ /dev/null @@ -1,1120 +0,0 @@ -# S2 Compression - -S2 is an extension of [Snappy](https://github.com/google/snappy). - -S2 is aimed for high throughput, which is why it features concurrent compression for bigger payloads. - -Decoding is compatible with Snappy compressed content, but content compressed with S2 cannot be decompressed by Snappy. -This means that S2 can seamlessly replace Snappy without converting compressed content. - -S2 can produce Snappy compatible output, faster and better than Snappy. -If you want full benefit of the changes you should use s2 without Snappy compatibility. - -S2 is designed to have high throughput on content that cannot be compressed. -This is important, so you don't have to worry about spending CPU cycles on already compressed data. - -## Benefits over Snappy - -* Better compression -* Adjustable compression (3 levels) -* Concurrent stream compression -* Faster decompression, even for Snappy compatible content -* Concurrent Snappy/S2 stream decompression -* Skip forward in compressed stream -* Random seeking with indexes -* Compatible with reading Snappy compressed content -* Smaller block size overhead on incompressible blocks -* Block concatenation -* Block Dictionary support -* Uncompressed stream mode -* Automatic stream size padding -* Snappy compatible block compression - -## Drawbacks over Snappy - -* Not optimized for 32 bit systems -* Streams use slightly more memory due to larger blocks and concurrency (configurable) - -# Usage - -Installation: `go get -u github.com/klauspost/compress/s2` - -Full package documentation: - -[![godoc][1]][2] - -[1]: https://godoc.org/github.com/klauspost/compress?status.svg -[2]: https://godoc.org/github.com/klauspost/compress/s2 - -## Compression - -```Go -func EncodeStream(src io.Reader, dst io.Writer) error { - enc := s2.NewWriter(dst) - _, err := io.Copy(enc, src) - if err != nil { - enc.Close() - return err - } - // Blocks until compression is done. - return enc.Close() -} -``` - -You should always call `enc.Close()`, otherwise you will leak resources and your encode will be incomplete. - -For the best throughput, you should attempt to reuse the `Writer` using the `Reset()` method. - -The Writer in S2 is always buffered, therefore `NewBufferedWriter` in Snappy can be replaced with `NewWriter` in S2. -It is possible to flush any buffered data using the `Flush()` method. -This will block until all data sent to the encoder has been written to the output. - -S2 also supports the `io.ReaderFrom` interface, which will consume all input from a reader. - -As a final method to compress data, if you have a single block of data you would like to have encoded as a stream, -a slightly more efficient method is to use the `EncodeBuffer` method. -This will take ownership of the buffer until the stream is closed. - -```Go -func EncodeStream(src []byte, dst io.Writer) error { - enc := s2.NewWriter(dst) - // The encoder owns the buffer until Flush or Close is called. - err := enc.EncodeBuffer(buf) - if err != nil { - enc.Close() - return err - } - // Blocks until compression is done. - return enc.Close() -} -``` - -Each call to `EncodeBuffer` will result in discrete blocks being created without buffering, -so it should only be used a single time per stream. -If you need to write several blocks, you should use the regular io.Writer interface. - - -## Decompression - -```Go -func DecodeStream(src io.Reader, dst io.Writer) error { - dec := s2.NewReader(src) - _, err := io.Copy(dst, dec) - return err -} -``` - -Similar to the Writer, a Reader can be reused using the `Reset` method. - -For the best possible throughput, there is a `EncodeBuffer(buf []byte)` function available. -However, it requires that the provided buffer isn't used after it is handed over to S2 and until the stream is flushed or closed. - -For smaller data blocks, there is also a non-streaming interface: `Encode()`, `EncodeBetter()` and `Decode()`. -Do however note that these functions (similar to Snappy) does not provide validation of data, -so data corruption may be undetected. Stream encoding provides CRC checks of data. - -It is possible to efficiently skip forward in a compressed stream using the `Skip()` method. -For big skips the decompressor is able to skip blocks without decompressing them. - -## Single Blocks - -Similar to Snappy S2 offers single block compression. -Blocks do not offer the same flexibility and safety as streams, -but may be preferable for very small payloads, less than 100K. - -Using a simple `dst := s2.Encode(nil, src)` will compress `src` and return the compressed result. -It is possible to provide a destination buffer. -If the buffer has a capacity of `s2.MaxEncodedLen(len(src))` it will be used. -If not a new will be allocated. - -Alternatively `EncodeBetter`/`EncodeBest` can also be used for better, but slightly slower compression. - -Similarly to decompress a block you can use `dst, err := s2.Decode(nil, src)`. -Again an optional destination buffer can be supplied. -The `s2.DecodedLen(src)` can be used to get the minimum capacity needed. -If that is not satisfied a new buffer will be allocated. - -Block function always operate on a single goroutine since it should only be used for small payloads. - -# Commandline tools - -Some very simply commandline tools are provided; `s2c` for compression and `s2d` for decompression. - -Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/compress/releases). - -Installing then requires Go to be installed. To install them, use: - -`go install github.com/klauspost/compress/s2/cmd/s2c@latest && go install github.com/klauspost/compress/s2/cmd/s2d@latest` - -To build binaries to the current folder use: - -`go build github.com/klauspost/compress/s2/cmd/s2c && go build github.com/klauspost/compress/s2/cmd/s2d` - - -## s2c - -``` -Usage: s2c [options] file1 file2 - -Compresses all files supplied as input separately. -Output files are written as 'filename.ext.s2' or 'filename.ext.snappy'. -By default output files will be overwritten. -Use - as the only file name to read from stdin and write to stdout. - -Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt -Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt - -File names beginning with 'http://' and 'https://' will be downloaded and compressed. -Only http response code 200 is accepted. - -Options: - -bench int - Run benchmark n times. No output will be written - -blocksize string - Max block size. Examples: 64K, 256K, 1M, 4M. Must be power of two and <= 4MB (default "4M") - -c Write all output to stdout. Multiple input files will be concatenated - -cpu int - Compress using this amount of threads (default 32) - -faster - Compress faster, but with a minor compression loss - -help - Display help - -index - Add seek index (default true) - -o string - Write output to another file. Single input file only - -pad string - Pad size to a multiple of this value, Examples: 500, 64K, 256K, 1M, 4M, etc (default "1") - -q Don't write any output to terminal, except errors - -rm - Delete source file(s) after successful compression - -safe - Do not overwrite output files - -slower - Compress more, but a lot slower - -snappy - Generate Snappy compatible output stream - -verify - Verify written files - -``` - -## s2d - -``` -Usage: s2d [options] file1 file2 - -Decompresses all files supplied as input. Input files must end with '.s2' or '.snappy'. -Output file names have the extension removed. By default output files will be overwritten. -Use - as the only file name to read from stdin and write to stdout. - -Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt -Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt - -File names beginning with 'http://' and 'https://' will be downloaded and decompressed. -Extensions on downloaded files are ignored. Only http response code 200 is accepted. - -Options: - -bench int - Run benchmark n times. No output will be written - -c Write all output to stdout. Multiple input files will be concatenated - -help - Display help - -o string - Write output to another file. Single input file only - -offset string - Start at offset. Examples: 92, 64K, 256K, 1M, 4M. Requires Index - -q Don't write any output to terminal, except errors - -rm - Delete source file(s) after successful decompression - -safe - Do not overwrite output files - -tail string - Return last of compressed file. Examples: 92, 64K, 256K, 1M, 4M. Requires Index - -verify - Verify files, but do not write output -``` - -## s2sx: self-extracting archives - -s2sx allows creating self-extracting archives with no dependencies. - -By default, executables are created for the same platforms as the host os, -but this can be overridden with `-os` and `-arch` parameters. - -Extracted files have 0666 permissions, except when untar option used. - -``` -Usage: s2sx [options] file1 file2 - -Compresses all files supplied as input separately. -If files have '.s2' extension they are assumed to be compressed already. -Output files are written as 'filename.s2sx' and with '.exe' for windows targets. -If output is big, an additional file with ".more" is written. This must be included as well. -By default output files will be overwritten. - -Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt -Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt - -Options: - -arch string - Destination architecture (default "amd64") - -c Write all output to stdout. Multiple input files will be concatenated - -cpu int - Compress using this amount of threads (default 32) - -help - Display help - -max string - Maximum executable size. Rest will be written to another file. (default "1G") - -os string - Destination operating system (default "windows") - -q Don't write any output to terminal, except errors - -rm - Delete source file(s) after successful compression - -safe - Do not overwrite output files - -untar - Untar on destination -``` - -Available platforms are: - - * darwin-amd64 - * darwin-arm64 - * linux-amd64 - * linux-arm - * linux-arm64 - * linux-mips64 - * linux-ppc64le - * windows-386 - * windows-amd64 - -By default, there is a size limit of 1GB for the output executable. - -When this is exceeded the remaining file content is written to a file called -output+`.more`. This file must be included for a successful extraction and -placed alongside the executable for a successful extraction. - -This file *must* have the same name as the executable, so if the executable is renamed, -so must the `.more` file. - -This functionality is disabled with stdin/stdout. - -### Self-extracting TAR files - -If you wrap a TAR file you can specify `-untar` to make it untar on the destination host. - -Files are extracted to the current folder with the path specified in the tar file. - -Note that tar files are not validated before they are wrapped. - -For security reasons files that move below the root folder are not allowed. - -# Performance - -This section will focus on comparisons to Snappy. -This package is solely aimed at replacing Snappy as a high speed compression package. -If you are mainly looking for better compression [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) -gives better compression, but typically at speeds slightly below "better" mode in this package. - -Compression is increased compared to Snappy, mostly around 5-20% and the throughput is typically 25-40% increased (single threaded) compared to the Snappy Go implementation. - -Streams are concurrently compressed. The stream will be distributed among all available CPU cores for the best possible throughput. - -A "better" compression mode is also available. This allows to trade a bit of speed for a minor compression gain. -The content compressed in this mode is fully compatible with the standard decoder. - -Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU): - -| File | S2 Speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller | -|---------------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------| -| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 16.33x | 10556 MB/s | 8.0% | 6.04x | 5252 MB/s | 14.7% | -| (1 CPU) | 1.08x | 940 MB/s | - | 0.46x | 400 MB/s | - | -| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 16.51x | 15224 MB/s | 31.70% | 9.47x | 8734 MB/s | 37.71% | -| (1 CPU) | 1.26x | 1157 MB/s | - | 0.60x | 556 MB/s | - | -| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12598 MB/s | -5.76% | 6.23x | 5675 MB/s | 3.62% | -| (1 CPU) | 1.02x | 932 MB/s | - | 0.47x | 432 MB/s | - | -| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 11.21x | 12116 MB/s | 15.95% | 3.24x | 3500 MB/s | 18.00% | -| (1 CPU) | 1.05x | 1135 MB/s | - | 0.27x | 292 MB/s | - | -| [apache.log](https://files.klauspost.com/compress/apache.log.zst) | 8.55x | 16673 MB/s | 20.54% | 5.85x | 11420 MB/s | 24.97% | -| (1 CPU) | 1.91x | 1771 MB/s | - | 0.53x | 1041 MB/s | - | -| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 15.76x | 14357 MB/s | 24.01% | 8.67x | 7891 MB/s | 33.68% | -| (1 CPU) | 1.17x | 1064 MB/s | - | 0.65x | 595 MB/s | - | -| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9835 MB/s | 2.34% | 6.85x | 4863 MB/s | 9.96% | -| (1 CPU) | 0.97x | 689 MB/s | - | 0.55x | 387 MB/s | - | -| sharnd.out.2gb | 9.11x | 13213 MB/s | 0.01% | 1.49x | 9184 MB/s | 0.01% | -| (1 CPU) | 0.88x | 5418 MB/s | - | 0.77x | 5417 MB/s | - | -| [sofia-air-quality-dataset csv](https://files.klauspost.com/compress/sofia-air-quality-dataset.tar.zst) | 22.00x | 11477 MB/s | 18.73% | 11.15x | 5817 MB/s | 27.88% | -| (1 CPU) | 1.23x | 642 MB/s | - | 0.71x | 642 MB/s | - | -| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 11.23x | 6520 MB/s | 5.9% | 5.35x | 3109 MB/s | 15.88% | -| (1 CPU) | 1.05x | 607 MB/s | - | 0.52x | 304 MB/s | - | -| [enwik9](https://files.klauspost.com/compress/enwik9.zst) | 19.28x | 8440 MB/s | 4.04% | 9.31x | 4076 MB/s | 18.04% | -| (1 CPU) | 1.12x | 488 MB/s | - | 0.57x | 250 MB/s | - | - -### Legend - -* `S2 Speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core. -* `S2 Throughput`: Throughput of S2 in MB/s. -* `S2 % smaller`: How many percent of the Snappy output size is S2 better. -* `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy. -* `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy. -* `"better" % smaller`: How many percent of the Snappy output size is S2 better when using "better" compression. - -There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads. - -Machine generated data gets by far the biggest compression boost, with size being reduced by up to 35% of Snappy size. - -The "better" compression mode sees a good improvement in all cases, but usually at a performance cost. - -Incompressible content (`sharnd.out.2gb`, 2GB random data) sees the smallest speedup. -This is likely dominated by synchronization overhead, which is confirmed by the fact that single threaded performance is higher (see above). - -## Decompression - -S2 attempts to create content that is also fast to decompress, except in "better" mode where the smallest representation is used. - -S2 vs Snappy **decompression** speed. Both operating on single core: - -| File | S2 Throughput | vs. Snappy | Better Throughput | vs. Snappy | -|-----------------------------------------------------------------------------------------------------|---------------|------------|-------------------|------------| -| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 2117 MB/s | 1.14x | 1738 MB/s | 0.94x | -| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 2401 MB/s | 1.25x | 2307 MB/s | 1.20x | -| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 2075 MB/s | 0.98x | 1764 MB/s | 0.83x | -| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 2967 MB/s | 1.05x | 2885 MB/s | 1.02x | -| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 4141 MB/s | 1.07x | 4184 MB/s | 1.08x | -| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 2264 MB/s | 1.12x | 2185 MB/s | 1.08x | -| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 1525 MB/s | 1.03x | 1347 MB/s | 0.91x | -| sharnd.out.2gb | 3813 MB/s | 0.79x | 3900 MB/s | 0.81x | -| [enwik9](http://mattmahoney.net/dc/textdata.html) | 1246 MB/s | 1.29x | 967 MB/s | 1.00x | -| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 1433 MB/s | 1.12x | 1203 MB/s | 0.94x | -| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 1284 MB/s | 1.32x | 1010 MB/s | 1.04x | - -### Legend - -* `S2 Throughput`: Decompression speed of S2 encoded content. -* `Better Throughput`: Decompression speed of S2 "better" encoded content. -* `vs Snappy`: Decompression speed of S2 "better" mode compared to Snappy and absolute speed. - - -While the decompression code hasn't changed, there is a significant speedup in decompression speed. -S2 prefers longer matches and will typically only find matches that are 6 bytes or longer. -While this reduces compression a bit, it improves decompression speed. - -The "better" compression mode will actively look for shorter matches, which is why it has a decompression speed quite similar to Snappy. - -Without assembly decompression is also very fast; single goroutine decompression speed. No assembly: - -| File | S2 Throughput | S2 throughput | -|--------------------------------|---------------|---------------| -| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s | -| 10gb.tar.s2 | 1.30x | 867.07 MB/s | -| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s | -| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s | -| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s | -| enwik9.s2 | 1.67x | 681.53 MB/s | -| adresser.json.s2 | 3.41x | 4230.53 MB/s | -| silesia.tar.s2 | 1.52x | 811.58 | - -Even though S2 typically compresses better than Snappy, decompression speed is always better. - -### Concurrent Stream Decompression - -For full stream decompression S2 offers a [DecodeConcurrent](https://pkg.go.dev/github.com/klauspost/compress/s2#Reader.DecodeConcurrent) -that will decode a full stream using multiple goroutines. - -Example scaling, AMD Ryzen 3950X, 16 cores, decompression using `s2d -bench=3 <input>`, best of 3: - -| Input | `-cpu=1` | `-cpu=2` | `-cpu=4` | `-cpu=8` | `-cpu=16` | -|-------------------------------------------|------------|------------|------------|------------|-------------| -| enwik10.snappy | 1098.6MB/s | 1819.8MB/s | 3625.6MB/s | 6910.6MB/s | 10818.2MB/s | -| enwik10.s2 | 1303.5MB/s | 2606.1MB/s | 4847.9MB/s | 8878.4MB/s | 9592.1MB/s | -| sofia-air-quality-dataset.tar.snappy | 1302.0MB/s | 2165.0MB/s | 4244.5MB/s | 8241.0MB/s | 12920.5MB/s | -| sofia-air-quality-dataset.tar.s2 | 1399.2MB/s | 2463.2MB/s | 5196.5MB/s | 9639.8MB/s | 11439.5MB/s | -| sofia-air-quality-dataset.tar.s2 (no asm) | 837.5MB/s | 1652.6MB/s | 3183.6MB/s | 5945.0MB/s | 9620.7MB/s | - -Scaling can be expected to be pretty linear until memory bandwidth is saturated. - -For now the DecodeConcurrent can only be used for full streams without seeking or combining with regular reads. - -## Block compression - - -When compressing blocks no concurrent compression is performed just as Snappy. -This is because blocks are for smaller payloads and generally will not benefit from concurrent compression. - -An important change is that incompressible blocks will not be more than at most 10 bytes bigger than the input. -In rare, worst case scenario Snappy blocks could be significantly bigger than the input. - -### Mixed content blocks - -The most reliable is a wide dataset. -For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z), -53927 files, total input size: 4,014,735,833 bytes. Single goroutine used. - -| * | Input | Output | Reduction | MB/s | -|-------------------|------------|------------|------------|------------| -| S2 | 4014735833 | 1059723369 | 73.60% | **936.73** | -| S2 Better | 4014735833 | 961580539 | 76.05% | 451.10 | -| S2 Best | 4014735833 | 899182886 | **77.60%** | 46.84 | -| Snappy | 4014735833 | 1128706759 | 71.89% | 790.15 | -| S2, Snappy Output | 4014735833 | 1093823291 | 72.75% | 936.60 | -| LZ4 | 4014735833 | 1063768713 | 73.50% | 452.02 | - -S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best". -"Better" mode provides the same compression speed as LZ4 with better compression ratio. - -When outputting Snappy compatible output it still delivers better throughput (150MB/s more) and better compression. - -As can be seen from the other benchmarks decompression should also be easier on the S2 generated output. - -Though they cannot be compared due to different decompression speeds here are the speed/size comparisons for -other Go compressors: - -| * | Input | Output | Reduction | MB/s | -|-------------------|------------|------------|-----------|--------| -| Zstd Fastest (Go) | 4014735833 | 794608518 | 80.21% | 236.04 | -| Zstd Best (Go) | 4014735833 | 704603356 | 82.45% | 35.63 | -| Deflate (Go) l1 | 4014735833 | 871294239 | 78.30% | 214.04 | -| Deflate (Go) l9 | 4014735833 | 730389060 | 81.81% | 41.17 | - -### Standard block compression - -Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns. -So individual benchmarks should only be seen as a guideline and the overall picture is more important. - -These micro-benchmarks are with data in cache and trained branch predictors. For a more realistic benchmark see the mixed content above. - -Block compression. Parallel benchmark running on 16 cores, 16 goroutines. - -AMD64 assembly is use for both S2 and Snappy. - -| Absolute Perf | Snappy size | S2 Size | Snappy Speed | S2 Speed | Snappy dec | S2 dec | -|-----------------------|-------------|---------|--------------|-------------|-------------|-------------| -| html | 22843 | 20868 | 16246 MB/s | 18617 MB/s | 40972 MB/s | 49263 MB/s | -| urls.10K | 335492 | 286541 | 7943 MB/s | 10201 MB/s | 22523 MB/s | 26484 MB/s | -| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 303228 MB/s | 718321 MB/s | 827552 MB/s | -| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 20180 MB/s | 33691 MB/s | 52421 MB/s | -| paper-100k.pdf | 85304 | 84202 | 167546 MB/s | 112988 MB/s | 326905 MB/s | 291944 MB/s | -| html_x_4 | 92234 | 20870 | 15194 MB/s | 54457 MB/s | 30843 MB/s | 32217 MB/s | -| alice29.txt | 88034 | 85934 | 5936 MB/s | 6540 MB/s | 12882 MB/s | 20044 MB/s | -| asyoulik.txt | 77503 | 79575 | 5517 MB/s | 6657 MB/s | 12735 MB/s | 22806 MB/s | -| lcet10.txt | 234661 | 220383 | 6235 MB/s | 6303 MB/s | 14519 MB/s | 18697 MB/s | -| plrabn12.txt | 319267 | 318196 | 5159 MB/s | 6074 MB/s | 11923 MB/s | 19901 MB/s | -| geo.protodata | 23335 | 18606 | 21220 MB/s | 25432 MB/s | 56271 MB/s | 62540 MB/s | -| kppkn.gtb | 69526 | 65019 | 9732 MB/s | 8905 MB/s | 18491 MB/s | 18969 MB/s | -| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 17179 MB/s | 31883 MB/s | 38874 MB/s | -| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13273 MB/s | 48056 MB/s | 52341 MB/s | -| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12824 MB/s | 32378 MB/s | 46322 MB/s | -| alice29.txt (20000B) | 12686 | 13516 | 7733 MB/s | 12160 MB/s | 30566 MB/s | 58969 MB/s | - - -Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size. - -Decompression speed is better than Snappy, except in one case. - -Since payloads are very small the variance in terms of size is rather big, so they should only be seen as a general guideline. - -Size is on average around Snappy, but varies on content type. -In cases where compression is worse, it usually is compensated by a speed boost. - - -### Better compression - -Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns. -So individual benchmarks should only be seen as a guideline and the overall picture is more important. - -| Absolute Perf | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec | Better dec | -|-----------------------|-------------|-------------|--------------|--------------|-------------|-------------| -| html | 22843 | 18972 | 16246 MB/s | 8621 MB/s | 40972 MB/s | 40292 MB/s | -| urls.10K | 335492 | 248079 | 7943 MB/s | 5104 MB/s | 22523 MB/s | 20981 MB/s | -| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 84429 MB/s | 718321 MB/s | 823698 MB/s | -| fireworks.jpeg (200B) | 146 | 149 | 8869 MB/s | 7125 MB/s | 33691 MB/s | 30101 MB/s | -| paper-100k.pdf | 85304 | 82887 | 167546 MB/s | 11087 MB/s | 326905 MB/s | 198869 MB/s | -| html_x_4 | 92234 | 18982 | 15194 MB/s | 29316 MB/s | 30843 MB/s | 30937 MB/s | -| alice29.txt | 88034 | 71611 | 5936 MB/s | 3709 MB/s | 12882 MB/s | 16611 MB/s | -| asyoulik.txt | 77503 | 65941 | 5517 MB/s | 3380 MB/s | 12735 MB/s | 14975 MB/s | -| lcet10.txt | 234661 | 184939 | 6235 MB/s | 3537 MB/s | 14519 MB/s | 16634 MB/s | -| plrabn12.txt | 319267 | 264990 | 5159 MB/s | 2960 MB/s | 11923 MB/s | 13382 MB/s | -| geo.protodata | 23335 | 17689 | 21220 MB/s | 10859 MB/s | 56271 MB/s | 57961 MB/s | -| kppkn.gtb | 69526 | 55398 | 9732 MB/s | 5206 MB/s | 18491 MB/s | 16524 MB/s | -| alice29.txt (128B) | 80 | 78 | 6691 MB/s | 7422 MB/s | 31883 MB/s | 34225 MB/s | -| alice29.txt (1000B) | 774 | 746 | 12204 MB/s | 5734 MB/s | 48056 MB/s | 42068 MB/s | -| alice29.txt (10000B) | 6648 | 6218 | 10044 MB/s | 6055 MB/s | 32378 MB/s | 28813 MB/s | -| alice29.txt (20000B) | 12686 | 11492 | 7733 MB/s | 3143 MB/s | 30566 MB/s | 27315 MB/s | - - -Except for the mostly incompressible JPEG image compression is better and usually in the -double digits in terms of percentage reduction over Snappy. - -The PDF sample shows a significant slowdown compared to Snappy, as this mode tries harder -to compress the data. Very small blocks are also not favorable for better compression, so throughput is way down. - -This mode aims to provide better compression at the expense of performance and achieves that -without a huge performance penalty, except on very small blocks. - -Decompression speed suffers a little compared to the regular S2 mode, -but still manages to be close to Snappy in spite of increased compression. - -# Best compression mode - -S2 offers a "best" compression mode. - -This will compress as much as possible with little regard to CPU usage. - -Mainly for offline compression, but where decompression speed should still -be high and compatible with other S2 compressed data. - -Some examples compared on 16 core CPU, amd64 assembly used: - -``` -* enwik10 -Default... 10000000000 -> 4759950115 [47.60%]; 1.03s, 9263.0MB/s -Better... 10000000000 -> 4084706676 [40.85%]; 2.16s, 4415.4MB/s -Best... 10000000000 -> 3615520079 [36.16%]; 42.259s, 225.7MB/s - -* github-june-2days-2019.json -Default... 6273951764 -> 1041700255 [16.60%]; 431ms, 13882.3MB/s -Better... 6273951764 -> 945841238 [15.08%]; 547ms, 10938.4MB/s -Best... 6273951764 -> 826392576 [13.17%]; 9.455s, 632.8MB/s - -* nyc-taxi-data-10M.csv -Default... 3325605752 -> 1093516949 [32.88%]; 324ms, 9788.7MB/s -Better... 3325605752 -> 885394158 [26.62%]; 491ms, 6459.4MB/s -Best... 3325605752 -> 773681257 [23.26%]; 8.29s, 412.0MB/s - -* 10gb.tar -Default... 10065157632 -> 5915541066 [58.77%]; 1.028s, 9337.4MB/s -Better... 10065157632 -> 5453844650 [54.19%]; 1.597s, 4862.7MB/s -Best... 10065157632 -> 5192495021 [51.59%]; 32.78s, 308.2MB/ - -* consensus.db.10gb -Default... 10737418240 -> 4549762344 [42.37%]; 882ms, 12118.4MB/s -Better... 10737418240 -> 4438535064 [41.34%]; 1.533s, 3500.9MB/s -Best... 10737418240 -> 4210602774 [39.21%]; 42.96s, 254.4MB/s -``` - -Decompression speed should be around the same as using the 'better' compression mode. - -## Dictionaries - -*Note: S2 dictionary compression is currently at an early implementation stage, with no assembly for -neither encoding nor decoding. Performance improvements can be expected in the future.* - -Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks. - -The same dictionary *must* be used for both encoding and decoding. -S2 does not keep track of whether the same dictionary is used, -and using the wrong dictionary will most often not result in an error when decompressing. - -Blocks encoded *without* dictionaries can be decompressed seamlessly *with* a dictionary. -This means it is possible to switch from an encoding without dictionaries to an encoding with dictionaries -and treat the blocks similarly. - -Similar to [zStandard dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression), -the same usage scenario applies to S2 dictionaries. - -> Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no universal dictionary). Hence, deploying one dictionary per type of data will provide the greatest benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. - -S2 further limits the dictionary to only be enabled on the first 64KB of a block. -This will remove any negative (speed) impacts of the dictionaries on bigger blocks. - -### Compression - -Using the [github_users_sample_set](https://github.com/facebook/zstd/releases/download/v1.1.3/github_users_sample_set.tar.zst) -and a 64KB dictionary trained with zStandard the following sizes can be achieved. - -| | Default | Better | Best | -|--------------------|------------------|------------------|-----------------------| -| Without Dictionary | 3362023 (44.92%) | 3083163 (41.19%) | 3057944 (40.86%) | -| With Dictionary | 921524 (12.31%) | 873154 (11.67%) | 785503 bytes (10.49%) | - -So for highly repetitive content, this case provides an almost 3x reduction in size. - -For less uniform data we will use the Go source code tree. -Compressing First 64KB of all `.go` files in `go/src`, Go 1.19.5, 8912 files, 51253563 bytes input: - -| | Default | Better | Best | -|--------------------|-------------------|-------------------|-------------------| -| Without Dictionary | 22955767 (44.79%) | 20189613 (39.39% | 19482828 (38.01%) | -| With Dictionary | 19654568 (38.35%) | 16289357 (31.78%) | 15184589 (29.63%) | -| Saving/file | 362 bytes | 428 bytes | 472 bytes | - - -### Creating Dictionaries - -There are no tools to create dictionaries in S2. -However, there are multiple ways to create a useful dictionary: - -#### Using a Sample File - -If your input is very uniform, you can just use a sample file as the dictionary. - -For example in the `github_users_sample_set` above, the average compression only goes up from -10.49% to 11.48% by using the first file as dictionary compared to using a dedicated dictionary. - -```Go - // Read a sample - sample, err := os.ReadFile("sample.json") - - // Create a dictionary. - dict := s2.MakeDict(sample, nil) - - // b := dict.Bytes() will provide a dictionary that can be saved - // and reloaded with s2.NewDict(b). - - // To encode: - encoded := dict.Encode(nil, file) - - // To decode: - decoded, err := dict.Decode(nil, file) -``` - -#### Using Zstandard - -Zstandard dictionaries can easily be converted to S2 dictionaries. - -This can be helpful to generate dictionaries for files that don't have a fixed structure. - - -Example, with training set files placed in `./training-set`: - -`λ zstd -r --train-fastcover training-set/* --maxdict=65536 -o name.dict` - -This will create a dictionary of 64KB, that can be converted to a dictionary like this: - -```Go - // Decode the Zstandard dictionary. - insp, err := zstd.InspectDictionary(zdict) - if err != nil { - panic(err) - } - - // We are only interested in the contents. - // Assume that files start with "// Copyright (c) 2023". - // Search for the longest match for that. - // This may save a few bytes. - dict := s2.MakeDict(insp.Content(), []byte("// Copyright (c) 2023")) - - // b := dict.Bytes() will provide a dictionary that can be saved - // and reloaded with s2.NewDict(b). - - // We can now encode using this dictionary - encodedWithDict := dict.Encode(nil, payload) - - // To decode content: - decoded, err := dict.Decode(nil, encodedWithDict) -``` - -It is recommended to save the dictionary returned by ` b:= dict.Bytes()`, since that will contain only the S2 dictionary. - -This dictionary can later be loaded using `s2.NewDict(b)`. The dictionary then no longer requires `zstd` to be initialized. - -Also note how `s2.MakeDict` allows you to search for a common starting sequence of your files. -This can be omitted, at the expense of a few bytes. - -# Snappy Compatibility - -S2 now offers full compatibility with Snappy. - -This means that the efficient encoders of S2 can be used to generate fully Snappy compatible output. - -There is a [snappy](https://github.com/klauspost/compress/tree/master/snappy) package that can be used by -simply changing imports from `github.com/golang/snappy` to `github.com/klauspost/compress/snappy`. -This uses "better" mode for all operations. -If you would like more control, you can use the s2 package as described below: - -## Blocks - -Snappy compatible blocks can be generated with the S2 encoder. -Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace - -| Snappy | S2 replacement | -|---------------------------|-----------------------| -| snappy.Encode(...) | s2.EncodeSnappy(...) | -| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) | - -`s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output. - -`s2.ConcatBlocks` is compatible with snappy blocks. - -Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z), -53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used: - -| Encoder | Size | MB/s | Reduction | -|-----------------------|------------|------------|------------| -| snappy.Encode | 1128706759 | 725.59 | 71.89% | -| s2.EncodeSnappy | 1093823291 | **899.16** | 72.75% | -| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% | -| s2.EncodeSnappyBest | 944507998 | 66.00 | **76.47%** | - -## Streams - -For streams, replace `enc = snappy.NewBufferedWriter(w)` with `enc = s2.NewWriter(w, s2.WriterSnappyCompat())`. -All other options are available, but note that block size limit is different for snappy. - -Comparison of different streams, AMD Ryzen 3950x, 16 cores. Size and throughput: - -| File | snappy.NewWriter | S2 Snappy | S2 Snappy, Better | S2 Snappy, Best | -|-----------------------------|--------------------------|---------------------------|--------------------------|-------------------------| -| nyc-taxi-data-10M.csv | 1316042016 - 539.47MB/s | 1307003093 - 10132.73MB/s | 1174534014 - 5002.44MB/s | 1115904679 - 177.97MB/s | -| enwik10 (xml) | 5088294643 - 451.13MB/s | 5175840939 - 9440.69MB/s | 4560784526 - 4487.21MB/s | 4340299103 - 158.92MB/s | -| 10gb.tar (mixed) | 6056946612 - 729.73MB/s | 6208571995 - 9978.05MB/s | 5741646126 - 4919.98MB/s | 5548973895 - 180.44MB/s | -| github-june-2days-2019.json | 1525176492 - 933.00MB/s | 1476519054 - 13150.12MB/s | 1400547532 - 5803.40MB/s | 1321887137 - 204.29MB/s | -| consensus.db.10gb (db) | 5412897703 - 1102.14MB/s | 5354073487 - 13562.91MB/s | 5335069899 - 5294.73MB/s | 5201000954 - 175.72MB/s | - -# Decompression - -All decompression functions map directly to equivalent s2 functions. - -| Snappy | S2 replacement | -|------------------------|--------------------| -| snappy.Decode(...) | s2.Decode(...) | -| snappy.DecodedLen(...) | s2.DecodedLen(...) | -| snappy.NewReader(...) | s2.NewReader(...) | - -Features like [quick forward skipping without decompression](https://pkg.go.dev/github.com/klauspost/compress/s2#Reader.Skip) -are also available for Snappy streams. - -If you know you are only decompressing snappy streams, setting [`ReaderMaxBlockSize(64<<10)`](https://pkg.go.dev/github.com/klauspost/compress/s2#ReaderMaxBlockSize) -on your Reader will reduce memory consumption. - -# Concatenating blocks and streams. - -Concatenating streams will concatenate the output of both without recompressing them. -While this is inefficient in terms of compression it might be usable in certain scenarios. -The 10 byte 'stream identifier' of the second stream can optionally be stripped, but it is not a requirement. - -Blocks can be concatenated using the `ConcatBlocks` function. - -Snappy blocks/streams can safely be concatenated with S2 blocks and streams. -Streams with indexes (see below) will currently not work on concatenated streams. - -# Stream Seek Index - -S2 and Snappy streams can have indexes. These indexes will allow random seeking within the compressed data. - -The index can either be appended to the stream as a skippable block or returned for separate storage. - -When the index is appended to a stream it will be skipped by regular decoders, -so the output remains compatible with other decoders. - -## Creating an Index - -To automatically add an index to a stream, add `WriterAddIndex()` option to your writer. -Then the index will be added to the stream when `Close()` is called. - -``` - // Add Index to stream... - enc := s2.NewWriter(w, s2.WriterAddIndex()) - io.Copy(enc, r) - enc.Close() -``` - -If you want to store the index separately, you can use `CloseIndex()` instead of the regular `Close()`. -This will return the index. Note that `CloseIndex()` should only be called once, and you shouldn't call `Close()`. - -``` - // Get index for separate storage... - enc := s2.NewWriter(w) - io.Copy(enc, r) - index, err := enc.CloseIndex() -``` - -The `index` can then be used needing to read from the stream. -This means the index can be used without needing to seek to the end of the stream -or for manually forwarding streams. See below. - -Finally, an existing S2/Snappy stream can be indexed using the `s2.IndexStream(r io.Reader)` function. - -## Using Indexes - -To use indexes there is a `ReadSeeker(random bool, index []byte) (*ReadSeeker, error)` function available. - -Calling ReadSeeker will return an [io.ReadSeeker](https://pkg.go.dev/io#ReadSeeker) compatible version of the reader. - -If 'random' is specified the returned io.Seeker can be used for random seeking, otherwise only forward seeking is supported. -Enabling random seeking requires the original input to support the [io.Seeker](https://pkg.go.dev/io#Seeker) interface. - -``` - dec := s2.NewReader(r) - rs, err := dec.ReadSeeker(false, nil) - rs.Seek(wantOffset, io.SeekStart) -``` - -Get a seeker to seek forward. Since no index is provided, the index is read from the stream. -This requires that an index was added and that `r` supports the [io.Seeker](https://pkg.go.dev/io#Seeker) interface. - -A custom index can be specified which will be used if supplied. -When using a custom index, it will not be read from the input stream. - -``` - dec := s2.NewReader(r) - rs, err := dec.ReadSeeker(false, index) - rs.Seek(wantOffset, io.SeekStart) -``` - -This will read the index from `index`. Since we specify non-random (forward only) seeking `r` does not have to be an io.Seeker - -``` - dec := s2.NewReader(r) - rs, err := dec.ReadSeeker(true, index) - rs.Seek(wantOffset, io.SeekStart) -``` - -Finally, since we specify that we want to do random seeking `r` must be an io.Seeker. - -The returned [ReadSeeker](https://pkg.go.dev/github.com/klauspost/compress/s2#ReadSeeker) contains a shallow reference to the existing Reader, -meaning changes performed to one is reflected in the other. - -To check if a stream contains an index at the end, the `(*Index).LoadStream(rs io.ReadSeeker) error` can be used. - -## Manually Forwarding Streams - -Indexes can also be read outside the decoder using the [Index](https://pkg.go.dev/github.com/klauspost/compress/s2#Index) type. -This can be used for parsing indexes, either separate or in streams. - -In some cases it may not be possible to serve a seekable stream. -This can for instance be an HTTP stream, where the Range request -is sent at the start of the stream. - -With a little bit of extra code it is still possible to use indexes -to forward to specific offset with a single forward skip. - -It is possible to load the index manually like this: -``` - var index s2.Index - _, err = index.Load(idxBytes) -``` - -This can be used to figure out how much to offset the compressed stream: - -``` - compressedOffset, uncompressedOffset, err := index.Find(wantOffset) -``` - -The `compressedOffset` is the number of bytes that should be skipped -from the beginning of the compressed file. - -The `uncompressedOffset` will then be offset of the uncompressed bytes returned -when decoding from that position. This will always be <= wantOffset. - -When creating a decoder it must be specified that it should *not* expect a stream identifier -at the beginning of the stream. Assuming the io.Reader `r` has been forwarded to `compressedOffset` -we create the decoder like this: - -``` - dec := s2.NewReader(r, s2.ReaderIgnoreStreamIdentifier()) -``` - -We are not completely done. We still need to forward the stream the uncompressed bytes we didn't want. -This is done using the regular "Skip" function: - -``` - err = dec.Skip(wantOffset - uncompressedOffset) -``` - -This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset. - -# Compact storage - -For compact storage [RemoveIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RemoveIndexHeaders) can be used to remove any redundant info from -a serialized index. If you remove the header it must be restored before [Loading](https://pkg.go.dev/github.com/klauspost/compress/s2#Index.Load). - -This is expected to save 20 bytes. These can be restored using [RestoreIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RestoreIndexHeaders). This removes a layer of security, but is the most compact representation. Returns nil if headers contains errors. - -## Index Format: - -Each block is structured as a snappy skippable block, with the chunk ID 0x99. - -The block can be read from the front, but contains information so it can be read from the back as well. - -Numbers are stored as fixed size little endian values or [zigzag encoded](https://developers.google.com/protocol-buffers/docs/encoding#signed_integers) [base 128 varints](https://developers.google.com/protocol-buffers/docs/encoding), -with un-encoded value length of 64 bits, unless other limits are specified. - -| Content | Format | -|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| -| ID, `[1]byte` | Always 0x99. | -| Data Length, `[3]byte` | 3 byte little-endian length of the chunk in bytes, following this. | -| Header `[6]byte` | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00". | -| UncompressedSize, Varint | Total Uncompressed size. | -| CompressedSize, Varint | Total Compressed size if known. Should be -1 if unknown. | -| EstBlockSize, Varint | Block Size, used for guessing uncompressed offsets. Must be >= 0. | -| Entries, Varint | Number of Entries in index, must be < 65536 and >=0. | -| HasUncompressedOffsets `byte` | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid. | -| UncompressedOffsets, [Entries]VarInt | Uncompressed offsets. See below how to decode. | -| CompressedOffsets, [Entries]VarInt | Compressed offsets. See below how to decode. | -| Block Size, `[4]byte` | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block. | -| Trailer `[6]byte` | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. | - -For regular streams the uncompressed offsets are fully predictable, -so `HasUncompressedOffsets` allows to specify that compressed blocks all have -exactly `EstBlockSize` bytes of uncompressed content. - -Entries *must* be in order, starting with the lowest offset, -and there *must* be no uncompressed offset duplicates. -Entries *may* point to the start of a skippable block, -but it is then not allowed to also have an entry for the next block since -that would give an uncompressed offset duplicate. - -There is no requirement for all blocks to be represented in the index. -In fact there is a maximum of 65536 block entries in an index. - -The writer can use any method to reduce the number of entries. -An implicit block start at 0,0 can be assumed. - -### Decoding entries: - -``` -// Read Uncompressed entries. -// Each assumes EstBlockSize delta from previous. -for each entry { - uOff = 0 - if HasUncompressedOffsets == 1 { - uOff = ReadVarInt // Read value from stream - } - - // Except for the first entry, use previous values. - if entryNum == 0 { - entry[entryNum].UncompressedOffset = uOff - continue - } - - // Uncompressed uses previous offset and adds EstBlockSize - entry[entryNum].UncompressedOffset = entry[entryNum-1].UncompressedOffset + EstBlockSize + uOff -} - - -// Guess that the first block will be 50% of uncompressed size. -// Integer truncating division must be used. -CompressGuess := EstBlockSize / 2 - -// Read Compressed entries. -// Each assumes CompressGuess delta from previous. -// CompressGuess is adjusted for each value. -for each entry { - cOff = ReadVarInt // Read value from stream - - // Except for the first entry, use previous values. - if entryNum == 0 { - entry[entryNum].CompressedOffset = cOff - continue - } - - // Compressed uses previous and our estimate. - entry[entryNum].CompressedOffset = entry[entryNum-1].CompressedOffset + CompressGuess + cOff - - // Adjust compressed offset for next loop, integer truncating division must be used. - CompressGuess += cOff/2 -} -``` - -To decode from any given uncompressed offset `(wantOffset)`: - -* Iterate entries until `entry[n].UncompressedOffset > wantOffset`. -* Start decoding from `entry[n-1].CompressedOffset`. -* Discard `entry[n-1].UncompressedOffset - wantOffset` bytes from the decoded stream. - -See [using indexes](https://github.com/klauspost/compress/tree/master/s2#using-indexes) for functions that perform the operations with a simpler interface. - - -# Format Extensions - -* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`. -* [Framed compressed blocks](https://github.com/google/snappy/blob/master/format_description.txt) can be up to 4MB (up from 64KB). -* Compressed blocks can have an offset of `0`, which indicates to repeat the last seen offset. - -Repeat offsets must be encoded as a [2.2.1. Copy with 1-byte offset (01)](https://github.com/google/snappy/blob/master/format_description.txt#L89), where the offset is 0. - -The length is specified by reading the 3-bit length specified in the tag and decode using this table: - -| Length | Actual Length | -|--------|----------------------| -| 0 | 4 | -| 1 | 5 | -| 2 | 6 | -| 3 | 7 | -| 4 | 8 | -| 5 | 8 + read 1 byte | -| 6 | 260 + read 2 bytes | -| 7 | 65540 + read 3 bytes | - -This allows any repeat offset + length to be represented by 2 to 5 bytes. -It also allows to emit matches longer than 64 bytes with one copy + one repeat instead of several 64 byte copies. - -Lengths are stored as little endian values. - -The first copy of a block cannot be a repeat offset and the offset is reset on every block in streams. - -Default streaming block size is 1MB. - -# Dictionary Encoding - -Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks. - -A dictionary provides an initial repeat value that can be used to point to a common header. - -Other than that the dictionary contains values that can be used as back-references. - -Often used data should be placed at the *end* of the dictionary since offsets < 2048 bytes will be smaller. - -## Format - -Dictionary *content* must at least 16 bytes and less or equal to 64KiB (65536 bytes). - -Encoding: `[repeat value (uvarint)][dictionary content...]` - -Before the dictionary content, an unsigned base-128 (uvarint) encoded value specifying the initial repeat offset. -This value is an offset into the dictionary content and not a back-reference offset, -so setting this to 0 will make the repeat value point to the first value of the dictionary. - -The value must be less than the dictionary length-8 - -## Encoding - -From the decoder point of view the dictionary content is seen as preceding the encoded content. - -`[dictionary content][decoded output]` - -Backreferences to the dictionary are encoded as ordinary backreferences that have an offset before the start of the decoded block. - -Matches copying from the dictionary are **not** allowed to cross from the dictionary into the decoded data. -However, if a copy ends at the end of the dictionary the next repeat will point to the start of the decoded buffer, which is allowed. - -The first match can be a repeat value, which will use the repeat offset stored in the dictionary. - -When 64KB (65536 bytes) has been en/decoded it is no longer allowed to reference the dictionary, -neither by a copy nor repeat operations. -If the boundary is crossed while copying from the dictionary, the operation should complete, -but the next instruction is not allowed to reference the dictionary. - -Valid blocks encoded *without* a dictionary can be decoded with any dictionary. -There are no checks whether the supplied dictionary is the correct for a block. -Because of this there is no overhead by using a dictionary. - -## Example - -This is the dictionary content. Elements are separated by `[]`. - -Dictionary: `[0x0a][Yesterday 25 bananas were added to Benjamins brown bag]`. - -Initial repeat offset is set at 10, which is the letter `2`. - -Encoded `[LIT "10"][REPEAT len=10][LIT "hich"][MATCH off=50 len=6][MATCH off=31 len=6][MATCH off=61 len=10]` - -Decoded: `[10][ bananas w][hich][ were ][brown ][were added]` - -Output: `10 bananas which were brown were added` - - -## Streams - -For streams each block can use the dictionary. - -The dictionary cannot not currently be provided on the stream. - - -# LICENSE - -This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation. - -Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/s2/decode.go b/vendor/github.com/klauspost/compress/s2/decode.go deleted file mode 100644 index 264ffd0a9..000000000 --- a/vendor/github.com/klauspost/compress/s2/decode.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "encoding/binary" - "errors" - "fmt" - "strconv" - - "github.com/klauspost/compress/internal/race" -) - -var ( - // ErrCorrupt reports that the input is invalid. - ErrCorrupt = errors.New("s2: corrupt input") - // ErrCRC reports that the input failed CRC validation (streams only) - ErrCRC = errors.New("s2: corrupt input, crc mismatch") - // ErrTooLarge reports that the uncompressed length is too large. - ErrTooLarge = errors.New("s2: decoded block is too large") - // ErrUnsupported reports that the input isn't supported. - ErrUnsupported = errors.New("s2: unsupported input") -) - -// DecodedLen returns the length of the decoded block. -func DecodedLen(src []byte) (int, error) { - v, _, err := decodedLen(src) - return v, err -} - -// decodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func decodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { - return 0, 0, ErrCorrupt - } - - const wordSize = 32 << (^uint(0) >> 32 & 1) - if wordSize == 32 && v > 0x7fffffff { - return 0, 0, ErrTooLarge - } - return int(v), n, nil -} - -const ( - decodeErrCodeCorrupt = 1 -) - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -func Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if dLen <= cap(dst) { - dst = dst[:dLen] - } else { - dst = make([]byte, dLen) - } - - race.WriteSlice(dst) - race.ReadSlice(src[s:]) - - if s2Decode(dst, src[s:]) != 0 { - return nil, ErrCorrupt - } - return dst, nil -} - -// s2DecodeDict writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read, and that len(dst) -// equals that length. -// -// It returns 0 on success or a decodeErrCodeXxx error code on failure. -func s2DecodeDict(dst, src []byte, dict *Dict) int { - if dict == nil { - return s2Decode(dst, src) - } - const debug = false - const debugErrs = debug - - if debug { - fmt.Println("Starting decode, dst len:", len(dst)) - } - var d, s, length int - offset := len(dict.dict) - dict.repeat - - // As long as we can read at least 5 bytes... - for s < len(src)-5 { - // Removing bounds checks is SLOWER, when if doing - // in := src[s:s+5] - // Checked on Go 1.18 - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - x = uint32(src[s-1]) - case x == 61: - in := src[s : s+3] - x = uint32(in[1]) | uint32(in[2])<<8 - s += 3 - case x == 62: - in := src[s : s+4] - // Load as 32 bit and shift down. - x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24 - x >>= 8 - s += 4 - case x == 63: - in := src[s : s+5] - x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24 - s += 5 - } - length = int(x) + 1 - if debug { - fmt.Println("literals, length:", length, "d-after:", d+length) - } - if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) { - if debugErrs { - fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s) - } - return decodeErrCodeCorrupt - } - - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - length = int(src[s-2]) >> 2 & 0x7 - if toffset == 0 { - if debug { - fmt.Print("(repeat) ") - } - // keep last offset - switch length { - case 5: - length = int(src[s]) + 4 - s += 1 - case 6: - in := src[s : s+2] - length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8) - s += 2 - case 7: - in := src[s : s+3] - length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16) - s += 3 - default: // 0-> 4 - } - } else { - offset = toffset - } - length += 4 - case tagCopy2: - in := src[s : s+3] - offset = int(uint32(in[1]) | uint32(in[2])<<8) - length = 1 + int(in[0])>>2 - s += 3 - - case tagCopy4: - in := src[s : s+5] - offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24) - length = 1 + int(in[0])>>2 - s += 5 - } - - if offset <= 0 || length > len(dst)-d { - if debugErrs { - fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d) - } - return decodeErrCodeCorrupt - } - - // copy from dict - if d < offset { - if d > MaxDictSrcOffset { - if debugErrs { - fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length) - } - return decodeErrCodeCorrupt - } - startOff := len(dict.dict) - offset + d - if startOff < 0 || startOff+length > len(dict.dict) { - if debugErrs { - fmt.Printf("offset (%d) + length (%d) bigger than dict (%d)\n", offset, length, len(dict.dict)) - } - return decodeErrCodeCorrupt - } - if debug { - fmt.Println("dict copy, length:", length, "offset:", offset, "d-after:", d+length, "dict start offset:", startOff) - } - copy(dst[d:d+length], dict.dict[startOff:]) - d += length - continue - } - - if debug { - fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length) - } - - // Copy from an earlier sub-slice of dst to a later sub-slice. - // If no overlap, use the built-in copy: - if offset > length { - copy(dst[d:d+length], dst[d-offset:]) - d += length - continue - } - - // Unlike the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - // - // We align the slices into a and b and show the compiler they are the same size. - // This allows the loop to run without bounds checks. - a := dst[d : d+length] - b := dst[d-offset:] - b = b[:len(a)] - for i := range a { - a[i] = b[i] - } - d += length - } - - // Remaining with extra checks... - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - length = int(x) + 1 - if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) { - if debugErrs { - fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s) - } - return decodeErrCodeCorrupt - } - if debug { - fmt.Println("literals, length:", length, "d-after:", d+length) - } - - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = int(src[s-2]) >> 2 & 0x7 - toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - if toffset == 0 { - if debug { - fmt.Print("(repeat) ") - } - // keep last offset - switch length { - case 5: - s += 1 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-1])) + 4 - case 6: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8) - case 7: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16) - default: // 0-> 4 - } - } else { - offset = toffset - } - length += 4 - case tagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - - case tagCopy4: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - if debugErrs { - fmt.Println("src went oob") - } - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-5])>>2 - offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - } - - if offset <= 0 || length > len(dst)-d { - if debugErrs { - fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d) - } - return decodeErrCodeCorrupt - } - - // copy from dict - if d < offset { - if d > MaxDictSrcOffset { - if debugErrs { - fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length) - } - return decodeErrCodeCorrupt - } - rOff := len(dict.dict) - (offset - d) - if debug { - fmt.Println("starting dict entry from dict offset", len(dict.dict)-rOff) - } - if rOff+length > len(dict.dict) { - if debugErrs { - fmt.Println("err: END offset", rOff+length, "bigger than dict", len(dict.dict), "dict offset:", rOff, "length:", length) - } - return decodeErrCodeCorrupt - } - if rOff < 0 { - if debugErrs { - fmt.Println("err: START offset", rOff, "less than 0", len(dict.dict), "dict offset:", rOff, "length:", length) - } - return decodeErrCodeCorrupt - } - copy(dst[d:d+length], dict.dict[rOff:]) - d += length - continue - } - - if debug { - fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length) - } - - // Copy from an earlier sub-slice of dst to a later sub-slice. - // If no overlap, use the built-in copy: - if offset > length { - copy(dst[d:d+length], dst[d-offset:]) - d += length - continue - } - - // Unlike the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - // - // We align the slices into a and b and show the compiler they are the same size. - // This allows the loop to run without bounds checks. - a := dst[d : d+length] - b := dst[d-offset:] - b = b[:len(a)] - for i := range a { - a[i] = b[i] - } - d += length - } - - if d != len(dst) { - if debugErrs { - fmt.Println("wanted length", len(dst), "got", d) - } - return decodeErrCodeCorrupt - } - return 0 -} diff --git a/vendor/github.com/klauspost/compress/s2/decode_amd64.s b/vendor/github.com/klauspost/compress/s2/decode_amd64.s deleted file mode 100644 index 9b105e03c..000000000 --- a/vendor/github.com/klauspost/compress/s2/decode_amd64.s +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" - -#define R_TMP0 AX -#define R_TMP1 BX -#define R_LEN CX -#define R_OFF DX -#define R_SRC SI -#define R_DST DI -#define R_DBASE R8 -#define R_DLEN R9 -#define R_DEND R10 -#define R_SBASE R11 -#define R_SLEN R12 -#define R_SEND R13 -#define R_TMP2 R14 -#define R_TMP3 R15 - -// The asm code generally follows the pure Go code in decode_other.go, except -// where marked with a "!!!". - -// func decode(dst, src []byte) int -// -// All local variables fit into registers. The non-zero stack size is only to -// spill registers and push args when issuing a CALL. The register allocation: -// - R_TMP0 scratch -// - R_TMP1 scratch -// - R_LEN length or x (shared) -// - R_OFF offset -// - R_SRC &src[s] -// - R_DST &dst[d] -// + R_DBASE dst_base -// + R_DLEN dst_len -// + R_DEND dst_base + dst_len -// + R_SBASE src_base -// + R_SLEN src_len -// + R_SEND src_base + src_len -// - R_TMP2 used by doCopy -// - R_TMP3 used by doCopy -// -// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the -// function, and after a CALL returns, and are not otherwise modified. -// -// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST. -// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC. -TEXT ·s2Decode(SB), NOSPLIT, $48-56 - // Initialize R_SRC, R_DST and R_DBASE-R_SEND. - MOVQ dst_base+0(FP), R_DBASE - MOVQ dst_len+8(FP), R_DLEN - MOVQ R_DBASE, R_DST - MOVQ R_DBASE, R_DEND - ADDQ R_DLEN, R_DEND - MOVQ src_base+24(FP), R_SBASE - MOVQ src_len+32(FP), R_SLEN - MOVQ R_SBASE, R_SRC - MOVQ R_SBASE, R_SEND - ADDQ R_SLEN, R_SEND - XORQ R_OFF, R_OFF - -loop: - // for s < len(src) - CMPQ R_SRC, R_SEND - JEQ end - - // R_LEN = uint32(src[s]) - // - // switch src[s] & 0x03 - MOVBLZX (R_SRC), R_LEN - MOVL R_LEN, R_TMP1 - ANDL $3, R_TMP1 - CMPL R_TMP1, $1 - JAE tagCopy - - // ---------------------------------------- - // The code below handles literal tags. - - // case tagLiteral: - // x := uint32(src[s] >> 2) - // switch - SHRL $2, R_LEN - CMPL R_LEN, $60 - JAE tagLit60Plus - - // case x < 60: - // s++ - INCQ R_SRC - -doLit: - // This is the end of the inner "switch", when we have a literal tag. - // - // We assume that R_LEN == x and x fits in a uint32, where x is the variable - // used in the pure Go decode_other.go code. - - // length = int(x) + 1 - // - // Unlike the pure Go code, we don't need to check if length <= 0 because - // R_LEN can hold 64 bits, so the increment cannot overflow. - INCQ R_LEN - - // Prepare to check if copying length bytes will run past the end of dst or - // src. - // - // R_TMP0 = len(dst) - d - // R_TMP1 = len(src) - s - MOVQ R_DEND, R_TMP0 - SUBQ R_DST, R_TMP0 - MOVQ R_SEND, R_TMP1 - SUBQ R_SRC, R_TMP1 - - // !!! Try a faster technique for short (16 or fewer bytes) copies. - // - // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { - // goto callMemmove // Fall back on calling runtime·memmove. - // } - // - // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s - // against 21 instead of 16, because it cannot assume that all of its input - // is contiguous in memory and so it needs to leave enough source bytes to - // read the next tag without refilling buffers, but Go's Decode assumes - // contiguousness (the src argument is a []byte). - CMPQ R_LEN, $16 - JGT callMemmove - CMPQ R_TMP0, $16 - JLT callMemmove - CMPQ R_TMP1, $16 - JLT callMemmove - - // !!! Implement the copy from src to dst as a 16-byte load and store. - // (Decode's documentation says that dst and src must not overlap.) - // - // This always copies 16 bytes, instead of only length bytes, but that's - // OK. If the input is a valid Snappy encoding then subsequent iterations - // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a - // non-nil error), so the overrun will be ignored. - // - // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or - // 16-byte loads and stores. This technique probably wouldn't be as - // effective on architectures that are fussier about alignment. - MOVOU 0(R_SRC), X0 - MOVOU X0, 0(R_DST) - - // d += length - // s += length - ADDQ R_LEN, R_DST - ADDQ R_LEN, R_SRC - JMP loop - -callMemmove: - // if length > len(dst)-d || length > len(src)-s { etc } - CMPQ R_LEN, R_TMP0 - JGT errCorrupt - CMPQ R_LEN, R_TMP1 - JGT errCorrupt - - // copy(dst[d:], src[s:s+length]) - // - // This means calling runtime·memmove(&dst[d], &src[s], length), so we push - // R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those - // three registers to the stack, to save local variables across the CALL. - MOVQ R_DST, 0(SP) - MOVQ R_SRC, 8(SP) - MOVQ R_LEN, 16(SP) - MOVQ R_DST, 24(SP) - MOVQ R_SRC, 32(SP) - MOVQ R_LEN, 40(SP) - MOVQ R_OFF, 48(SP) - CALL runtime·memmove(SB) - - // Restore local variables: unspill registers from the stack and - // re-calculate R_DBASE-R_SEND. - MOVQ 24(SP), R_DST - MOVQ 32(SP), R_SRC - MOVQ 40(SP), R_LEN - MOVQ 48(SP), R_OFF - MOVQ dst_base+0(FP), R_DBASE - MOVQ dst_len+8(FP), R_DLEN - MOVQ R_DBASE, R_DEND - ADDQ R_DLEN, R_DEND - MOVQ src_base+24(FP), R_SBASE - MOVQ src_len+32(FP), R_SLEN - MOVQ R_SBASE, R_SEND - ADDQ R_SLEN, R_SEND - - // d += length - // s += length - ADDQ R_LEN, R_DST - ADDQ R_LEN, R_SRC - JMP loop - -tagLit60Plus: - // !!! This fragment does the - // - // s += x - 58; if uint(s) > uint(len(src)) { etc } - // - // checks. In the asm version, we code it once instead of once per switch case. - ADDQ R_LEN, R_SRC - SUBQ $58, R_SRC - CMPQ R_SRC, R_SEND - JA errCorrupt - - // case x == 60: - CMPL R_LEN, $61 - JEQ tagLit61 - JA tagLit62Plus - - // x = uint32(src[s-1]) - MOVBLZX -1(R_SRC), R_LEN - JMP doLit - -tagLit61: - // case x == 61: - // x = uint32(src[s-2]) | uint32(src[s-1])<<8 - MOVWLZX -2(R_SRC), R_LEN - JMP doLit - -tagLit62Plus: - CMPL R_LEN, $62 - JA tagLit63 - - // case x == 62: - // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - // We read one byte, safe to read one back, since we are just reading tag. - // x = binary.LittleEndian.Uint32(src[s-1:]) >> 8 - MOVL -4(R_SRC), R_LEN - SHRL $8, R_LEN - JMP doLit - -tagLit63: - // case x == 63: - // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - MOVL -4(R_SRC), R_LEN - JMP doLit - -// The code above handles literal tags. -// ---------------------------------------- -// The code below handles copy tags. - -tagCopy4: - // case tagCopy4: - // s += 5 - ADDQ $5, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // length = 1 + int(src[s-5])>>2 - SHRQ $2, R_LEN - INCQ R_LEN - - // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - MOVLQZX -4(R_SRC), R_OFF - JMP doCopy - -tagCopy2: - // case tagCopy2: - // s += 3 - ADDQ $3, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // length = 1 + int(src[s-3])>>2 - SHRQ $2, R_LEN - INCQ R_LEN - - // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - MOVWQZX -2(R_SRC), R_OFF - JMP doCopy - -tagCopy: - // We have a copy tag. We assume that: - // - R_TMP1 == src[s] & 0x03 - // - R_LEN == src[s] - CMPQ R_TMP1, $2 - JEQ tagCopy2 - JA tagCopy4 - - // case tagCopy1: - // s += 2 - ADDQ $2, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - // length = 4 + int(src[s-2])>>2&0x7 - MOVBQZX -1(R_SRC), R_TMP1 - MOVQ R_LEN, R_TMP0 - SHRQ $2, R_LEN - ANDQ $0xe0, R_TMP0 - ANDQ $7, R_LEN - SHLQ $3, R_TMP0 - ADDQ $4, R_LEN - ORQ R_TMP1, R_TMP0 - - // check if repeat code, ZF set by ORQ. - JZ repeatCode - - // This is a regular copy, transfer our temporary value to R_OFF (length) - MOVQ R_TMP0, R_OFF - JMP doCopy - -// This is a repeat code. -repeatCode: - // If length < 9, reuse last offset, with the length already calculated. - CMPQ R_LEN, $9 - JL doCopyRepeat - - // Read additional bytes for length. - JE repeatLen1 - - // Rare, so the extra branch shouldn't hurt too much. - CMPQ R_LEN, $10 - JE repeatLen2 - JMP repeatLen3 - -// Read repeat lengths. -repeatLen1: - // s ++ - ADDQ $1, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // length = src[s-1] + 8 - MOVBQZX -1(R_SRC), R_LEN - ADDL $8, R_LEN - JMP doCopyRepeat - -repeatLen2: - // s +=2 - ADDQ $2, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + (1 << 8) - MOVWQZX -2(R_SRC), R_LEN - ADDL $260, R_LEN - JMP doCopyRepeat - -repeatLen3: - // s +=3 - ADDQ $3, R_SRC - - // if uint(s) > uint(len(src)) { etc } - CMPQ R_SRC, R_SEND - JA errCorrupt - - // length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + (1 << 16) - // Read one byte further back (just part of the tag, shifted out) - MOVL -4(R_SRC), R_LEN - SHRL $8, R_LEN - ADDL $65540, R_LEN - JMP doCopyRepeat - -doCopy: - // This is the end of the outer "switch", when we have a copy tag. - // - // We assume that: - // - R_LEN == length && R_LEN > 0 - // - R_OFF == offset - - // if d < offset { etc } - MOVQ R_DST, R_TMP1 - SUBQ R_DBASE, R_TMP1 - CMPQ R_TMP1, R_OFF - JLT errCorrupt - - // Repeat values can skip the test above, since any offset > 0 will be in dst. -doCopyRepeat: - // if offset <= 0 { etc } - CMPQ R_OFF, $0 - JLE errCorrupt - - // if length > len(dst)-d { etc } - MOVQ R_DEND, R_TMP1 - SUBQ R_DST, R_TMP1 - CMPQ R_LEN, R_TMP1 - JGT errCorrupt - - // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length - // - // Set: - // - R_TMP2 = len(dst)-d - // - R_TMP3 = &dst[d-offset] - MOVQ R_DEND, R_TMP2 - SUBQ R_DST, R_TMP2 - MOVQ R_DST, R_TMP3 - SUBQ R_OFF, R_TMP3 - - // !!! Try a faster technique for short (16 or fewer bytes) forward copies. - // - // First, try using two 8-byte load/stores, similar to the doLit technique - // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is - // still OK if offset >= 8. Note that this has to be two 8-byte load/stores - // and not one 16-byte load/store, and the first store has to be before the - // second load, due to the overlap if offset is in the range [8, 16). - // - // if length > 16 || offset < 8 || len(dst)-d < 16 { - // goto slowForwardCopy - // } - // copy 16 bytes - // d += length - CMPQ R_LEN, $16 - JGT slowForwardCopy - CMPQ R_OFF, $8 - JLT slowForwardCopy - CMPQ R_TMP2, $16 - JLT slowForwardCopy - MOVQ 0(R_TMP3), R_TMP0 - MOVQ R_TMP0, 0(R_DST) - MOVQ 8(R_TMP3), R_TMP1 - MOVQ R_TMP1, 8(R_DST) - ADDQ R_LEN, R_DST - JMP loop - -slowForwardCopy: - // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we - // can still try 8-byte load stores, provided we can overrun up to 10 extra - // bytes. As above, the overrun will be fixed up by subsequent iterations - // of the outermost loop. - // - // The C++ snappy code calls this technique IncrementalCopyFastPath. Its - // commentary says: - // - // ---- - // - // The main part of this loop is a simple copy of eight bytes at a time - // until we've copied (at least) the requested amount of bytes. However, - // if d and d-offset are less than eight bytes apart (indicating a - // repeating pattern of length < 8), we first need to expand the pattern in - // order to get the correct results. For instance, if the buffer looks like - // this, with the eight-byte <d-offset> and <d> patterns marked as - // intervals: - // - // abxxxxxxxxxxxx - // [------] d-offset - // [------] d - // - // a single eight-byte copy from <d-offset> to <d> will repeat the pattern - // once, after which we can move <d> two bytes without moving <d-offset>: - // - // ababxxxxxxxxxx - // [------] d-offset - // [------] d - // - // and repeat the exercise until the two no longer overlap. - // - // This allows us to do very well in the special case of one single byte - // repeated many times, without taking a big hit for more general cases. - // - // The worst case of extra writing past the end of the match occurs when - // offset == 1 and length == 1; the last copy will read from byte positions - // [0..7] and write to [4..11], whereas it was only supposed to write to - // position 1. Thus, ten excess bytes. - // - // ---- - // - // That "10 byte overrun" worst case is confirmed by Go's - // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy - // and finishSlowForwardCopy algorithm. - // - // if length > len(dst)-d-10 { - // goto verySlowForwardCopy - // } - SUBQ $10, R_TMP2 - CMPQ R_LEN, R_TMP2 - JGT verySlowForwardCopy - - // We want to keep the offset, so we use R_TMP2 from here. - MOVQ R_OFF, R_TMP2 - -makeOffsetAtLeast8: - // !!! As above, expand the pattern so that offset >= 8 and we can use - // 8-byte load/stores. - // - // for offset < 8 { - // copy 8 bytes from dst[d-offset:] to dst[d:] - // length -= offset - // d += offset - // offset += offset - // // The two previous lines together means that d-offset, and therefore - // // R_TMP3, is unchanged. - // } - CMPQ R_TMP2, $8 - JGE fixUpSlowForwardCopy - MOVQ (R_TMP3), R_TMP1 - MOVQ R_TMP1, (R_DST) - SUBQ R_TMP2, R_LEN - ADDQ R_TMP2, R_DST - ADDQ R_TMP2, R_TMP2 - JMP makeOffsetAtLeast8 - -fixUpSlowForwardCopy: - // !!! Add length (which might be negative now) to d (implied by R_DST being - // &dst[d]) so that d ends up at the right place when we jump back to the - // top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if - // length is positive, copying the remaining length bytes will write to the - // right place. - MOVQ R_DST, R_TMP0 - ADDQ R_LEN, R_DST - -finishSlowForwardCopy: - // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative - // length means that we overrun, but as above, that will be fixed up by - // subsequent iterations of the outermost loop. - CMPQ R_LEN, $0 - JLE loop - MOVQ (R_TMP3), R_TMP1 - MOVQ R_TMP1, (R_TMP0) - ADDQ $8, R_TMP3 - ADDQ $8, R_TMP0 - SUBQ $8, R_LEN - JMP finishSlowForwardCopy - -verySlowForwardCopy: - // verySlowForwardCopy is a simple implementation of forward copy. In C - // parlance, this is a do/while loop instead of a while loop, since we know - // that length > 0. In Go syntax: - // - // for { - // dst[d] = dst[d - offset] - // d++ - // length-- - // if length == 0 { - // break - // } - // } - MOVB (R_TMP3), R_TMP1 - MOVB R_TMP1, (R_DST) - INCQ R_TMP3 - INCQ R_DST - DECQ R_LEN - JNZ verySlowForwardCopy - JMP loop - -// The code above handles copy tags. -// ---------------------------------------- - -end: - // This is the end of the "for s < len(src)". - // - // if d != len(dst) { etc } - CMPQ R_DST, R_DEND - JNE errCorrupt - - // return 0 - MOVQ $0, ret+48(FP) - RET - -errCorrupt: - // return decodeErrCodeCorrupt - MOVQ $1, ret+48(FP) - RET diff --git a/vendor/github.com/klauspost/compress/s2/decode_arm64.s b/vendor/github.com/klauspost/compress/s2/decode_arm64.s deleted file mode 100644 index 78e463f34..000000000 --- a/vendor/github.com/klauspost/compress/s2/decode_arm64.s +++ /dev/null @@ -1,574 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" - -#define R_TMP0 R2 -#define R_TMP1 R3 -#define R_LEN R4 -#define R_OFF R5 -#define R_SRC R6 -#define R_DST R7 -#define R_DBASE R8 -#define R_DLEN R9 -#define R_DEND R10 -#define R_SBASE R11 -#define R_SLEN R12 -#define R_SEND R13 -#define R_TMP2 R14 -#define R_TMP3 R15 - -// TEST_SRC will check if R_SRC is <= SRC_END -#define TEST_SRC() \ - CMP R_SEND, R_SRC \ - BGT errCorrupt - -// MOVD R_SRC, R_TMP1 -// SUB R_SBASE, R_TMP1, R_TMP1 -// CMP R_SLEN, R_TMP1 -// BGT errCorrupt - -// The asm code generally follows the pure Go code in decode_other.go, except -// where marked with a "!!!". - -// func decode(dst, src []byte) int -// -// All local variables fit into registers. The non-zero stack size is only to -// spill registers and push args when issuing a CALL. The register allocation: -// - R_TMP0 scratch -// - R_TMP1 scratch -// - R_LEN length or x -// - R_OFF offset -// - R_SRC &src[s] -// - R_DST &dst[d] -// + R_DBASE dst_base -// + R_DLEN dst_len -// + R_DEND dst_base + dst_len -// + R_SBASE src_base -// + R_SLEN src_len -// + R_SEND src_base + src_len -// - R_TMP2 used by doCopy -// - R_TMP3 used by doCopy -// -// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the -// function, and after a CALL returns, and are not otherwise modified. -// -// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST. -// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC. -TEXT ·s2Decode(SB), NOSPLIT, $56-56 - // Initialize R_SRC, R_DST and R_DBASE-R_SEND. - MOVD dst_base+0(FP), R_DBASE - MOVD dst_len+8(FP), R_DLEN - MOVD R_DBASE, R_DST - MOVD R_DBASE, R_DEND - ADD R_DLEN, R_DEND, R_DEND - MOVD src_base+24(FP), R_SBASE - MOVD src_len+32(FP), R_SLEN - MOVD R_SBASE, R_SRC - MOVD R_SBASE, R_SEND - ADD R_SLEN, R_SEND, R_SEND - MOVD $0, R_OFF - -loop: - // for s < len(src) - CMP R_SEND, R_SRC - BEQ end - - // R_LEN = uint32(src[s]) - // - // switch src[s] & 0x03 - MOVBU (R_SRC), R_LEN - MOVW R_LEN, R_TMP1 - ANDW $3, R_TMP1 - MOVW $1, R1 - CMPW R1, R_TMP1 - BGE tagCopy - - // ---------------------------------------- - // The code below handles literal tags. - - // case tagLiteral: - // x := uint32(src[s] >> 2) - // switch - MOVW $60, R1 - LSRW $2, R_LEN, R_LEN - CMPW R_LEN, R1 - BLS tagLit60Plus - - // case x < 60: - // s++ - ADD $1, R_SRC, R_SRC - -doLit: - // This is the end of the inner "switch", when we have a literal tag. - // - // We assume that R_LEN == x and x fits in a uint32, where x is the variable - // used in the pure Go decode_other.go code. - - // length = int(x) + 1 - // - // Unlike the pure Go code, we don't need to check if length <= 0 because - // R_LEN can hold 64 bits, so the increment cannot overflow. - ADD $1, R_LEN, R_LEN - - // Prepare to check if copying length bytes will run past the end of dst or - // src. - // - // R_TMP0 = len(dst) - d - // R_TMP1 = len(src) - s - MOVD R_DEND, R_TMP0 - SUB R_DST, R_TMP0, R_TMP0 - MOVD R_SEND, R_TMP1 - SUB R_SRC, R_TMP1, R_TMP1 - - // !!! Try a faster technique for short (16 or fewer bytes) copies. - // - // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { - // goto callMemmove // Fall back on calling runtime·memmove. - // } - // - // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s - // against 21 instead of 16, because it cannot assume that all of its input - // is contiguous in memory and so it needs to leave enough source bytes to - // read the next tag without refilling buffers, but Go's Decode assumes - // contiguousness (the src argument is a []byte). - CMP $16, R_LEN - BGT callMemmove - CMP $16, R_TMP0 - BLT callMemmove - CMP $16, R_TMP1 - BLT callMemmove - - // !!! Implement the copy from src to dst as a 16-byte load and store. - // (Decode's documentation says that dst and src must not overlap.) - // - // This always copies 16 bytes, instead of only length bytes, but that's - // OK. If the input is a valid Snappy encoding then subsequent iterations - // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a - // non-nil error), so the overrun will be ignored. - // - // Note that on arm64, it is legal and cheap to issue unaligned 8-byte or - // 16-byte loads and stores. This technique probably wouldn't be as - // effective on architectures that are fussier about alignment. - LDP 0(R_SRC), (R_TMP2, R_TMP3) - STP (R_TMP2, R_TMP3), 0(R_DST) - - // d += length - // s += length - ADD R_LEN, R_DST, R_DST - ADD R_LEN, R_SRC, R_SRC - B loop - -callMemmove: - // if length > len(dst)-d || length > len(src)-s { etc } - CMP R_TMP0, R_LEN - BGT errCorrupt - CMP R_TMP1, R_LEN - BGT errCorrupt - - // copy(dst[d:], src[s:s+length]) - // - // This means calling runtime·memmove(&dst[d], &src[s], length), so we push - // R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those - // three registers to the stack, to save local variables across the CALL. - MOVD R_DST, 8(RSP) - MOVD R_SRC, 16(RSP) - MOVD R_LEN, 24(RSP) - MOVD R_DST, 32(RSP) - MOVD R_SRC, 40(RSP) - MOVD R_LEN, 48(RSP) - MOVD R_OFF, 56(RSP) - CALL runtime·memmove(SB) - - // Restore local variables: unspill registers from the stack and - // re-calculate R_DBASE-R_SEND. - MOVD 32(RSP), R_DST - MOVD 40(RSP), R_SRC - MOVD 48(RSP), R_LEN - MOVD 56(RSP), R_OFF - MOVD dst_base+0(FP), R_DBASE - MOVD dst_len+8(FP), R_DLEN - MOVD R_DBASE, R_DEND - ADD R_DLEN, R_DEND, R_DEND - MOVD src_base+24(FP), R_SBASE - MOVD src_len+32(FP), R_SLEN - MOVD R_SBASE, R_SEND - ADD R_SLEN, R_SEND, R_SEND - - // d += length - // s += length - ADD R_LEN, R_DST, R_DST - ADD R_LEN, R_SRC, R_SRC - B loop - -tagLit60Plus: - // !!! This fragment does the - // - // s += x - 58; if uint(s) > uint(len(src)) { etc } - // - // checks. In the asm version, we code it once instead of once per switch case. - ADD R_LEN, R_SRC, R_SRC - SUB $58, R_SRC, R_SRC - TEST_SRC() - - // case x == 60: - MOVW $61, R1 - CMPW R1, R_LEN - BEQ tagLit61 - BGT tagLit62Plus - - // x = uint32(src[s-1]) - MOVBU -1(R_SRC), R_LEN - B doLit - -tagLit61: - // case x == 61: - // x = uint32(src[s-2]) | uint32(src[s-1])<<8 - MOVHU -2(R_SRC), R_LEN - B doLit - -tagLit62Plus: - CMPW $62, R_LEN - BHI tagLit63 - - // case x == 62: - // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - MOVHU -3(R_SRC), R_LEN - MOVBU -1(R_SRC), R_TMP1 - ORR R_TMP1<<16, R_LEN - B doLit - -tagLit63: - // case x == 63: - // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - MOVWU -4(R_SRC), R_LEN - B doLit - - // The code above handles literal tags. - // ---------------------------------------- - // The code below handles copy tags. - -tagCopy4: - // case tagCopy4: - // s += 5 - ADD $5, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - MOVD R_SRC, R_TMP1 - SUB R_SBASE, R_TMP1, R_TMP1 - CMP R_SLEN, R_TMP1 - BGT errCorrupt - - // length = 1 + int(src[s-5])>>2 - MOVD $1, R1 - ADD R_LEN>>2, R1, R_LEN - - // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - MOVWU -4(R_SRC), R_OFF - B doCopy - -tagCopy2: - // case tagCopy2: - // s += 3 - ADD $3, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - TEST_SRC() - - // length = 1 + int(src[s-3])>>2 - MOVD $1, R1 - ADD R_LEN>>2, R1, R_LEN - - // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - MOVHU -2(R_SRC), R_OFF - B doCopy - -tagCopy: - // We have a copy tag. We assume that: - // - R_TMP1 == src[s] & 0x03 - // - R_LEN == src[s] - CMP $2, R_TMP1 - BEQ tagCopy2 - BGT tagCopy4 - - // case tagCopy1: - // s += 2 - ADD $2, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - TEST_SRC() - - // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - // Calculate offset in R_TMP0 in case it is a repeat. - MOVD R_LEN, R_TMP0 - AND $0xe0, R_TMP0 - MOVBU -1(R_SRC), R_TMP1 - ORR R_TMP0<<3, R_TMP1, R_TMP0 - - // length = 4 + int(src[s-2])>>2&0x7 - MOVD $7, R1 - AND R_LEN>>2, R1, R_LEN - ADD $4, R_LEN, R_LEN - - // check if repeat code with offset 0. - CMP $0, R_TMP0 - BEQ repeatCode - - // This is a regular copy, transfer our temporary value to R_OFF (offset) - MOVD R_TMP0, R_OFF - B doCopy - - // This is a repeat code. -repeatCode: - // If length < 9, reuse last offset, with the length already calculated. - CMP $9, R_LEN - BLT doCopyRepeat - BEQ repeatLen1 - CMP $10, R_LEN - BEQ repeatLen2 - -repeatLen3: - // s +=3 - ADD $3, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - TEST_SRC() - - // length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + 65540 - MOVBU -1(R_SRC), R_TMP0 - MOVHU -3(R_SRC), R_LEN - ORR R_TMP0<<16, R_LEN, R_LEN - ADD $65540, R_LEN, R_LEN - B doCopyRepeat - -repeatLen2: - // s +=2 - ADD $2, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - TEST_SRC() - - // length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + 260 - MOVHU -2(R_SRC), R_LEN - ADD $260, R_LEN, R_LEN - B doCopyRepeat - -repeatLen1: - // s +=1 - ADD $1, R_SRC, R_SRC - - // if uint(s) > uint(len(src)) { etc } - TEST_SRC() - - // length = src[s-1] + 8 - MOVBU -1(R_SRC), R_LEN - ADD $8, R_LEN, R_LEN - B doCopyRepeat - -doCopy: - // This is the end of the outer "switch", when we have a copy tag. - // - // We assume that: - // - R_LEN == length && R_LEN > 0 - // - R_OFF == offset - - // if d < offset { etc } - MOVD R_DST, R_TMP1 - SUB R_DBASE, R_TMP1, R_TMP1 - CMP R_OFF, R_TMP1 - BLT errCorrupt - - // Repeat values can skip the test above, since any offset > 0 will be in dst. -doCopyRepeat: - - // if offset <= 0 { etc } - CMP $0, R_OFF - BLE errCorrupt - - // if length > len(dst)-d { etc } - MOVD R_DEND, R_TMP1 - SUB R_DST, R_TMP1, R_TMP1 - CMP R_TMP1, R_LEN - BGT errCorrupt - - // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length - // - // Set: - // - R_TMP2 = len(dst)-d - // - R_TMP3 = &dst[d-offset] - MOVD R_DEND, R_TMP2 - SUB R_DST, R_TMP2, R_TMP2 - MOVD R_DST, R_TMP3 - SUB R_OFF, R_TMP3, R_TMP3 - - // !!! Try a faster technique for short (16 or fewer bytes) forward copies. - // - // First, try using two 8-byte load/stores, similar to the doLit technique - // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is - // still OK if offset >= 8. Note that this has to be two 8-byte load/stores - // and not one 16-byte load/store, and the first store has to be before the - // second load, due to the overlap if offset is in the range [8, 16). - // - // if length > 16 || offset < 8 || len(dst)-d < 16 { - // goto slowForwardCopy - // } - // copy 16 bytes - // d += length - CMP $16, R_LEN - BGT slowForwardCopy - CMP $8, R_OFF - BLT slowForwardCopy - CMP $16, R_TMP2 - BLT slowForwardCopy - MOVD 0(R_TMP3), R_TMP0 - MOVD R_TMP0, 0(R_DST) - MOVD 8(R_TMP3), R_TMP1 - MOVD R_TMP1, 8(R_DST) - ADD R_LEN, R_DST, R_DST - B loop - -slowForwardCopy: - // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we - // can still try 8-byte load stores, provided we can overrun up to 10 extra - // bytes. As above, the overrun will be fixed up by subsequent iterations - // of the outermost loop. - // - // The C++ snappy code calls this technique IncrementalCopyFastPath. Its - // commentary says: - // - // ---- - // - // The main part of this loop is a simple copy of eight bytes at a time - // until we've copied (at least) the requested amount of bytes. However, - // if d and d-offset are less than eight bytes apart (indicating a - // repeating pattern of length < 8), we first need to expand the pattern in - // order to get the correct results. For instance, if the buffer looks like - // this, with the eight-byte <d-offset> and <d> patterns marked as - // intervals: - // - // abxxxxxxxxxxxx - // [------] d-offset - // [------] d - // - // a single eight-byte copy from <d-offset> to <d> will repeat the pattern - // once, after which we can move <d> two bytes without moving <d-offset>: - // - // ababxxxxxxxxxx - // [------] d-offset - // [------] d - // - // and repeat the exercise until the two no longer overlap. - // - // This allows us to do very well in the special case of one single byte - // repeated many times, without taking a big hit for more general cases. - // - // The worst case of extra writing past the end of the match occurs when - // offset == 1 and length == 1; the last copy will read from byte positions - // [0..7] and write to [4..11], whereas it was only supposed to write to - // position 1. Thus, ten excess bytes. - // - // ---- - // - // That "10 byte overrun" worst case is confirmed by Go's - // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy - // and finishSlowForwardCopy algorithm. - // - // if length > len(dst)-d-10 { - // goto verySlowForwardCopy - // } - SUB $10, R_TMP2, R_TMP2 - CMP R_TMP2, R_LEN - BGT verySlowForwardCopy - - // We want to keep the offset, so we use R_TMP2 from here. - MOVD R_OFF, R_TMP2 - -makeOffsetAtLeast8: - // !!! As above, expand the pattern so that offset >= 8 and we can use - // 8-byte load/stores. - // - // for offset < 8 { - // copy 8 bytes from dst[d-offset:] to dst[d:] - // length -= offset - // d += offset - // offset += offset - // // The two previous lines together means that d-offset, and therefore - // // R_TMP3, is unchanged. - // } - CMP $8, R_TMP2 - BGE fixUpSlowForwardCopy - MOVD (R_TMP3), R_TMP1 - MOVD R_TMP1, (R_DST) - SUB R_TMP2, R_LEN, R_LEN - ADD R_TMP2, R_DST, R_DST - ADD R_TMP2, R_TMP2, R_TMP2 - B makeOffsetAtLeast8 - -fixUpSlowForwardCopy: - // !!! Add length (which might be negative now) to d (implied by R_DST being - // &dst[d]) so that d ends up at the right place when we jump back to the - // top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if - // length is positive, copying the remaining length bytes will write to the - // right place. - MOVD R_DST, R_TMP0 - ADD R_LEN, R_DST, R_DST - -finishSlowForwardCopy: - // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative - // length means that we overrun, but as above, that will be fixed up by - // subsequent iterations of the outermost loop. - MOVD $0, R1 - CMP R1, R_LEN - BLE loop - MOVD (R_TMP3), R_TMP1 - MOVD R_TMP1, (R_TMP0) - ADD $8, R_TMP3, R_TMP3 - ADD $8, R_TMP0, R_TMP0 - SUB $8, R_LEN, R_LEN - B finishSlowForwardCopy - -verySlowForwardCopy: - // verySlowForwardCopy is a simple implementation of forward copy. In C - // parlance, this is a do/while loop instead of a while loop, since we know - // that length > 0. In Go syntax: - // - // for { - // dst[d] = dst[d - offset] - // d++ - // length-- - // if length == 0 { - // break - // } - // } - MOVB (R_TMP3), R_TMP1 - MOVB R_TMP1, (R_DST) - ADD $1, R_TMP3, R_TMP3 - ADD $1, R_DST, R_DST - SUB $1, R_LEN, R_LEN - CBNZ R_LEN, verySlowForwardCopy - B loop - - // The code above handles copy tags. - // ---------------------------------------- - -end: - // This is the end of the "for s < len(src)". - // - // if d != len(dst) { etc } - CMP R_DEND, R_DST - BNE errCorrupt - - // return 0 - MOVD $0, ret+48(FP) - RET - -errCorrupt: - // return decodeErrCodeCorrupt - MOVD $1, R_TMP0 - MOVD R_TMP0, ret+48(FP) - RET diff --git a/vendor/github.com/klauspost/compress/s2/decode_asm.go b/vendor/github.com/klauspost/compress/s2/decode_asm.go deleted file mode 100644 index cb3576edd..000000000 --- a/vendor/github.com/klauspost/compress/s2/decode_asm.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (amd64 || arm64) && !appengine && gc && !noasm -// +build amd64 arm64 -// +build !appengine -// +build gc -// +build !noasm - -package s2 - -// decode has the same semantics as in decode_other.go. -// -//go:noescape -func s2Decode(dst, src []byte) int diff --git a/vendor/github.com/klauspost/compress/s2/decode_other.go b/vendor/github.com/klauspost/compress/s2/decode_other.go deleted file mode 100644 index 2cb55c2c7..000000000 --- a/vendor/github.com/klauspost/compress/s2/decode_other.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (!amd64 && !arm64) || appengine || !gc || noasm -// +build !amd64,!arm64 appengine !gc noasm - -package s2 - -import ( - "fmt" - "strconv" -) - -// decode writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read, and that len(dst) -// equals that length. -// -// It returns 0 on success or a decodeErrCodeXxx error code on failure. -func s2Decode(dst, src []byte) int { - const debug = false - if debug { - fmt.Println("Starting decode, dst len:", len(dst)) - } - var d, s, length int - offset := 0 - - // As long as we can read at least 5 bytes... - for s < len(src)-5 { - // Removing bounds checks is SLOWER, when if doing - // in := src[s:s+5] - // Checked on Go 1.18 - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - x = uint32(src[s-1]) - case x == 61: - in := src[s : s+3] - x = uint32(in[1]) | uint32(in[2])<<8 - s += 3 - case x == 62: - in := src[s : s+4] - // Load as 32 bit and shift down. - x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24 - x >>= 8 - s += 4 - case x == 63: - in := src[s : s+5] - x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24 - s += 5 - } - length = int(x) + 1 - if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) { - if debug { - fmt.Println("corrupt: lit size", length) - } - return decodeErrCodeCorrupt - } - if debug { - fmt.Println("literals, length:", length, "d-after:", d+length) - } - - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - length = int(src[s-2]) >> 2 & 0x7 - if toffset == 0 { - if debug { - fmt.Print("(repeat) ") - } - // keep last offset - switch length { - case 5: - length = int(src[s]) + 4 - s += 1 - case 6: - in := src[s : s+2] - length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8) - s += 2 - case 7: - in := src[s : s+3] - length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16) - s += 3 - default: // 0-> 4 - } - } else { - offset = toffset - } - length += 4 - case tagCopy2: - in := src[s : s+3] - offset = int(uint32(in[1]) | uint32(in[2])<<8) - length = 1 + int(in[0])>>2 - s += 3 - - case tagCopy4: - in := src[s : s+5] - offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24) - length = 1 + int(in[0])>>2 - s += 5 - } - - if offset <= 0 || d < offset || length > len(dst)-d { - if debug { - fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d) - } - - return decodeErrCodeCorrupt - } - - if debug { - fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length) - } - - // Copy from an earlier sub-slice of dst to a later sub-slice. - // If no overlap, use the built-in copy: - if offset > length { - copy(dst[d:d+length], dst[d-offset:]) - d += length - continue - } - - // Unlike the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - // - // We align the slices into a and b and show the compiler they are the same size. - // This allows the loop to run without bounds checks. - a := dst[d : d+length] - b := dst[d-offset:] - b = b[:len(a)] - for i := range a { - a[i] = b[i] - } - d += length - } - - // Remaining with extra checks... - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - length = int(x) + 1 - if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) { - if debug { - fmt.Println("corrupt: lit size", length) - } - return decodeErrCodeCorrupt - } - if debug { - fmt.Println("literals, length:", length, "d-after:", d+length) - } - - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = int(src[s-2]) >> 2 & 0x7 - toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - if toffset == 0 { - if debug { - fmt.Print("(repeat) ") - } - // keep last offset - switch length { - case 5: - s += 1 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-1])) + 4 - case 6: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8) - case 7: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16) - default: // 0-> 4 - } - } else { - offset = toffset - } - length += 4 - case tagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - - case tagCopy4: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-5])>>2 - offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - } - - if offset <= 0 || d < offset || length > len(dst)-d { - if debug { - fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d) - } - return decodeErrCodeCorrupt - } - - if debug { - fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length) - } - - // Copy from an earlier sub-slice of dst to a later sub-slice. - // If no overlap, use the built-in copy: - if offset > length { - copy(dst[d:d+length], dst[d-offset:]) - d += length - continue - } - - // Unlike the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - // - // We align the slices into a and b and show the compiler they are the same size. - // This allows the loop to run without bounds checks. - a := dst[d : d+length] - b := dst[d-offset:] - b = b[:len(a)] - for i := range a { - a[i] = b[i] - } - d += length - } - - if d != len(dst) { - return decodeErrCodeCorrupt - } - return 0 -} diff --git a/vendor/github.com/klauspost/compress/s2/dict.go b/vendor/github.com/klauspost/compress/s2/dict.go deleted file mode 100644 index f125ad096..000000000 --- a/vendor/github.com/klauspost/compress/s2/dict.go +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright (c) 2022+ Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "bytes" - "encoding/binary" - "sync" -) - -const ( - // MinDictSize is the minimum dictionary size when repeat has been read. - MinDictSize = 16 - - // MaxDictSize is the maximum dictionary size when repeat has been read. - MaxDictSize = 65536 - - // MaxDictSrcOffset is the maximum offset where a dictionary entry can start. - MaxDictSrcOffset = 65535 -) - -// Dict contains a dictionary that can be used for encoding and decoding s2 -type Dict struct { - dict []byte - repeat int // Repeat as index of dict - - fast, better, best sync.Once - fastTable *[1 << 14]uint16 - - betterTableShort *[1 << 14]uint16 - betterTableLong *[1 << 17]uint16 - - bestTableShort *[1 << 16]uint32 - bestTableLong *[1 << 19]uint32 -} - -// NewDict will read a dictionary. -// It will return nil if the dictionary is invalid. -func NewDict(dict []byte) *Dict { - if len(dict) == 0 { - return nil - } - var d Dict - // Repeat is the first value of the dict - r, n := binary.Uvarint(dict) - if n <= 0 { - return nil - } - dict = dict[n:] - d.dict = dict - if cap(d.dict) < len(d.dict)+16 { - d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...) - } - if len(dict) < MinDictSize || len(dict) > MaxDictSize { - return nil - } - d.repeat = int(r) - if d.repeat > len(dict) { - return nil - } - return &d -} - -// Bytes will return a serialized version of the dictionary. -// The output can be sent to NewDict. -func (d *Dict) Bytes() []byte { - dst := make([]byte, binary.MaxVarintLen16+len(d.dict)) - return append(dst[:binary.PutUvarint(dst, uint64(d.repeat))], d.dict...) -} - -// MakeDict will create a dictionary. -// 'data' must be at least MinDictSize. -// If data is longer than MaxDictSize only the last MaxDictSize bytes will be used. -// If searchStart is set the start repeat value will be set to the last -// match of this content. -// If no matches are found, it will attempt to find shorter matches. -// This content should match the typical start of a block. -// If at least 4 bytes cannot be matched, repeat is set to start of block. -func MakeDict(data []byte, searchStart []byte) *Dict { - if len(data) == 0 { - return nil - } - if len(data) > MaxDictSize { - data = data[len(data)-MaxDictSize:] - } - var d Dict - dict := data - d.dict = dict - if cap(d.dict) < len(d.dict)+16 { - d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...) - } - if len(dict) < MinDictSize { - return nil - } - - // Find the longest match possible, last entry if multiple. - for s := len(searchStart); s > 4; s-- { - if idx := bytes.LastIndex(data, searchStart[:s]); idx >= 0 && idx <= len(data)-8 { - d.repeat = idx - break - } - } - - return &d -} - -// MakeDictManual will create a dictionary. -// 'data' must be at least MinDictSize and less than or equal to MaxDictSize. -// A manual first repeat index into data must be provided. -// It must be less than len(data)-8. -func MakeDictManual(data []byte, firstIdx uint16) *Dict { - if len(data) < MinDictSize || int(firstIdx) >= len(data)-8 || len(data) > MaxDictSize { - return nil - } - var d Dict - dict := data - d.dict = dict - if cap(d.dict) < len(d.dict)+16 { - d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...) - } - - d.repeat = int(firstIdx) - return &d -} - -// Encode returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func (d *Dict) Encode(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if cap(dst) < n { - dst = make([]byte, n) - } else { - dst = dst[:n] - } - - // The block starts with the varint-encoded length of the decompressed bytes. - dstP := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:dstP] - } - if len(src) < minNonLiteralBlockSize { - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] - } - n := encodeBlockDictGo(dst[dstP:], src, d) - if n > 0 { - dstP += n - return dst[:dstP] - } - // Not compressible - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] -} - -// EncodeBetter returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// EncodeBetter compresses better than Encode but typically with a -// 10-40% speed decrease on both compression and decompression. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func (d *Dict) EncodeBetter(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - dstP := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:dstP] - } - if len(src) < minNonLiteralBlockSize { - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] - } - n := encodeBlockBetterDict(dst[dstP:], src, d) - if n > 0 { - dstP += n - return dst[:dstP] - } - // Not compressible - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] -} - -// EncodeBest returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// EncodeBest compresses as good as reasonably possible but with a -// big speed decrease. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func (d *Dict) EncodeBest(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - dstP := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:dstP] - } - if len(src) < minNonLiteralBlockSize { - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] - } - n := encodeBlockBest(dst[dstP:], src, d) - if n > 0 { - dstP += n - return dst[:dstP] - } - // Not compressible - dstP += emitLiteral(dst[dstP:], src) - return dst[:dstP] -} - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -func (d *Dict) Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if dLen <= cap(dst) { - dst = dst[:dLen] - } else { - dst = make([]byte, dLen) - } - if s2DecodeDict(dst, src[s:], d) != 0 { - return nil, ErrCorrupt - } - return dst, nil -} - -func (d *Dict) initFast() { - d.fast.Do(func() { - const ( - tableBits = 14 - maxTableSize = 1 << tableBits - ) - - var table [maxTableSize]uint16 - // We stop so any entry of length 8 can always be read. - for i := 0; i < len(d.dict)-8-2; i += 3 { - x0 := load64(d.dict, i) - h0 := hash6(x0, tableBits) - h1 := hash6(x0>>8, tableBits) - h2 := hash6(x0>>16, tableBits) - table[h0] = uint16(i) - table[h1] = uint16(i + 1) - table[h2] = uint16(i + 2) - } - d.fastTable = &table - }) -} - -func (d *Dict) initBetter() { - d.better.Do(func() { - const ( - // Long hash matches. - lTableBits = 17 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits - ) - - var lTable [maxLTableSize]uint16 - var sTable [maxSTableSize]uint16 - - // We stop so any entry of length 8 can always be read. - for i := 0; i < len(d.dict)-8; i++ { - cv := load64(d.dict, i) - lTable[hash7(cv, lTableBits)] = uint16(i) - sTable[hash4(cv, sTableBits)] = uint16(i) - } - d.betterTableShort = &sTable - d.betterTableLong = &lTable - }) -} - -func (d *Dict) initBest() { - d.best.Do(func() { - const ( - // Long hash matches. - lTableBits = 19 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 16 - maxSTableSize = 1 << sTableBits - ) - - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 - - // We stop so any entry of length 8 can always be read. - for i := 0; i < len(d.dict)-8; i++ { - cv := load64(d.dict, i) - hashL := hash8(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL := lTable[hashL] - candidateS := sTable[hashS] - lTable[hashL] = uint32(i) | candidateL<<16 - sTable[hashS] = uint32(i) | candidateS<<16 - } - d.bestTableShort = &sTable - d.bestTableLong = &lTable - }) -} diff --git a/vendor/github.com/klauspost/compress/s2/encode.go b/vendor/github.com/klauspost/compress/s2/encode.go deleted file mode 100644 index 20b802270..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode.go +++ /dev/null @@ -1,414 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "encoding/binary" - "math" - "math/bits" - "sync" - - "github.com/klauspost/compress/internal/race" -) - -// Encode returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func Encode(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if cap(dst) < n { - dst = make([]byte, n) - } else { - dst = dst[:n] - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - n := encodeBlock(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -var estblockPool [2]sync.Pool - -// EstimateBlockSize will perform a very fast compression -// without outputting the result and return the compressed output size. -// The function returns -1 if no improvement could be achieved. -// Using actual compression will most often produce better compression than the estimate. -func EstimateBlockSize(src []byte) (d int) { - if len(src) <= inputMargin || int64(len(src)) > 0xffffffff { - return -1 - } - if len(src) <= 1024 { - const sz, pool = 2048, 0 - tmp, ok := estblockPool[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer estblockPool[pool].Put(tmp) - - d = calcBlockSizeSmall(src, tmp) - } else { - const sz, pool = 32768, 1 - tmp, ok := estblockPool[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer estblockPool[pool].Put(tmp) - - d = calcBlockSize(src, tmp) - } - - if d == 0 { - return -1 - } - // Size of the varint encoded block size. - d += (bits.Len64(uint64(len(src))) + 7) / 7 - - if d >= len(src) { - return -1 - } - return d -} - -// EncodeBetter returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// EncodeBetter compresses better than Encode but typically with a -// 10-40% speed decrease on both compression and decompression. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func EncodeBetter(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - n := encodeBlockBetter(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// EncodeBest returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// EncodeBest compresses as good as reasonably possible but with a -// big speed decrease. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func EncodeBest(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - n := encodeBlockBest(dst[d:], src, nil) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// EncodeSnappy returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The output is Snappy compatible and will likely decompress faster. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func EncodeSnappy(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if cap(dst) < n { - dst = make([]byte, n) - } else { - dst = dst[:n] - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - - n := encodeBlockSnappy(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// EncodeSnappyBetter returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The output is Snappy compatible and will likely decompress faster. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func EncodeSnappyBetter(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if cap(dst) < n { - dst = make([]byte, n) - } else { - dst = dst[:n] - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - - n := encodeBlockBetterSnappy(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// EncodeSnappyBest returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The output is Snappy compatible and will likely decompress faster. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -// -// The blocks will require the same amount of memory to decode as encoding, -// and does not make for concurrent decoding. -// Also note that blocks do not contain CRC information, so corruption may be undetected. -// -// If you need to encode larger amounts of data, consider using -// the streaming interface which gives all of these features. -func EncodeSnappyBest(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if cap(dst) < n { - dst = make([]byte, n) - } else { - dst = dst[:n] - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - - n := encodeBlockBestSnappy(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// ConcatBlocks will concatenate the supplied blocks and append them to the supplied destination. -// If the destination is nil or too small, a new will be allocated. -// The blocks are not validated, so garbage in = garbage out. -// dst may not overlap block data. -// Any data in dst is preserved as is, so it will not be considered a block. -func ConcatBlocks(dst []byte, blocks ...[]byte) ([]byte, error) { - totalSize := uint64(0) - compSize := 0 - for _, b := range blocks { - l, hdr, err := decodedLen(b) - if err != nil { - return nil, err - } - totalSize += uint64(l) - compSize += len(b) - hdr - } - if totalSize == 0 { - dst = append(dst, 0) - return dst, nil - } - if totalSize > math.MaxUint32 { - return nil, ErrTooLarge - } - var tmp [binary.MaxVarintLen32]byte - hdrSize := binary.PutUvarint(tmp[:], totalSize) - wantSize := hdrSize + compSize - - if cap(dst)-len(dst) < wantSize { - dst = append(make([]byte, 0, wantSize+len(dst)), dst...) - } - dst = append(dst, tmp[:hdrSize]...) - for _, b := range blocks { - _, hdr, err := decodedLen(b) - if err != nil { - return nil, err - } - dst = append(dst, b[hdr:]...) - } - return dst, nil -} - -// inputMargin is the minimum number of extra input bytes to keep, inside -// encodeBlock's inner loop. On some architectures, this margin lets us -// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) -// literals can be implemented as a single load to and store from a 16-byte -// register. That literal's actual length can be as short as 1 byte, so this -// can copy up to 15 bytes too much, but that's OK as subsequent iterations of -// the encoding loop will fix up the copy overrun, and this inputMargin ensures -// that we don't overrun the dst and src buffers. -const inputMargin = 8 - -// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that -// will be accepted by the encoder. -const minNonLiteralBlockSize = 32 - -const intReduction = 2 - (1 << (^uint(0) >> 63)) // 1 (32 bits) or 0 (64 bits) - -// MaxBlockSize is the maximum value where MaxEncodedLen will return a valid block size. -// Blocks this big are highly discouraged, though. -// Half the size on 32 bit systems. -const MaxBlockSize = (1<<(32-intReduction) - 1) - binary.MaxVarintLen32 - 5 - -// MaxEncodedLen returns the maximum length of a snappy block, given its -// uncompressed length. -// -// It will return a negative value if srcLen is too large to encode. -// 32 bit platforms will have lower thresholds for rejecting big content. -func MaxEncodedLen(srcLen int) int { - n := uint64(srcLen) - if intReduction == 1 { - // 32 bits - if n > math.MaxInt32 { - // Also includes negative. - return -1 - } - } else if n > 0xffffffff { - // 64 bits - // Also includes negative. - return -1 - } - // Size of the varint encoded block size. - n = n + uint64((bits.Len64(n)+7)/7) - - // Add maximum size of encoding block as literals. - n += uint64(literalExtraSize(int64(srcLen))) - if intReduction == 1 { - // 32 bits - if n > math.MaxInt32 { - return -1 - } - } else if n > 0xffffffff { - // 64 bits - // Also includes negative. - return -1 - } - return int(n) -} diff --git a/vendor/github.com/klauspost/compress/s2/encode_all.go b/vendor/github.com/klauspost/compress/s2/encode_all.go deleted file mode 100644 index 997704569..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode_all.go +++ /dev/null @@ -1,1068 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "bytes" - "encoding/binary" - "fmt" - "math/bits" -) - -func load32(b []byte, i int) uint32 { - return binary.LittleEndian.Uint32(b[i:]) -} - -func load64(b []byte, i int) uint64 { - return binary.LittleEndian.Uint64(b[i:]) -} - -// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash6(u uint64, h uint8) uint32 { - const prime6bytes = 227718039650203 - return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) -} - -func encodeGo(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - if len(src) == 0 { - return dst[:d] - } - if len(src) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], src) - return dst[:d] - } - n := encodeBlockGo(dst[d:], src) - if n > 0 { - d += n - return dst[:d] - } - // Not compressible - d += emitLiteral(dst[d:], src) - return dst[:d] -} - -// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockGo(dst, src []byte) (d int) { - // Initialize the hash table. - const ( - tableBits = 14 - maxTableSize = 1 << tableBits - - debug = false - ) - - var table [maxTableSize]uint32 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - repeat := 1 - - for { - candidate := 0 - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - if nextS > sLimit { - goto emitRemainder - } - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - candidate = int(table[hash0]) - candidate2 := int(table[hash1]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - - // Bail if we exceed the maximum size. - if d+(base-nextEmit) > dstLimit { - return 0 - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - if nextEmit > 0 { - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], repeat, s-base) - } else { - // First match, cannot be repeat. - d += emitCopy(dst[d:], repeat, s-base) - } - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - - if uint32(cv) == load32(src, candidate) { - break - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - break - } - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards. - // The top bytes will be rechecked to get the full match. - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopy(dst[d:], repeat, s-base) - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if debug && s == candidate { - panic("s == candidate") - } - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -func encodeBlockSnappyGo(dst, src []byte) (d int) { - // Initialize the hash table. - const ( - tableBits = 14 - maxTableSize = 1 << tableBits - ) - - var table [maxTableSize]uint32 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - repeat := 1 - - for { - candidate := 0 - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - if nextS > sLimit { - goto emitRemainder - } - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - candidate = int(table[hash0]) - candidate2 := int(table[hash1]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - // Bail if we exceed the maximum size. - if d+(base-nextEmit) > dstLimit { - return 0 - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeat(dst[d:], repeat, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - cv = load64(src, s) - continue - } - - if uint32(cv) == load32(src, candidate) { - break - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - break - } - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeat(dst[d:], repeat, s-base) - if false { - // Validate match. - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockDictGo(dst, src []byte, dict *Dict) (d int) { - // Initialize the hash table. - const ( - tableBits = 14 - maxTableSize = 1 << tableBits - maxAhead = 8 // maximum bytes ahead without checking sLimit - - debug = false - ) - dict.initFast() - - var table [maxTableSize]uint32 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - if sLimit > MaxDictSrcOffset-maxAhead { - sLimit = MaxDictSrcOffset - maxAhead - } - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form can start with a dict entry (copy or repeat). - s := 0 - - // Convert dict repeat to offset - repeat := len(dict.dict) - dict.repeat - cv := load64(src, 0) - - // While in dict -searchDict: - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - if nextS > sLimit { - if debug { - fmt.Println("slimit reached", s, nextS) - } - break searchDict - } - candidateDict := int(dict.fastTable[hash0]) - candidateDict2 := int(dict.fastTable[hash1]) - candidate2 := int(table[hash1]) - candidate := int(table[hash0]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - - if repeat > s { - candidate := len(dict.dict) - repeat + s - if repeat-s >= 4 && uint32(cv) == load32(dict.dict, candidate) { - // Extend back - base := s - for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; { - i-- - base-- - } - // Bail if we exceed the maximum size. - if d+(base-nextEmit) > dstLimit { - return 0 - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - if debug && nextEmit != base { - fmt.Println("emitted ", base-nextEmit, "literals") - } - s += 4 - candidate += 4 - for candidate < len(dict.dict)-8 && s <= len(src)-8 { - if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - d += emitRepeat(dst[d:], repeat, s-base) - if debug { - fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - break searchDict - } - cv = load64(src, s) - continue - } - } else if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - if debug && nextEmit != base { - fmt.Println("emitted ", base-nextEmit, "literals") - } - - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - if nextEmit > 0 { - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], repeat, s-base) - } else { - // First match, cannot be repeat. - d += emitCopy(dst[d:], repeat, s-base) - } - - nextEmit = s - if s >= sLimit { - break searchDict - } - if debug { - fmt.Println("emitted reg repeat", s-base, "s:", s) - } - cv = load64(src, s) - continue searchDict - } - if s == 0 { - cv = load64(src, nextS) - s = nextS - continue searchDict - } - // Start with table. These matches will always be closer. - if uint32(cv) == load32(src, candidate) { - goto emitMatch - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - goto emitMatch - } - - // Check dict. Dicts have longer offsets, so we want longer matches. - if cv == load64(dict.dict, candidateDict) { - table[hash2] = uint32(s + 2) - goto emitDict - } - - candidateDict = int(dict.fastTable[hash2]) - // Check if upper 7 bytes match - if candidateDict2 >= 1 { - if cv^load64(dict.dict, candidateDict2-1) < (1 << 8) { - table[hash2] = uint32(s + 2) - candidateDict = candidateDict2 - s++ - goto emitDict - } - } - - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - goto emitMatch - } - if candidateDict >= 2 { - // Check if upper 6 bytes match - if cv^load64(dict.dict, candidateDict-2) < (1 << 16) { - s += 2 - goto emitDict - } - } - - cv = load64(src, nextS) - s = nextS - continue searchDict - - emitDict: - { - if debug { - if load32(dict.dict, candidateDict) != load32(src, s) { - panic("dict emit mismatch") - } - } - // Extend backwards. - // The top bytes will be rechecked to get the full match. - for candidateDict > 0 && s > nextEmit && dict.dict[candidateDict-1] == src[s-1] { - candidateDict-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - if debug && nextEmit != s { - fmt.Println("emitted ", s-nextEmit, "literals") - } - { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = s + (len(dict.dict)) - candidateDict - - // Extend the 4-byte match as long as possible. - s += 4 - candidateDict += 4 - for s <= len(src)-8 && len(dict.dict)-candidateDict >= 8 { - if diff := load64(src, s) ^ load64(dict.dict, candidateDict); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateDict += 8 - } - - // Matches longer than 64 are split. - if s <= sLimit || s-base < 8 { - d += emitCopy(dst[d:], repeat, s-base) - } else { - // Split to ensure we don't start a copy within next block - d += emitCopy(dst[d:], repeat, 4) - d += emitRepeat(dst[d:], repeat, s-base-4) - } - if false { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := dict.dict[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - if debug { - fmt.Println("emitted dict copy, length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - break searchDict - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index and continue loop to try new candidate. - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>8, tableBits) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s - 1) - cv = load64(src, s) - } - continue - } - emitMatch: - - // Extend backwards. - // The top bytes will be rechecked to get the full match. - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - if debug && nextEmit != s { - fmt.Println("emitted ", s-nextEmit, "literals") - } - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopy(dst[d:], repeat, s-base) - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - if debug { - fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - break searchDict - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if debug && s == candidate { - panic("s == candidate") - } - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - - // Search without dict: - if repeat > s { - repeat = 0 - } - - // No more dict - sLimit = len(src) - inputMargin - if s >= sLimit { - goto emitRemainder - } - if debug { - fmt.Println("non-dict matching at", s, "repeat:", repeat) - } - cv = load64(src, s) - if debug { - fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s) - } - for { - candidate := 0 - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - if nextS > sLimit { - goto emitRemainder - } - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - candidate = int(table[hash0]) - candidate2 := int(table[hash1]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - if repeat > 0 && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - // Bail if we exceed the maximum size. - if d+(base-nextEmit) > dstLimit { - return 0 - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - if debug && nextEmit != base { - fmt.Println("emitted ", base-nextEmit, "literals") - } - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - if nextEmit > 0 { - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], repeat, s-base) - } else { - // First match, cannot be repeat. - d += emitCopy(dst[d:], repeat, s-base) - } - if debug { - fmt.Println("emitted src repeat length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - cv = load64(src, s) - continue - } - - if uint32(cv) == load32(src, candidate) { - break - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - break - } - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards. - // The top bytes will be rechecked to get the full match. - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - if debug && nextEmit != s { - fmt.Println("emitted ", s-nextEmit, "literals") - } - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopy(dst[d:], repeat, s-base) - if debug { - // Validate match. - if s <= candidate { - panic("s <= candidate") - } - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - if debug { - fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if debug && s == candidate { - panic("s == candidate") - } - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - if debug && nextEmit != s { - fmt.Println("emitted ", len(src)-nextEmit, "literals") - } - } - return d -} diff --git a/vendor/github.com/klauspost/compress/s2/encode_amd64.go b/vendor/github.com/klauspost/compress/s2/encode_amd64.go deleted file mode 100644 index 7aadd255f..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode_amd64.go +++ /dev/null @@ -1,317 +0,0 @@ -//go:build !appengine && !noasm && gc -// +build !appengine,!noasm,gc - -package s2 - -import ( - "sync" - - "github.com/klauspost/compress/internal/race" -) - -const hasAmd64Asm = true - -var encPools [4]sync.Pool - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlock(dst, src []byte) (d int) { - race.ReadSlice(src) - race.WriteSlice(dst) - - const ( - // Use 12 bit table when less than... - limit12B = 16 << 10 - // Use 10 bit table when less than... - limit10B = 4 << 10 - // Use 8 bit table when less than... - limit8B = 512 - ) - - if len(src) >= 4<<20 { - const sz, pool = 65536, 0 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeBlockAsm(dst, src, tmp) - } - if len(src) >= limit12B { - const sz, pool = 65536, 0 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeBlockAsm4MB(dst, src, tmp) - } - if len(src) >= limit10B { - const sz, pool = 16384, 1 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeBlockAsm12B(dst, src, tmp) - } - if len(src) >= limit8B { - const sz, pool = 4096, 2 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeBlockAsm10B(dst, src, tmp) - } - if len(src) < minNonLiteralBlockSize { - return 0 - } - const sz, pool = 1024, 3 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeBlockAsm8B(dst, src, tmp) -} - -var encBetterPools [5]sync.Pool - -// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBetter(dst, src []byte) (d int) { - race.ReadSlice(src) - race.WriteSlice(dst) - - const ( - // Use 12 bit table when less than... - limit12B = 16 << 10 - // Use 10 bit table when less than... - limit10B = 4 << 10 - // Use 8 bit table when less than... - limit8B = 512 - ) - - if len(src) > 4<<20 { - const sz, pool = 589824, 0 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeBetterBlockAsm(dst, src, tmp) - } - if len(src) >= limit12B { - const sz, pool = 589824, 0 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - - return encodeBetterBlockAsm4MB(dst, src, tmp) - } - if len(src) >= limit10B { - const sz, pool = 81920, 0 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - - return encodeBetterBlockAsm12B(dst, src, tmp) - } - if len(src) >= limit8B { - const sz, pool = 20480, 1 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeBetterBlockAsm10B(dst, src, tmp) - } - if len(src) < minNonLiteralBlockSize { - return 0 - } - - const sz, pool = 5120, 2 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeBetterBlockAsm8B(dst, src, tmp) -} - -// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockSnappy(dst, src []byte) (d int) { - race.ReadSlice(src) - race.WriteSlice(dst) - - const ( - // Use 12 bit table when less than... - limit12B = 16 << 10 - // Use 10 bit table when less than... - limit10B = 4 << 10 - // Use 8 bit table when less than... - limit8B = 512 - ) - if len(src) > 65536 { - const sz, pool = 65536, 0 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeSnappyBlockAsm(dst, src, tmp) - } - if len(src) >= limit12B { - const sz, pool = 65536, 0 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeSnappyBlockAsm64K(dst, src, tmp) - } - if len(src) >= limit10B { - const sz, pool = 16384, 1 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeSnappyBlockAsm12B(dst, src, tmp) - } - if len(src) >= limit8B { - const sz, pool = 4096, 2 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeSnappyBlockAsm10B(dst, src, tmp) - } - if len(src) < minNonLiteralBlockSize { - return 0 - } - const sz, pool = 1024, 3 - tmp, ok := encPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encPools[pool].Put(tmp) - return encodeSnappyBlockAsm8B(dst, src, tmp) -} - -// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBetterSnappy(dst, src []byte) (d int) { - race.ReadSlice(src) - race.WriteSlice(dst) - - const ( - // Use 12 bit table when less than... - limit12B = 16 << 10 - // Use 10 bit table when less than... - limit10B = 4 << 10 - // Use 8 bit table when less than... - limit8B = 512 - ) - if len(src) > 65536 { - const sz, pool = 589824, 0 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeSnappyBetterBlockAsm(dst, src, tmp) - } - - if len(src) >= limit12B { - const sz, pool = 294912, 4 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - - return encodeSnappyBetterBlockAsm64K(dst, src, tmp) - } - if len(src) >= limit10B { - const sz, pool = 81920, 0 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - - return encodeSnappyBetterBlockAsm12B(dst, src, tmp) - } - if len(src) >= limit8B { - const sz, pool = 20480, 1 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeSnappyBetterBlockAsm10B(dst, src, tmp) - } - if len(src) < minNonLiteralBlockSize { - return 0 - } - - const sz, pool = 5120, 2 - tmp, ok := encBetterPools[pool].Get().(*[sz]byte) - if !ok { - tmp = &[sz]byte{} - } - race.WriteSlice(tmp[:]) - defer encBetterPools[pool].Put(tmp) - return encodeSnappyBetterBlockAsm8B(dst, src, tmp) -} diff --git a/vendor/github.com/klauspost/compress/s2/encode_best.go b/vendor/github.com/klauspost/compress/s2/encode_best.go deleted file mode 100644 index 47bac7423..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode_best.go +++ /dev/null @@ -1,796 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "fmt" - "math" - "math/bits" -) - -// encodeBlockBest encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBest(dst, src []byte, dict *Dict) (d int) { - // Initialize the hash tables. - const ( - // Long hash matches. - lTableBits = 19 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 16 - maxSTableSize = 1 << sTableBits - - inputMargin = 8 + 2 - - debug = false - ) - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - if len(src) < minNonLiteralBlockSize { - return 0 - } - sLimitDict := len(src) - inputMargin - if sLimitDict > MaxDictSrcOffset-inputMargin { - sLimitDict = MaxDictSrcOffset - inputMargin - } - - var lTable [maxLTableSize]uint64 - var sTable [maxSTableSize]uint64 - - // Bail if we can't compress to at least this. - dstLimit := len(src) - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - repeat := 1 - if dict != nil { - dict.initBest() - s = 0 - repeat = len(dict.dict) - dict.repeat - } - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - const lowbitMask = 0xffffffff - getCur := func(x uint64) int { - return int(x & lowbitMask) - } - getPrev := func(x uint64) int { - return int(x >> 32) - } - const maxSkip = 64 - - for { - type match struct { - offset int - s int - length int - score int - rep, dict bool - } - var best match - for { - // Next src position to check - nextS := (s-nextEmit)>>8 + 1 - if nextS > maxSkip { - nextS = s + maxSkip - } else { - nextS += s - } - if nextS > sLimit { - goto emitRemainder - } - if dict != nil && s >= MaxDictSrcOffset { - dict = nil - if repeat > s { - repeat = math.MinInt32 - } - } - hashL := hash8(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL := lTable[hashL] - candidateS := sTable[hashS] - - score := func(m match) int { - // Matches that are longer forward are penalized since we must emit it as a literal. - score := m.length - m.s - if nextEmit == m.s { - // If we do not have to emit literals, we save 1 byte - score++ - } - offset := m.s - m.offset - if m.rep { - return score - emitRepeatSize(offset, m.length) - } - return score - emitCopySize(offset, m.length) - } - - matchAt := func(offset, s int, first uint32, rep bool) match { - if best.length != 0 && best.s-best.offset == s-offset { - // Don't retest if we have the same offset. - return match{offset: offset, s: s} - } - if load32(src, offset) != first { - return match{offset: offset, s: s} - } - m := match{offset: offset, s: s, length: 4 + offset, rep: rep} - s += 4 - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[m.length] { - m.length++ - s++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, m.length); diff != 0 { - m.length += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - m.length += 8 - } - m.length -= offset - m.score = score(m) - if m.score <= -m.s { - // Eliminate if no savings, we might find a better one. - m.length = 0 - } - return m - } - matchDict := func(candidate, s int, first uint32, rep bool) match { - if s >= MaxDictSrcOffset { - return match{offset: candidate, s: s} - } - // Calculate offset as if in continuous array with s - offset := -len(dict.dict) + candidate - if best.length != 0 && best.s-best.offset == s-offset && !rep { - // Don't retest if we have the same offset. - return match{offset: offset, s: s} - } - - if load32(dict.dict, candidate) != first { - return match{offset: offset, s: s} - } - m := match{offset: offset, s: s, length: 4 + candidate, rep: rep, dict: true} - s += 4 - if !rep { - for s < sLimitDict && m.length < len(dict.dict) { - if len(src)-s < 8 || len(dict.dict)-m.length < 8 { - if src[s] == dict.dict[m.length] { - m.length++ - s++ - continue - } - break - } - if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 { - m.length += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - m.length += 8 - } - } else { - for s < len(src) && m.length < len(dict.dict) { - if len(src)-s < 8 || len(dict.dict)-m.length < 8 { - if src[s] == dict.dict[m.length] { - m.length++ - s++ - continue - } - break - } - if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 { - m.length += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - m.length += 8 - } - } - m.length -= candidate - m.score = score(m) - if m.score <= -m.s { - // Eliminate if no savings, we might find a better one. - m.length = 0 - } - return m - } - - bestOf := func(a, b match) match { - if b.length == 0 { - return a - } - if a.length == 0 { - return b - } - as := a.score + b.s - bs := b.score + a.s - if as >= bs { - return a - } - return b - } - - if s > 0 { - best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false)) - best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false)) - best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false)) - } - if dict != nil { - candidateL := dict.bestTableLong[hashL] - candidateS := dict.bestTableShort[hashS] - best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false)) - best = bestOf(best, matchDict(int(candidateL>>16), s, uint32(cv), false)) - best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false)) - best = bestOf(best, matchDict(int(candidateS>>16), s, uint32(cv), false)) - } - { - if (dict == nil || repeat <= s) && repeat > 0 { - best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true)) - } else if s-repeat < -4 && dict != nil { - candidate := len(dict.dict) - (repeat - s) - best = bestOf(best, matchDict(candidate, s, uint32(cv), true)) - candidate++ - best = bestOf(best, matchDict(candidate, s+1, uint32(cv>>8), true)) - } - - if best.length > 0 { - hashS := hash4(cv>>8, sTableBits) - // s+1 - nextShort := sTable[hashS] - s := s + 1 - cv := load64(src, s) - hashL := hash8(cv, lTableBits) - nextLong := lTable[hashL] - best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false)) - best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false)) - best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false)) - best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false)) - - // Dict at + 1 - if dict != nil { - candidateL := dict.bestTableLong[hashL] - candidateS := dict.bestTableShort[hashS] - - best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false)) - best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false)) - } - - // s+2 - if true { - hashS := hash4(cv>>8, sTableBits) - - nextShort = sTable[hashS] - s++ - cv = load64(src, s) - hashL := hash8(cv, lTableBits) - nextLong = lTable[hashL] - - if (dict == nil || repeat <= s) && repeat > 0 { - // Repeat at + 2 - best = bestOf(best, matchAt(s-repeat, s, uint32(cv), true)) - } else if repeat-s > 4 && dict != nil { - candidate := len(dict.dict) - (repeat - s) - best = bestOf(best, matchDict(candidate, s, uint32(cv), true)) - } - best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false)) - best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false)) - best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false)) - best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false)) - - // Dict at +2 - // Very small gain - if dict != nil { - candidateL := dict.bestTableLong[hashL] - candidateS := dict.bestTableShort[hashS] - - best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false)) - best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false)) - } - } - // Search for a match at best match end, see if that is better. - // Allow some bytes at the beginning to mismatch. - // Sweet spot is around 1-2 bytes, but depends on input. - // The skipped bytes are tested in Extend backwards, - // and still picked up as part of the match if they do. - const skipBeginning = 2 - const skipEnd = 1 - if sAt := best.s + best.length - skipEnd; sAt < sLimit { - - sBack := best.s + skipBeginning - skipEnd - backL := best.length - skipBeginning - // Load initial values - cv = load64(src, sBack) - - // Grab candidates... - next := lTable[hash8(load64(src, sAt), lTableBits)] - - if checkAt := getCur(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) - } - if checkAt := getPrev(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) - } - // Disabled: Extremely small gain - if false { - next = sTable[hash4(load64(src, sAt), sTableBits)] - if checkAt := getCur(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) - } - if checkAt := getPrev(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false)) - } - } - } - } - } - - // Update table - lTable[hashL] = uint64(s) | candidateL<<32 - sTable[hashS] = uint64(s) | candidateS<<32 - - if best.length > 0 { - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards, not needed for repeats... - s = best.s - if !best.rep && !best.dict { - for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] { - best.offset-- - best.length++ - s-- - } - } - if false && best.offset >= s { - panic(fmt.Errorf("t %d >= s %d", best.offset, s)) - } - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := s - best.offset - s += best.length - - if offset > 65535 && s-base <= 5 && !best.rep { - // Bail if the match is equal or worse to the encoding. - s = best.s + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - if debug && nextEmit != base { - fmt.Println("EMIT", base-nextEmit, "literals. base-after:", base) - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - if best.rep { - if nextEmit > 0 || best.dict { - if debug { - fmt.Println("REPEAT, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best) - } - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], offset, best.length) - } else { - // First match without dict cannot be a repeat. - if debug { - fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best) - } - d += emitCopy(dst[d:], offset, best.length) - } - } else { - if debug { - fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best) - } - d += emitCopy(dst[d:], offset, best.length) - } - repeat = offset - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Fill tables... - for i := best.s + 1; i < s; i++ { - cv0 := load64(src, i) - long0 := hash8(cv0, lTableBits) - short0 := hash4(cv0, sTableBits) - lTable[long0] = uint64(i) | lTable[long0]<<32 - sTable[short0] = uint64(i) | sTable[short0]<<32 - } - cv = load64(src, s) - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - if debug && nextEmit != s { - fmt.Println("emitted ", len(src)-nextEmit, "literals") - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -// encodeBlockBestSnappy encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBestSnappy(dst, src []byte) (d int) { - // Initialize the hash tables. - const ( - // Long hash matches. - lTableBits = 19 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 16 - maxSTableSize = 1 << sTableBits - - inputMargin = 8 + 2 - ) - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - if len(src) < minNonLiteralBlockSize { - return 0 - } - - var lTable [maxLTableSize]uint64 - var sTable [maxSTableSize]uint64 - - // Bail if we can't compress to at least this. - dstLimit := len(src) - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - repeat := 1 - const lowbitMask = 0xffffffff - getCur := func(x uint64) int { - return int(x & lowbitMask) - } - getPrev := func(x uint64) int { - return int(x >> 32) - } - const maxSkip = 64 - - for { - type match struct { - offset int - s int - length int - score int - } - var best match - for { - // Next src position to check - nextS := (s-nextEmit)>>8 + 1 - if nextS > maxSkip { - nextS = s + maxSkip - } else { - nextS += s - } - if nextS > sLimit { - goto emitRemainder - } - hashL := hash8(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL := lTable[hashL] - candidateS := sTable[hashS] - - score := func(m match) int { - // Matches that are longer forward are penalized since we must emit it as a literal. - score := m.length - m.s - if nextEmit == m.s { - // If we do not have to emit literals, we save 1 byte - score++ - } - offset := m.s - m.offset - - return score - emitCopyNoRepeatSize(offset, m.length) - } - - matchAt := func(offset, s int, first uint32) match { - if best.length != 0 && best.s-best.offset == s-offset { - // Don't retest if we have the same offset. - return match{offset: offset, s: s} - } - if load32(src, offset) != first { - return match{offset: offset, s: s} - } - m := match{offset: offset, s: s, length: 4 + offset} - s += 4 - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, m.length); diff != 0 { - m.length += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - m.length += 8 - } - m.length -= offset - m.score = score(m) - if m.score <= -m.s { - // Eliminate if no savings, we might find a better one. - m.length = 0 - } - return m - } - - bestOf := func(a, b match) match { - if b.length == 0 { - return a - } - if a.length == 0 { - return b - } - as := a.score + b.s - bs := b.score + a.s - if as >= bs { - return a - } - return b - } - - best = bestOf(matchAt(getCur(candidateL), s, uint32(cv)), matchAt(getPrev(candidateL), s, uint32(cv))) - best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv))) - best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv))) - - { - best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8))) - if best.length > 0 { - // s+1 - nextShort := sTable[hash4(cv>>8, sTableBits)] - s := s + 1 - cv := load64(src, s) - nextLong := lTable[hash8(cv, lTableBits)] - best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv))) - best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv))) - best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv))) - best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv))) - // Repeat at + 2 - best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8))) - - // s+2 - if true { - nextShort = sTable[hash4(cv>>8, sTableBits)] - s++ - cv = load64(src, s) - nextLong = lTable[hash8(cv, lTableBits)] - best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv))) - best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv))) - best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv))) - best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv))) - } - // Search for a match at best match end, see if that is better. - if sAt := best.s + best.length; sAt < sLimit { - sBack := best.s - backL := best.length - // Load initial values - cv = load64(src, sBack) - // Search for mismatch - next := lTable[hash8(load64(src, sAt), lTableBits)] - //next := sTable[hash4(load64(src, sAt), sTableBits)] - - if checkAt := getCur(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv))) - } - if checkAt := getPrev(next) - backL; checkAt > 0 { - best = bestOf(best, matchAt(checkAt, sBack, uint32(cv))) - } - } - } - } - - // Update table - lTable[hashL] = uint64(s) | candidateL<<32 - sTable[hashS] = uint64(s) | candidateS<<32 - - if best.length > 0 { - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards, not needed for repeats... - s = best.s - if true { - for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] { - best.offset-- - best.length++ - s-- - } - } - if false && best.offset >= s { - panic(fmt.Errorf("t %d >= s %d", best.offset, s)) - } - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := s - best.offset - - s += best.length - - if offset > 65535 && s-base <= 5 { - // Bail if the match is equal or worse to the encoding. - s = best.s + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - d += emitCopyNoRepeat(dst[d:], offset, best.length) - repeat = offset - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Fill tables... - for i := best.s + 1; i < s; i++ { - cv0 := load64(src, i) - long0 := hash8(cv0, lTableBits) - short0 := hash4(cv0, sTableBits) - lTable[long0] = uint64(i) | lTable[long0]<<32 - sTable[short0] = uint64(i) | sTable[short0]<<32 - } - cv = load64(src, s) - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -// emitCopySize returns the size to encode the offset+length -// -// It assumes that: -// -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -func emitCopySize(offset, length int) int { - if offset >= 65536 { - i := 0 - if length > 64 { - length -= 64 - if length >= 4 { - // Emit remaining as repeats - return 5 + emitRepeatSize(offset, length) - } - i = 5 - } - if length == 0 { - return i - } - return i + 5 - } - - // Offset no more than 2 bytes. - if length > 64 { - if offset < 2048 { - // Emit 8 bytes, then rest as repeats... - return 2 + emitRepeatSize(offset, length-8) - } - // Emit remaining as repeats, at least 4 bytes remain. - return 3 + emitRepeatSize(offset, length-60) - } - if length >= 12 || offset >= 2048 { - return 3 - } - // Emit the remaining copy, encoded as 2 bytes. - return 2 -} - -// emitCopyNoRepeatSize returns the size to encode the offset+length -// -// It assumes that: -// -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -func emitCopyNoRepeatSize(offset, length int) int { - if offset >= 65536 { - return 5 + 5*(length/64) - } - - // Offset no more than 2 bytes. - if length > 64 { - // Emit remaining as repeats, at least 4 bytes remain. - return 3 + 3*(length/60) - } - if length >= 12 || offset >= 2048 { - return 3 - } - // Emit the remaining copy, encoded as 2 bytes. - return 2 -} - -// emitRepeatSize returns the number of bytes required to encode a repeat. -// Length must be at least 4 and < 1<<24 -func emitRepeatSize(offset, length int) int { - // Repeat offset, make length cheaper - if length <= 4+4 || (length < 8+4 && offset < 2048) { - return 2 - } - if length < (1<<8)+4+4 { - return 3 - } - if length < (1<<16)+(1<<8)+4 { - return 4 - } - const maxRepeat = (1 << 24) - 1 - length -= (1 << 16) - 4 - left := 0 - if length > maxRepeat { - left = length - maxRepeat + 4 - } - if left > 0 { - return 5 + emitRepeatSize(offset, left) - } - return 5 -} diff --git a/vendor/github.com/klauspost/compress/s2/encode_better.go b/vendor/github.com/klauspost/compress/s2/encode_better.go deleted file mode 100644 index 544cb1e17..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode_better.go +++ /dev/null @@ -1,1106 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "bytes" - "fmt" - "math/bits" -) - -// hash4 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4(u uint64, h uint8) uint32 { - const prime4bytes = 2654435761 - return (uint32(u) * prime4bytes) >> ((32 - h) & 31) -} - -// hash5 returns the hash of the lowest 5 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash5(u uint64, h uint8) uint32 { - const prime5bytes = 889523592379 - return uint32(((u << (64 - 40)) * prime5bytes) >> ((64 - h) & 63)) -} - -// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash7(u uint64, h uint8) uint32 { - const prime7bytes = 58295818150454627 - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) -} - -// hash8 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash8(u uint64, h uint8) uint32 { - const prime8bytes = 0xcf1bbcdcb7a56463 - return uint32((u * prime8bytes) >> ((64 - h) & 63)) -} - -// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBetterGo(dst, src []byte) (d int) { - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - if len(src) < minNonLiteralBlockSize { - return 0 - } - - // Initialize the hash tables. - const ( - // Long hash matches. - lTableBits = 17 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits - ) - - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 6 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We initialize repeat to 0, so we never match on first attempt - repeat := 0 - - for { - candidateL := 0 - nextS := 0 - for { - // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 - if nextS > sLimit { - goto emitRemainder - } - hashL := hash7(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL = int(lTable[hashL]) - candidateS := int(sTable[hashS]) - lTable[hashL] = uint32(s) - sTable[hashS] = uint32(s) - - valLong := load64(src, candidateL) - valShort := load64(src, candidateS) - - // If long matches at least 8 bytes, use that. - if cv == valLong { - break - } - if cv == valShort { - candidateL = candidateS - break - } - - // Check repeat at offset checkRep. - const checkRep = 1 - // Minimum length of a repeat. Tested with various values. - // While 4-5 offers improvements in some, 6 reduces - // regressions significantly. - const wantRepeatBytes = 6 - const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep) - if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + wantRepeatBytes + checkRep - s += wantRepeatBytes + checkRep - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidate] { - s++ - candidate++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], repeat, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - // Index in-between - index0 := base + 1 - index1 := s - 2 - - for index0 < index1 { - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 2 - index1 -= 2 - } - - cv = load64(src, s) - continue - } - - // Long likely matches 7, so take that. - if uint32(cv) == uint32(valLong) { - break - } - - // Check our short candidate - if uint32(cv) == uint32(valShort) { - // Try a long candidate at s+1 - hashL = hash7(cv>>8, lTableBits) - candidateL = int(lTable[hashL]) - lTable[hashL] = uint32(s + 1) - if uint32(cv>>8) == load32(src, candidateL) { - s++ - break - } - // Use our short candidate. - candidateL = candidateS - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] { - candidateL-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := base - candidateL - - // Extend the 4-byte match as long as possible. - s += 4 - candidateL += 4 - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidateL] { - s++ - candidateL++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateL += 8 - } - - if offset > 65535 && s-base <= 5 && repeat != offset { - // Bail if the match is equal or worse to the encoding. - s = nextS + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - if repeat == offset { - d += emitRepeat(dst[d:], offset, s-base) - } else { - d += emitCopy(dst[d:], offset, s-base) - repeat = offset - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index short & long - index0 := base + 1 - index1 := s - 2 - - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - // lTable could be postponed, but very minor difference. - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 1 - index1 -= 1 - cv = load64(src, s) - - // Index large values sparsely in between. - // We do two starting from different offsets for speed. - index2 := (index0 + index1 + 1) >> 1 - for index2 < index1 { - lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0) - lTable[hash7(load64(src, index2), lTableBits)] = uint32(index2) - index0 += 2 - index2 += 2 - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -// encodeBlockBetterSnappyGo encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBetterSnappyGo(dst, src []byte) (d int) { - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - if len(src) < minNonLiteralBlockSize { - return 0 - } - - // Initialize the hash tables. - const ( - // Long hash matches. - lTableBits = 16 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits - ) - - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 6 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We initialize repeat to 0, so we never match on first attempt - repeat := 0 - const maxSkip = 100 - - for { - candidateL := 0 - nextS := 0 - for { - // Next src position to check - nextS = (s-nextEmit)>>7 + 1 - if nextS > maxSkip { - nextS = s + maxSkip - } else { - nextS += s - } - - if nextS > sLimit { - goto emitRemainder - } - hashL := hash7(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL = int(lTable[hashL]) - candidateS := int(sTable[hashS]) - lTable[hashL] = uint32(s) - sTable[hashS] = uint32(s) - - if uint32(cv) == load32(src, candidateL) { - break - } - - // Check our short candidate - if uint32(cv) == load32(src, candidateS) { - // Try a long candidate at s+1 - hashL = hash7(cv>>8, lTableBits) - candidateL = int(lTable[hashL]) - lTable[hashL] = uint32(s + 1) - if uint32(cv>>8) == load32(src, candidateL) { - s++ - break - } - // Use our short candidate. - candidateL = candidateS - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] { - candidateL-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := base - candidateL - - // Extend the 4-byte match as long as possible. - s += 4 - candidateL += 4 - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidateL] { - s++ - candidateL++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateL += 8 - } - - if offset > 65535 && s-base <= 5 && repeat != offset { - // Bail if the match is equal or worse to the encoding. - s = nextS + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - d += emitCopyNoRepeat(dst[d:], offset, s-base) - repeat = offset - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index short & long - index0 := base + 1 - index1 := s - 2 - - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 1 - index1 -= 1 - cv = load64(src, s) - - // Index large values sparsely in between. - // We do two starting from different offsets for speed. - index2 := (index0 + index1 + 1) >> 1 - for index2 < index1 { - lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0) - lTable[hash7(load64(src, index2), lTableBits)] = uint32(index2) - index0 += 2 - index2 += 2 - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} - -// encodeBlockBetterDict encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlockBetterDict(dst, src []byte, dict *Dict) (d int) { - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - // Initialize the hash tables. - const ( - // Long hash matches. - lTableBits = 17 - maxLTableSize = 1 << lTableBits - - // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits - - maxAhead = 8 // maximum bytes ahead without checking sLimit - - debug = false - ) - - sLimit := len(src) - inputMargin - if sLimit > MaxDictSrcOffset-maxAhead { - sLimit = MaxDictSrcOffset - maxAhead - } - if len(src) < minNonLiteralBlockSize { - return 0 - } - - dict.initBetter() - - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 6 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 0 - cv := load64(src, s) - - // We initialize repeat to 0, so we never match on first attempt - repeat := len(dict.dict) - dict.repeat - - // While in dict -searchDict: - for { - candidateL := 0 - nextS := 0 - for { - // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 - if nextS > sLimit { - break searchDict - } - hashL := hash7(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL = int(lTable[hashL]) - candidateS := int(sTable[hashS]) - dictL := int(dict.betterTableLong[hashL]) - dictS := int(dict.betterTableShort[hashS]) - lTable[hashL] = uint32(s) - sTable[hashS] = uint32(s) - - valLong := load64(src, candidateL) - valShort := load64(src, candidateS) - - // If long matches at least 8 bytes, use that. - if s != 0 { - if cv == valLong { - goto emitMatch - } - if cv == valShort { - candidateL = candidateS - goto emitMatch - } - } - - // Check dict repeat. - if repeat >= s+4 { - candidate := len(dict.dict) - repeat + s - if candidate > 0 && uint32(cv) == load32(dict.dict, candidate) { - // Extend back - base := s - for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - if debug && nextEmit != base { - fmt.Println("emitted ", base-nextEmit, "literals") - } - s += 4 - candidate += 4 - for candidate < len(dict.dict)-8 && s <= len(src)-8 { - if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - d += emitRepeat(dst[d:], repeat, s-base) - if debug { - fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s) - } - nextEmit = s - if s >= sLimit { - break searchDict - } - // Index in-between - index0 := base + 1 - index1 := s - 2 - - cv = load64(src, s) - for index0 < index1 { - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 2 - index1 -= 2 - } - continue - } - } - // Don't try to find match at s==0 - if s == 0 { - cv = load64(src, nextS) - s = nextS - continue - } - - // Long likely matches 7, so take that. - if uint32(cv) == uint32(valLong) { - goto emitMatch - } - - // Long dict... - if uint32(cv) == load32(dict.dict, dictL) { - candidateL = dictL - goto emitDict - } - - // Check our short candidate - if uint32(cv) == uint32(valShort) { - // Try a long candidate at s+1 - hashL = hash7(cv>>8, lTableBits) - candidateL = int(lTable[hashL]) - lTable[hashL] = uint32(s + 1) - if uint32(cv>>8) == load32(src, candidateL) { - s++ - goto emitMatch - } - // Use our short candidate. - candidateL = candidateS - goto emitMatch - } - if uint32(cv) == load32(dict.dict, dictS) { - // Try a long candidate at s+1 - hashL = hash7(cv>>8, lTableBits) - candidateL = int(lTable[hashL]) - lTable[hashL] = uint32(s + 1) - if uint32(cv>>8) == load32(src, candidateL) { - s++ - goto emitMatch - } - candidateL = dictS - goto emitDict - } - cv = load64(src, nextS) - s = nextS - } - emitDict: - { - if debug { - if load32(dict.dict, candidateL) != load32(src, s) { - panic("dict emit mismatch") - } - } - // Extend backwards. - // The top bytes will be rechecked to get the full match. - for candidateL > 0 && s > nextEmit && dict.dict[candidateL-1] == src[s-1] { - candidateL-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteral(dst[d:], src[nextEmit:s]) - if debug && nextEmit != s { - fmt.Println("emitted ", s-nextEmit, "literals") - } - { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - offset := s + (len(dict.dict)) - candidateL - - // Extend the 4-byte match as long as possible. - s += 4 - candidateL += 4 - for s <= len(src)-8 && len(dict.dict)-candidateL >= 8 { - if diff := load64(src, s) ^ load64(dict.dict, candidateL); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateL += 8 - } - - if repeat == offset { - if debug { - fmt.Println("emitted dict repeat, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL) - } - d += emitRepeat(dst[d:], offset, s-base) - } else { - if debug { - fmt.Println("emitted dict copy, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL) - } - // Matches longer than 64 are split. - if s <= sLimit || s-base < 8 { - d += emitCopy(dst[d:], offset, s-base) - } else { - // Split to ensure we don't start a copy within next block. - d += emitCopy(dst[d:], offset, 4) - d += emitRepeat(dst[d:], offset, s-base-4) - } - repeat = offset - } - if false { - // Validate match. - if s <= candidateL { - panic("s <= candidate") - } - a := src[base:s] - b := dict.dict[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - nextEmit = s - if s >= sLimit { - break searchDict - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index short & long - index0 := base + 1 - index1 := s - 2 - - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 1 - index1 -= 1 - cv = load64(src, s) - - // index every second long in between. - for index0 < index1 { - lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0) - lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1) - index0 += 2 - index1 -= 2 - } - } - continue - } - emitMatch: - - // Extend backwards - for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] { - candidateL-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := base - candidateL - - // Extend the 4-byte match as long as possible. - s += 4 - candidateL += 4 - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidateL] { - s++ - candidateL++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateL += 8 - } - - if offset > 65535 && s-base <= 5 && repeat != offset { - // Bail if the match is equal or worse to the encoding. - s = nextS + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - if debug && nextEmit != s { - fmt.Println("emitted ", s-nextEmit, "literals") - } - if repeat == offset { - if debug { - fmt.Println("emitted match repeat, length", s-base, "offset:", offset, "s:", s) - } - d += emitRepeat(dst[d:], offset, s-base) - } else { - if debug { - fmt.Println("emitted match copy, length", s-base, "offset:", offset, "s:", s) - } - d += emitCopy(dst[d:], offset, s-base) - repeat = offset - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index short & long - index0 := base + 1 - index1 := s - 2 - - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 1 - index1 -= 1 - cv = load64(src, s) - - // Index large values sparsely in between. - // We do two starting from different offsets for speed. - index2 := (index0 + index1 + 1) >> 1 - for index2 < index1 { - lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0) - lTable[hash7(load64(src, index2), lTableBits)] = uint32(index2) - index0 += 2 - index2 += 2 - } - } - - // Search without dict: - if repeat > s { - repeat = 0 - } - - // No more dict - sLimit = len(src) - inputMargin - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - if debug { - fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s) - } - for { - candidateL := 0 - nextS := 0 - for { - // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 - if nextS > sLimit { - goto emitRemainder - } - hashL := hash7(cv, lTableBits) - hashS := hash4(cv, sTableBits) - candidateL = int(lTable[hashL]) - candidateS := int(sTable[hashS]) - lTable[hashL] = uint32(s) - sTable[hashS] = uint32(s) - - valLong := load64(src, candidateL) - valShort := load64(src, candidateS) - - // If long matches at least 8 bytes, use that. - if cv == valLong { - break - } - if cv == valShort { - candidateL = candidateS - break - } - - // Check repeat at offset checkRep. - const checkRep = 1 - // Minimum length of a repeat. Tested with various values. - // While 4-5 offers improvements in some, 6 reduces - // regressions significantly. - const wantRepeatBytes = 6 - const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep) - if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteral(dst[d:], src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + wantRepeatBytes + checkRep - s += wantRepeatBytes + checkRep - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidate] { - s++ - candidate++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - // same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset. - d += emitRepeat(dst[d:], repeat, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - // Index in-between - index0 := base + 1 - index1 := s - 2 - - for index0 < index1 { - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 2 - index1 -= 2 - } - - cv = load64(src, s) - continue - } - - // Long likely matches 7, so take that. - if uint32(cv) == uint32(valLong) { - break - } - - // Check our short candidate - if uint32(cv) == uint32(valShort) { - // Try a long candidate at s+1 - hashL = hash7(cv>>8, lTableBits) - candidateL = int(lTable[hashL]) - lTable[hashL] = uint32(s + 1) - if uint32(cv>>8) == load32(src, candidateL) { - s++ - break - } - // Use our short candidate. - candidateL = candidateS - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] { - candidateL-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - base := s - offset := base - candidateL - - // Extend the 4-byte match as long as possible. - s += 4 - candidateL += 4 - for s < len(src) { - if len(src)-s < 8 { - if src[s] == src[candidateL] { - s++ - candidateL++ - continue - } - break - } - if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidateL += 8 - } - - if offset > 65535 && s-base <= 5 && repeat != offset { - // Bail if the match is equal or worse to the encoding. - s = nextS + 1 - if s >= sLimit { - goto emitRemainder - } - cv = load64(src, s) - continue - } - - d += emitLiteral(dst[d:], src[nextEmit:base]) - if repeat == offset { - d += emitRepeat(dst[d:], offset, s-base) - } else { - d += emitCopy(dst[d:], offset, s-base) - repeat = offset - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - - // Index short & long - index0 := base + 1 - index1 := s - 2 - - cv0 := load64(src, index0) - cv1 := load64(src, index1) - lTable[hash7(cv0, lTableBits)] = uint32(index0) - sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1) - - lTable[hash7(cv1, lTableBits)] = uint32(index1) - sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1) - index0 += 1 - index1 -= 1 - cv = load64(src, s) - - // Index large values sparsely in between. - // We do two starting from different offsets for speed. - index2 := (index0 + index1 + 1) >> 1 - for index2 < index1 { - lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0) - lTable[hash7(load64(src, index2), lTableBits)] = uint32(index2) - index0 += 2 - index2 += 2 - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} diff --git a/vendor/github.com/klauspost/compress/s2/encode_go.go b/vendor/github.com/klauspost/compress/s2/encode_go.go deleted file mode 100644 index dd1c973ca..000000000 --- a/vendor/github.com/klauspost/compress/s2/encode_go.go +++ /dev/null @@ -1,729 +0,0 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - -package s2 - -import ( - "bytes" - "math/bits" -) - -const hasAmd64Asm = false - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) -func encodeBlock(dst, src []byte) (d int) { - if len(src) < minNonLiteralBlockSize { - return 0 - } - return encodeBlockGo(dst, src) -} - -// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) -func encodeBlockBetter(dst, src []byte) (d int) { - return encodeBlockBetterGo(dst, src) -} - -// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) -func encodeBlockBetterSnappy(dst, src []byte) (d int) { - return encodeBlockBetterSnappyGo(dst, src) -} - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// -// len(dst) >= MaxEncodedLen(len(src)) -func encodeBlockSnappy(dst, src []byte) (d int) { - if len(src) < minNonLiteralBlockSize { - return 0 - } - return encodeBlockSnappyGo(dst, src) -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 0 <= len(lit) && len(lit) <= math.MaxUint32 -func emitLiteral(dst, lit []byte) int { - if len(lit) == 0 { - return 0 - } - const num = 63<<2 | tagLiteral - i, n := 0, uint(len(lit)-1) - switch { - case n < 60: - dst[0] = uint8(n)<<2 | tagLiteral - i = 1 - case n < 1<<8: - dst[1] = uint8(n) - dst[0] = 60<<2 | tagLiteral - i = 2 - case n < 1<<16: - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 61<<2 | tagLiteral - i = 3 - case n < 1<<24: - dst[3] = uint8(n >> 16) - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 62<<2 | tagLiteral - i = 4 - default: - dst[4] = uint8(n >> 24) - dst[3] = uint8(n >> 16) - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 63<<2 | tagLiteral - i = 5 - } - return i + copy(dst[i:], lit) -} - -// emitRepeat writes a repeat chunk and returns the number of bytes written. -// Length must be at least 4 and < 1<<24 -func emitRepeat(dst []byte, offset, length int) int { - // Repeat offset, make length cheaper - length -= 4 - if length <= 4 { - dst[0] = uint8(length)<<2 | tagCopy1 - dst[1] = 0 - return 2 - } - if length < 8 && offset < 2048 { - // Encode WITH offset - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 - return 2 - } - if length < (1<<8)+4 { - length -= 4 - dst[2] = uint8(length) - dst[1] = 0 - dst[0] = 5<<2 | tagCopy1 - return 3 - } - if length < (1<<16)+(1<<8) { - length -= 1 << 8 - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 6<<2 | tagCopy1 - return 4 - } - const maxRepeat = (1 << 24) - 1 - length -= 1 << 16 - left := 0 - if length > maxRepeat { - left = length - maxRepeat + 4 - length = maxRepeat - 4 - } - dst[4] = uint8(length >> 16) - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 7<<2 | tagCopy1 - if left > 0 { - return 5 + emitRepeat(dst[5:], offset, left) - } - return 5 -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -func emitCopy(dst []byte, offset, length int) int { - if offset >= 65536 { - i := 0 - if length > 64 { - // Emit a length 64 copy, encoded as 5 bytes. - dst[4] = uint8(offset >> 24) - dst[3] = uint8(offset >> 16) - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 63<<2 | tagCopy4 - length -= 64 - if length >= 4 { - // Emit remaining as repeats - return 5 + emitRepeat(dst[5:], offset, length) - } - i = 5 - } - if length == 0 { - return i - } - // Emit a copy, offset encoded as 4 bytes. - dst[i+0] = uint8(length-1)<<2 | tagCopy4 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - dst[i+3] = uint8(offset >> 16) - dst[i+4] = uint8(offset >> 24) - return i + 5 - } - - // Offset no more than 2 bytes. - if length > 64 { - off := 3 - if offset < 2048 { - // emit 8 bytes as tagCopy1, rest as repeats. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 - length -= 8 - off = 2 - } else { - // Emit a length 60 copy, encoded as 3 bytes. - // Emit remaining as repeat value (minimum 4 bytes). - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 59<<2 | tagCopy2 - length -= 60 - } - // Emit remaining as repeats, at least 4 bytes remain. - return off + emitRepeat(dst[off:], offset, length) - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = uint8(length-1)<<2 | tagCopy2 - return 3 - } - // Emit the remaining copy, encoded as 2 bytes. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - return 2 -} - -// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -func emitCopyNoRepeat(dst []byte, offset, length int) int { - if offset >= 65536 { - i := 0 - if length > 64 { - // Emit a length 64 copy, encoded as 5 bytes. - dst[4] = uint8(offset >> 24) - dst[3] = uint8(offset >> 16) - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 63<<2 | tagCopy4 - length -= 64 - if length >= 4 { - // Emit remaining as repeats - return 5 + emitCopyNoRepeat(dst[5:], offset, length) - } - i = 5 - } - if length == 0 { - return i - } - // Emit a copy, offset encoded as 4 bytes. - dst[i+0] = uint8(length-1)<<2 | tagCopy4 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - dst[i+3] = uint8(offset >> 16) - dst[i+4] = uint8(offset >> 24) - return i + 5 - } - - // Offset no more than 2 bytes. - if length > 64 { - // Emit a length 60 copy, encoded as 3 bytes. - // Emit remaining as repeat value (minimum 4 bytes). - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 59<<2 | tagCopy2 - length -= 60 - // Emit remaining as repeats, at least 4 bytes remain. - return 3 + emitCopyNoRepeat(dst[3:], offset, length) - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = uint8(length-1)<<2 | tagCopy2 - return 3 - } - // Emit the remaining copy, encoded as 2 bytes. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - return 2 -} - -// matchLen returns how many bytes match in a and b -// -// It assumes that: -// -// len(a) <= len(b) -func matchLen(a []byte, b []byte) int { - b = b[:len(a)] - var checked int - if len(a) > 4 { - // Try 4 bytes first - if diff := load32(a, 0) ^ load32(b, 0); diff != 0 { - return bits.TrailingZeros32(diff) >> 3 - } - // Switch to 8 byte matching. - checked = 4 - a = a[4:] - b = b[4:] - for len(a) >= 8 { - b = b[:len(a)] - if diff := load64(a, 0) ^ load64(b, 0); diff != 0 { - return checked + (bits.TrailingZeros64(diff) >> 3) - } - checked += 8 - a = a[8:] - b = b[8:] - } - } - b = b[:len(a)] - for i := range a { - if a[i] != b[i] { - return int(i) + checked - } - } - return len(a) + checked -} - -// input must be > inputMargin -func calcBlockSize(src []byte, _ *[32768]byte) (d int) { - // Initialize the hash table. - const ( - tableBits = 13 - maxTableSize = 1 << tableBits - ) - - var table [maxTableSize]uint32 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - repeat := 1 - - for { - candidate := 0 - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - if nextS > sLimit { - goto emitRemainder - } - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - candidate = int(table[hash0]) - candidate2 := int(table[hash1]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteralSize(src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeatSize(repeat, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - cv = load64(src, s) - continue - } - - if uint32(cv) == load32(src, candidate) { - break - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - break - } - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteralSize(src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeatSize(repeat, s-base) - if false { - // Validate match. - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteralSize(src[nextEmit:]) - } - return d -} - -// length must be > inputMargin. -func calcBlockSizeSmall(src []byte, _ *[2048]byte) (d int) { - // Initialize the hash table. - const ( - tableBits = 9 - maxTableSize = 1 << tableBits - ) - - var table [maxTableSize]uint32 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // Bail if we can't compress to at least this. - dstLimit := len(src) - len(src)>>5 - 5 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - cv := load64(src, s) - - // We search for a repeat at -1, but don't output repeats when nextEmit == 0 - repeat := 1 - - for { - candidate := 0 - for { - // Next src position to check - nextS := s + (s-nextEmit)>>6 + 4 - if nextS > sLimit { - goto emitRemainder - } - hash0 := hash6(cv, tableBits) - hash1 := hash6(cv>>8, tableBits) - candidate = int(table[hash0]) - candidate2 := int(table[hash1]) - table[hash0] = uint32(s) - table[hash1] = uint32(s + 1) - hash2 := hash6(cv>>16, tableBits) - - // Check repeat at offset checkRep. - const checkRep = 1 - if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { - base := s + checkRep - // Extend back - for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { - i-- - base-- - } - d += emitLiteralSize(src[nextEmit:base]) - - // Extend forward - candidate := s - repeat + 4 + checkRep - s += 4 + checkRep - for s <= sLimit { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeatSize(repeat, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - cv = load64(src, s) - continue - } - - if uint32(cv) == load32(src, candidate) { - break - } - candidate = int(table[hash2]) - if uint32(cv>>8) == load32(src, candidate2) { - table[hash2] = uint32(s + 2) - candidate = candidate2 - s++ - break - } - table[hash2] = uint32(s + 2) - if uint32(cv>>16) == load32(src, candidate) { - s += 2 - break - } - - cv = load64(src, nextS) - s = nextS - } - - // Extend backwards - for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] { - candidate-- - s-- - } - - // Bail if we exceed the maximum size. - if d+(s-nextEmit) > dstLimit { - return 0 - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - d += emitLiteralSize(src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - repeat = base - candidate - - // Extend the 4-byte match as long as possible. - s += 4 - candidate += 4 - for s <= len(src)-8 { - if diff := load64(src, s) ^ load64(src, candidate); diff != 0 { - s += bits.TrailingZeros64(diff) >> 3 - break - } - s += 8 - candidate += 8 - } - - d += emitCopyNoRepeatSize(repeat, s-base) - if false { - // Validate match. - a := src[base:s] - b := src[base-repeat : base-repeat+(s-base)] - if !bytes.Equal(a, b) { - panic("mismatch") - } - } - - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - if d > dstLimit { - // Do we have space for more, if not bail. - return 0 - } - // Check for an immediate match, otherwise start search at s+1 - x := load64(src, s-2) - m2Hash := hash6(x, tableBits) - currHash := hash6(x>>16, tableBits) - candidate = int(table[currHash]) - table[m2Hash] = uint32(s - 2) - table[currHash] = uint32(s) - if uint32(x>>16) != load32(src, candidate) { - cv = load64(src, s+1) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - // Bail if we exceed the maximum size. - if d+len(src)-nextEmit > dstLimit { - return 0 - } - d += emitLiteralSize(src[nextEmit:]) - } - return d -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 0 <= len(lit) && len(lit) <= math.MaxUint32 -func emitLiteralSize(lit []byte) int { - if len(lit) == 0 { - return 0 - } - switch { - case len(lit) <= 60: - return len(lit) + 1 - case len(lit) <= 1<<8: - return len(lit) + 2 - case len(lit) <= 1<<16: - return len(lit) + 3 - case len(lit) <= 1<<24: - return len(lit) + 4 - default: - return len(lit) + 5 - } -} - -func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) { - panic("cvtLZ4BlockAsm should be unreachable") -} - -func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) { - panic("cvtLZ4BlockSnappyAsm should be unreachable") -} - -func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) { - panic("cvtLZ4sBlockAsm should be unreachable") -} - -func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) { - panic("cvtLZ4sBlockSnappyAsm should be unreachable") -} diff --git a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go deleted file mode 100644 index f43aa8154..000000000 --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go +++ /dev/null @@ -1,228 +0,0 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. - -//go:build !appengine && !noasm && gc && !noasm - -package s2 - -func _dummy_() - -// encodeBlockAsm encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4294967295 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int - -// encodeBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4194304 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int - -// encodeBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 16383 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int - -// encodeBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4095 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int - -// encodeBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 511 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int - -// encodeBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4294967295 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int - -// encodeBetterBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4194304 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int - -// encodeBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 16383 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int - -// encodeBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4095 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int - -// encodeBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 511 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int - -// encodeSnappyBlockAsm encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4294967295 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int - -// encodeSnappyBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 65535 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int - -// encodeSnappyBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 16383 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int - -// encodeSnappyBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4095 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int - -// encodeSnappyBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 511 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int - -// encodeSnappyBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4294967295 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int - -// encodeSnappyBetterBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 65535 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int - -// encodeSnappyBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 16383 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int - -// encodeSnappyBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4095 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int - -// encodeSnappyBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 511 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int - -// calcBlockSize encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 4294967295 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func calcBlockSize(src []byte, tmp *[32768]byte) int - -// calcBlockSizeSmall encodes a non-empty src to a guaranteed-large-enough dst. -// Maximum input 1024 bytes. -// It assumes that the varint-encoded length of the decompressed bytes has already been written. -// -//go:noescape -func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes with margin of 0 bytes -// 0 <= len(lit) && len(lit) <= math.MaxUint32 -// -//go:noescape -func emitLiteral(dst []byte, lit []byte) int - -// emitRepeat writes a repeat chunk and returns the number of bytes written. -// Length must be at least 4 and < 1<<32 -// -//go:noescape -func emitRepeat(dst []byte, offset int, length int) int - -// emitCopy writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -// -//go:noescape -func emitCopy(dst []byte, offset int, length int) int - -// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= math.MaxUint32 -// 4 <= length && length <= 1 << 24 -// -//go:noescape -func emitCopyNoRepeat(dst []byte, offset int, length int) int - -// matchLen returns how many bytes match in a and b -// -// It assumes that: -// -// len(a) <= len(b) -// -//go:noescape -func matchLen(a []byte, b []byte) int - -// cvtLZ4Block converts an LZ4 block to S2 -// -//go:noescape -func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) - -// cvtLZ4sBlock converts an LZ4s block to S2 -// -//go:noescape -func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) - -// cvtLZ4Block converts an LZ4 block to Snappy -// -//go:noescape -func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) - -// cvtLZ4sBlock converts an LZ4s block to Snappy -// -//go:noescape -func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) diff --git a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s deleted file mode 100644 index df9be687b..000000000 --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s +++ /dev/null @@ -1,21303 +0,0 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. - -//go:build !appengine && !noasm && gc && !noasm - -#include "textflag.h" - -// func _dummy_() -TEXT ·_dummy_(SB), $0 -#ifdef GOAMD64_v4 -#ifndef GOAMD64_v3 -#define GOAMD64_v3 -#endif -#endif - RET - -// func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBlockAsm(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBlockAsm: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBlockAsm - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBlockAsm: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBlockAsm - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ R9, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeBlockAsm - LEAL 1(DX), DI - MOVL 12(SP), R8 - MOVL DI, SI - SUBL 16(SP), SI - JZ repeat_extend_back_end_encodeBlockAsm - -repeat_extend_back_loop_encodeBlockAsm: - CMPL DI, R8 - JBE repeat_extend_back_end_encodeBlockAsm - MOVB -1(BX)(SI*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeBlockAsm - LEAL -1(DI), DI - DECL SI - JNZ repeat_extend_back_loop_encodeBlockAsm - -repeat_extend_back_end_encodeBlockAsm: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 5(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeBlockAsm: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeBlockAsm - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeBlockAsm - CMPL SI, $0x00010000 - JB three_bytes_repeat_emit_encodeBlockAsm - CMPL SI, $0x01000000 - JB four_bytes_repeat_emit_encodeBlockAsm - MOVB $0xfc, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_repeat_emit_encodeBlockAsm - -four_bytes_repeat_emit_encodeBlockAsm: - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_repeat_emit_encodeBlockAsm - -three_bytes_repeat_emit_encodeBlockAsm: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeBlockAsm - -two_bytes_repeat_emit_encodeBlockAsm: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeBlockAsm - JMP memmove_long_repeat_emit_encodeBlockAsm - -one_byte_repeat_emit_encodeBlockAsm: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm - -emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm - -emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm - -emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_repeat_emit_encodeBlockAsm: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeBlockAsm - -memmove_long_repeat_emit_encodeBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R12 - SHRQ $0x05, R12 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(R10)(R13*1), R11 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(R10)(R13*1), X4 - MOVOU -16(R10)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R9, R13 - JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeBlockAsm: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R9 - SUBL DX, R9 - LEAQ (BX)(DX*1), R10 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_repeat_extend_encodeBlockAsm: - CMPL R9, $0x10 - JB matchlen_match8_repeat_extend_encodeBlockAsm - MOVQ (R10)(R12*1), R11 - MOVQ 8(R10)(R12*1), R13 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm - XORQ 8(SI)(R12*1), R13 - JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm - LEAL -16(R9), R9 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm - -matchlen_bsf_16repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm - -matchlen_match8_repeat_extend_encodeBlockAsm: - CMPL R9, $0x08 - JB matchlen_match4_repeat_extend_encodeBlockAsm - MOVQ (R10)(R12*1), R11 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm - LEAL -8(R9), R9 - LEAL 8(R12), R12 - JMP matchlen_match4_repeat_extend_encodeBlockAsm - -matchlen_bsf_8_repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm - -matchlen_match4_repeat_extend_encodeBlockAsm: - CMPL R9, $0x04 - JB matchlen_match2_repeat_extend_encodeBlockAsm - MOVL (R10)(R12*1), R11 - CMPL (SI)(R12*1), R11 - JNE matchlen_match2_repeat_extend_encodeBlockAsm - LEAL -4(R9), R9 - LEAL 4(R12), R12 - -matchlen_match2_repeat_extend_encodeBlockAsm: - CMPL R9, $0x01 - JE matchlen_match1_repeat_extend_encodeBlockAsm - JB repeat_extend_forward_end_encodeBlockAsm - MOVW (R10)(R12*1), R11 - CMPW (SI)(R12*1), R11 - JNE matchlen_match1_repeat_extend_encodeBlockAsm - LEAL 2(R12), R12 - SUBL $0x02, R9 - JZ repeat_extend_forward_end_encodeBlockAsm - -matchlen_match1_repeat_extend_encodeBlockAsm: - MOVB (R10)(R12*1), R11 - CMPB (SI)(R12*1), R11 - JNE repeat_extend_forward_end_encodeBlockAsm - LEAL 1(R12), R12 - -repeat_extend_forward_end_encodeBlockAsm: - ADDL R12, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - TESTL R8, R8 - JZ repeat_as_copy_encodeBlockAsm - - // emitRepeat -emit_repeat_again_match_repeat_encodeBlockAsm: - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_match_repeat_encodeBlockAsm - CMPL R8, $0x0c - JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm - CMPL DI, $0x00000800 - JB repeat_two_offset_match_repeat_encodeBlockAsm - -cant_repeat_two_offset_match_repeat_encodeBlockAsm: - CMPL SI, $0x00000104 - JB repeat_three_match_repeat_encodeBlockAsm - CMPL SI, $0x00010100 - JB repeat_four_match_repeat_encodeBlockAsm - CMPL SI, $0x0100ffff - JB repeat_five_match_repeat_encodeBlockAsm - LEAL -16842747(SI), SI - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_repeat_encodeBlockAsm - -repeat_five_match_repeat_encodeBlockAsm: - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_four_match_repeat_encodeBlockAsm: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_three_match_repeat_encodeBlockAsm: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_match_repeat_encodeBlockAsm: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_offset_match_repeat_encodeBlockAsm: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_as_copy_encodeBlockAsm: - // emitCopy - CMPL DI, $0x00010000 - JB two_byte_offset_repeat_as_copy_encodeBlockAsm - CMPL SI, $0x40 - JBE four_bytes_remain_repeat_as_copy_encodeBlockAsm - MOVB $0xff, (CX) - MOVL DI, 1(CX) - LEAL -64(SI), SI - ADDQ $0x05, CX - CMPL SI, $0x04 - JB four_bytes_remain_repeat_as_copy_encodeBlockAsm - - // emitRepeat -emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy: - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy - CMPL SI, $0x0100ffff - JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy - LEAL -16842747(SI), SI - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy - -repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy: - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -four_bytes_remain_repeat_as_copy_encodeBlockAsm: - TESTL SI, SI - JZ repeat_end_emit_encodeBlockAsm - XORL R8, R8 - LEAL -1(R8)(SI*4), SI - MOVB SI, (CX) - MOVL DI, 1(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm - -two_byte_offset_repeat_as_copy_encodeBlockAsm: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm - CMPL DI, $0x00000800 - JAE long_offset_short_repeat_as_copy_encodeBlockAsm - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB DI, 1(CX) - MOVL DI, R9 - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - SUBL $0x08, SI - - // emitRepeat - LEAL -4(SI), SI - JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - -emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - CMPL SI, $0x0100ffff - JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - LEAL -16842747(SI), SI - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b - -repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -long_offset_short_repeat_as_copy_encodeBlockAsm: - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - - // emitRepeat -emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short: - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short - CMPL SI, $0x0100ffff - JB repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short - LEAL -16842747(SI), SI - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short - -repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short: - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -two_byte_offset_short_repeat_as_copy_encodeBlockAsm: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm - -emit_copy_three_repeat_as_copy_encodeBlockAsm: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeBlockAsm: - MOVL DX, 12(SP) - JMP search_loop_encodeBlockAsm - -no_repeat_found_encodeBlockAsm: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBlockAsm - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeBlockAsm - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeBlockAsm - MOVL 20(SP), DX - JMP search_loop_encodeBlockAsm - -candidate3_match_encodeBlockAsm: - ADDL $0x02, DX - JMP candidate_match_encodeBlockAsm - -candidate2_match_encodeBlockAsm: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeBlockAsm: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBlockAsm - -match_extend_back_loop_encodeBlockAsm: - CMPL DX, DI - JBE match_extend_back_end_encodeBlockAsm - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBlockAsm - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBlockAsm - JMP match_extend_back_loop_encodeBlockAsm - -match_extend_back_end_encodeBlockAsm: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 5(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBlockAsm: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeBlockAsm - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeBlockAsm - CMPL R8, $0x00010000 - JB three_bytes_match_emit_encodeBlockAsm - CMPL R8, $0x01000000 - JB four_bytes_match_emit_encodeBlockAsm - MOVB $0xfc, (CX) - MOVL R8, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_match_emit_encodeBlockAsm - -four_bytes_match_emit_encodeBlockAsm: - MOVL R8, R10 - SHRL $0x10, R10 - MOVB $0xf8, (CX) - MOVW R8, 1(CX) - MOVB R10, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeBlockAsm - -three_bytes_match_emit_encodeBlockAsm: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBlockAsm - -two_bytes_match_emit_encodeBlockAsm: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeBlockAsm - JMP memmove_long_match_emit_encodeBlockAsm - -one_byte_match_emit_encodeBlockAsm: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBlockAsm: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeBlockAsm - -emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm - -emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm - -emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBlockAsm: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeBlockAsm - -memmove_long_match_emit_encodeBlockAsm: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeBlockAsm: -match_nolit_loop_encodeBlockAsm: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeBlockAsm: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeBlockAsm - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeBlockAsm - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeBlockAsm - -matchlen_bsf_16match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm - -matchlen_match8_match_nolit_encodeBlockAsm: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeBlockAsm - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeBlockAsm - -matchlen_bsf_8_match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm - -matchlen_match4_match_nolit_encodeBlockAsm: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeBlockAsm - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeBlockAsm - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeBlockAsm: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeBlockAsm - JB match_nolit_end_encodeBlockAsm - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeBlockAsm - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeBlockAsm - -matchlen_match1_match_nolit_encodeBlockAsm: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeBlockAsm - LEAL 1(R10), R10 - -match_nolit_end_encodeBlockAsm: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL SI, $0x00010000 - JB two_byte_offset_match_nolit_encodeBlockAsm - CMPL R10, $0x40 - JBE four_bytes_remain_match_nolit_encodeBlockAsm - MOVB $0xff, (CX) - MOVL SI, 1(CX) - LEAL -64(R10), R10 - ADDQ $0x05, CX - CMPL R10, $0x04 - JB four_bytes_remain_match_nolit_encodeBlockAsm - - // emitRepeat -emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy: - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy - -cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm_emit_copy - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm_emit_copy - CMPL R10, $0x0100ffff - JB repeat_five_match_nolit_encodeBlockAsm_emit_copy - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy - -repeat_five_match_nolit_encodeBlockAsm_emit_copy: - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_four_match_nolit_encodeBlockAsm_emit_copy: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_three_match_nolit_encodeBlockAsm_emit_copy: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_match_nolit_encodeBlockAsm_emit_copy: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -four_bytes_remain_match_nolit_encodeBlockAsm: - TESTL R10, R10 - JZ match_nolit_emitcopy_end_encodeBlockAsm - XORL DI, DI - LEAL -1(DI)(R10*4), R10 - MOVB R10, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -two_byte_offset_match_nolit_encodeBlockAsm: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBlockAsm - CMPL SI, $0x00000800 - JAE long_offset_short_match_nolit_encodeBlockAsm - MOVL $0x00000001, DI - LEAL 16(DI), DI - MOVB SI, 1(CX) - MOVL SI, R8 - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, DI - MOVB DI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b - -emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b: - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b - CMPL R10, $0x0100ffff - JB repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b - -repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -long_offset_short_match_nolit_encodeBlockAsm: - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - - // emitRepeat -emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short: - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm_emit_copy_short - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm_emit_copy_short - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm_emit_copy_short - CMPL R10, $0x0100ffff - JB repeat_five_match_nolit_encodeBlockAsm_emit_copy_short - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short - -repeat_five_match_nolit_encodeBlockAsm_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_four_match_nolit_encodeBlockAsm_emit_copy_short: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_three_match_nolit_encodeBlockAsm_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_match_nolit_encodeBlockAsm_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -two_byte_offset_short_match_nolit_encodeBlockAsm: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeBlockAsm - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBlockAsm - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm - -emit_copy_three_match_nolit_encodeBlockAsm: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeBlockAsm: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBlockAsm - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBlockAsm: - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x10, R8 - IMULQ R9, R8 - SHRQ $0x32, R8 - SHLQ $0x10, SI - IMULQ R9, SI - SHRQ $0x32, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeBlockAsm - INCL DX - JMP search_loop_encodeBlockAsm - -emit_remainder_encodeBlockAsm: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 5(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBlockAsm: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBlockAsm - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBlockAsm - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBlockAsm - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeBlockAsm - CMPL DX, $0x01000000 - JB four_bytes_emit_remainder_encodeBlockAsm - MOVB $0xfc, (CX) - MOVL DX, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_emit_remainder_encodeBlockAsm - -four_bytes_emit_remainder_encodeBlockAsm: - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeBlockAsm - -three_bytes_emit_remainder_encodeBlockAsm: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBlockAsm - -two_bytes_emit_remainder_encodeBlockAsm: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBlockAsm - JMP memmove_long_emit_remainder_encodeBlockAsm - -one_byte_emit_remainder_encodeBlockAsm: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm - -emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBlockAsm: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBlockAsm - -memmove_long_emit_remainder_encodeBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBlockAsm: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBlockAsm4MB(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBlockAsm4MB: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBlockAsm4MB - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBlockAsm4MB: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBlockAsm4MB - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ R9, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeBlockAsm4MB - LEAL 1(DX), DI - MOVL 12(SP), R8 - MOVL DI, SI - SUBL 16(SP), SI - JZ repeat_extend_back_end_encodeBlockAsm4MB - -repeat_extend_back_loop_encodeBlockAsm4MB: - CMPL DI, R8 - JBE repeat_extend_back_end_encodeBlockAsm4MB - MOVB -1(BX)(SI*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeBlockAsm4MB - LEAL -1(DI), DI - DECL SI - JNZ repeat_extend_back_loop_encodeBlockAsm4MB - -repeat_extend_back_end_encodeBlockAsm4MB: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 4(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeBlockAsm4MB: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeBlockAsm4MB - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeBlockAsm4MB - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeBlockAsm4MB - CMPL SI, $0x00010000 - JB three_bytes_repeat_emit_encodeBlockAsm4MB - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_repeat_emit_encodeBlockAsm4MB - -three_bytes_repeat_emit_encodeBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeBlockAsm4MB - -two_bytes_repeat_emit_encodeBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeBlockAsm4MB - JMP memmove_long_repeat_emit_encodeBlockAsm4MB - -one_byte_repeat_emit_encodeBlockAsm4MB: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB - -emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB - -emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB - -emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_repeat_emit_encodeBlockAsm4MB: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeBlockAsm4MB - -memmove_long_repeat_emit_encodeBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R12 - SHRQ $0x05, R12 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(R10)(R13*1), R11 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(R10)(R13*1), X4 - MOVOU -16(R10)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R9, R13 - JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeBlockAsm4MB: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R9 - SUBL DX, R9 - LEAQ (BX)(DX*1), R10 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB: - CMPL R9, $0x10 - JB matchlen_match8_repeat_extend_encodeBlockAsm4MB - MOVQ (R10)(R12*1), R11 - MOVQ 8(R10)(R12*1), R13 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB - XORQ 8(SI)(R12*1), R13 - JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm4MB - LEAL -16(R9), R9 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB - -matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB - -matchlen_match8_repeat_extend_encodeBlockAsm4MB: - CMPL R9, $0x08 - JB matchlen_match4_repeat_extend_encodeBlockAsm4MB - MOVQ (R10)(R12*1), R11 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB - LEAL -8(R9), R9 - LEAL 8(R12), R12 - JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB - -matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB - -matchlen_match4_repeat_extend_encodeBlockAsm4MB: - CMPL R9, $0x04 - JB matchlen_match2_repeat_extend_encodeBlockAsm4MB - MOVL (R10)(R12*1), R11 - CMPL (SI)(R12*1), R11 - JNE matchlen_match2_repeat_extend_encodeBlockAsm4MB - LEAL -4(R9), R9 - LEAL 4(R12), R12 - -matchlen_match2_repeat_extend_encodeBlockAsm4MB: - CMPL R9, $0x01 - JE matchlen_match1_repeat_extend_encodeBlockAsm4MB - JB repeat_extend_forward_end_encodeBlockAsm4MB - MOVW (R10)(R12*1), R11 - CMPW (SI)(R12*1), R11 - JNE matchlen_match1_repeat_extend_encodeBlockAsm4MB - LEAL 2(R12), R12 - SUBL $0x02, R9 - JZ repeat_extend_forward_end_encodeBlockAsm4MB - -matchlen_match1_repeat_extend_encodeBlockAsm4MB: - MOVB (R10)(R12*1), R11 - CMPB (SI)(R12*1), R11 - JNE repeat_extend_forward_end_encodeBlockAsm4MB - LEAL 1(R12), R12 - -repeat_extend_forward_end_encodeBlockAsm4MB: - ADDL R12, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - TESTL R8, R8 - JZ repeat_as_copy_encodeBlockAsm4MB - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_match_repeat_encodeBlockAsm4MB - CMPL R8, $0x0c - JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB - CMPL DI, $0x00000800 - JB repeat_two_offset_match_repeat_encodeBlockAsm4MB - -cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB: - CMPL SI, $0x00000104 - JB repeat_three_match_repeat_encodeBlockAsm4MB - CMPL SI, $0x00010100 - JB repeat_four_match_repeat_encodeBlockAsm4MB - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_four_match_repeat_encodeBlockAsm4MB: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_three_match_repeat_encodeBlockAsm4MB: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_match_repeat_encodeBlockAsm4MB: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_offset_match_repeat_encodeBlockAsm4MB: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_as_copy_encodeBlockAsm4MB: - // emitCopy - CMPL DI, $0x00010000 - JB two_byte_offset_repeat_as_copy_encodeBlockAsm4MB - CMPL SI, $0x40 - JBE four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB - MOVB $0xff, (CX) - MOVL DI, 1(CX) - LEAL -64(SI), SI - ADDQ $0x05, CX - CMPL SI, $0x04 - JB four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB: - TESTL SI, SI - JZ repeat_end_emit_encodeBlockAsm4MB - XORL R8, R8 - LEAL -1(R8)(SI*4), SI - MOVB SI, (CX) - MOVL DI, 1(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -two_byte_offset_repeat_as_copy_encodeBlockAsm4MB: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB - CMPL DI, $0x00000800 - JAE long_offset_short_repeat_as_copy_encodeBlockAsm4MB - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - SUBL $0x08, SI - - // emitRepeat - LEAL -4(SI), SI - JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -long_offset_short_repeat_as_copy_encodeBlockAsm4MB: - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short - CMPL SI, $0x00010100 - JB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short - LEAL -65536(SI), SI - MOVL SI, DI - MOVW $0x001d, (CX) - MOVW SI, 2(CX) - SARL $0x10, DI - MOVB DI, 4(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm4MB - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm4MB - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm4MB - -emit_copy_three_repeat_as_copy_encodeBlockAsm4MB: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeBlockAsm4MB: - MOVL DX, 12(SP) - JMP search_loop_encodeBlockAsm4MB - -no_repeat_found_encodeBlockAsm4MB: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBlockAsm4MB - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeBlockAsm4MB - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeBlockAsm4MB - MOVL 20(SP), DX - JMP search_loop_encodeBlockAsm4MB - -candidate3_match_encodeBlockAsm4MB: - ADDL $0x02, DX - JMP candidate_match_encodeBlockAsm4MB - -candidate2_match_encodeBlockAsm4MB: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeBlockAsm4MB: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBlockAsm4MB - -match_extend_back_loop_encodeBlockAsm4MB: - CMPL DX, DI - JBE match_extend_back_end_encodeBlockAsm4MB - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBlockAsm4MB - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBlockAsm4MB - JMP match_extend_back_loop_encodeBlockAsm4MB - -match_extend_back_end_encodeBlockAsm4MB: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 4(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBlockAsm4MB: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeBlockAsm4MB - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeBlockAsm4MB - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeBlockAsm4MB - CMPL R8, $0x00010000 - JB three_bytes_match_emit_encodeBlockAsm4MB - MOVL R8, R10 - SHRL $0x10, R10 - MOVB $0xf8, (CX) - MOVW R8, 1(CX) - MOVB R10, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeBlockAsm4MB - -three_bytes_match_emit_encodeBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBlockAsm4MB - -two_bytes_match_emit_encodeBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeBlockAsm4MB - JMP memmove_long_match_emit_encodeBlockAsm4MB - -one_byte_match_emit_encodeBlockAsm4MB: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBlockAsm4MB: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBlockAsm4MB: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeBlockAsm4MB - -memmove_long_match_emit_encodeBlockAsm4MB: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeBlockAsm4MB: -match_nolit_loop_encodeBlockAsm4MB: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeBlockAsm4MB: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeBlockAsm4MB - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm4MB - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeBlockAsm4MB - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB - -matchlen_bsf_16match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB - -matchlen_match8_match_nolit_encodeBlockAsm4MB: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeBlockAsm4MB - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm4MB - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeBlockAsm4MB - -matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB - -matchlen_match4_match_nolit_encodeBlockAsm4MB: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeBlockAsm4MB - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeBlockAsm4MB - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeBlockAsm4MB: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeBlockAsm4MB - JB match_nolit_end_encodeBlockAsm4MB - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeBlockAsm4MB - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeBlockAsm4MB - -matchlen_match1_match_nolit_encodeBlockAsm4MB: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeBlockAsm4MB - LEAL 1(R10), R10 - -match_nolit_end_encodeBlockAsm4MB: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL SI, $0x00010000 - JB two_byte_offset_match_nolit_encodeBlockAsm4MB - CMPL R10, $0x40 - JBE four_bytes_remain_match_nolit_encodeBlockAsm4MB - MOVB $0xff, (CX) - MOVL SI, 1(CX) - LEAL -64(R10), R10 - ADDQ $0x05, CX - CMPL R10, $0x04 - JB four_bytes_remain_match_nolit_encodeBlockAsm4MB - - // emitRepeat - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy - -cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -four_bytes_remain_match_nolit_encodeBlockAsm4MB: - TESTL R10, R10 - JZ match_nolit_emitcopy_end_encodeBlockAsm4MB - XORL DI, DI - LEAL -1(DI)(R10*4), R10 - MOVB R10, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -two_byte_offset_match_nolit_encodeBlockAsm4MB: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBlockAsm4MB - CMPL SI, $0x00000800 - JAE long_offset_short_match_nolit_encodeBlockAsm4MB - MOVL $0x00000001, DI - LEAL 16(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -long_offset_short_match_nolit_encodeBlockAsm4MB: - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - - // emitRepeat - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short - CMPL R10, $0x00010100 - JB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short - LEAL -65536(R10), R10 - MOVL R10, SI - MOVW $0x001d, (CX) - MOVW R10, 2(CX) - SARL $0x10, SI - MOVB SI, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short: - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -two_byte_offset_short_match_nolit_encodeBlockAsm4MB: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeBlockAsm4MB - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBlockAsm4MB - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm4MB - -emit_copy_three_match_nolit_encodeBlockAsm4MB: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeBlockAsm4MB: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBlockAsm4MB - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBlockAsm4MB: - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x10, R8 - IMULQ R9, R8 - SHRQ $0x32, R8 - SHLQ $0x10, SI - IMULQ R9, SI - SHRQ $0x32, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeBlockAsm4MB - INCL DX - JMP search_loop_encodeBlockAsm4MB - -emit_remainder_encodeBlockAsm4MB: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 4(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBlockAsm4MB: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBlockAsm4MB - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBlockAsm4MB - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBlockAsm4MB - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeBlockAsm4MB - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeBlockAsm4MB - -three_bytes_emit_remainder_encodeBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBlockAsm4MB - -two_bytes_emit_remainder_encodeBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBlockAsm4MB - JMP memmove_long_emit_remainder_encodeBlockAsm4MB - -one_byte_emit_remainder_encodeBlockAsm4MB: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBlockAsm4MB: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBlockAsm4MB: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBlockAsm4MB - -memmove_long_emit_remainder_encodeBlockAsm4MB: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBlockAsm4MB: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBlockAsm12B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000080, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBlockAsm12B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBlockAsm12B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBlockAsm12B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBlockAsm12B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x000000cf1bbcdcbb, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x18, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - SHLQ $0x18, R11 - IMULQ R9, R11 - SHRQ $0x34, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x18, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeBlockAsm12B - LEAL 1(DX), DI - MOVL 12(SP), R8 - MOVL DI, SI - SUBL 16(SP), SI - JZ repeat_extend_back_end_encodeBlockAsm12B - -repeat_extend_back_loop_encodeBlockAsm12B: - CMPL DI, R8 - JBE repeat_extend_back_end_encodeBlockAsm12B - MOVB -1(BX)(SI*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeBlockAsm12B - LEAL -1(DI), DI - DECL SI - JNZ repeat_extend_back_loop_encodeBlockAsm12B - -repeat_extend_back_end_encodeBlockAsm12B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeBlockAsm12B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeBlockAsm12B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeBlockAsm12B - JB three_bytes_repeat_emit_encodeBlockAsm12B - -three_bytes_repeat_emit_encodeBlockAsm12B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeBlockAsm12B - -two_bytes_repeat_emit_encodeBlockAsm12B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeBlockAsm12B - JMP memmove_long_repeat_emit_encodeBlockAsm12B - -one_byte_repeat_emit_encodeBlockAsm12B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_repeat_emit_encodeBlockAsm12B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeBlockAsm12B - -memmove_long_repeat_emit_encodeBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R12 - SHRQ $0x05, R12 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R13*1), R11 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R13*1), X4 - MOVOU -16(R10)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R9, R13 - JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeBlockAsm12B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R9 - SUBL DX, R9 - LEAQ (BX)(DX*1), R10 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_repeat_extend_encodeBlockAsm12B: - CMPL R9, $0x10 - JB matchlen_match8_repeat_extend_encodeBlockAsm12B - MOVQ (R10)(R12*1), R11 - MOVQ 8(R10)(R12*1), R13 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm12B - XORQ 8(SI)(R12*1), R13 - JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm12B - LEAL -16(R9), R9 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B - -matchlen_bsf_16repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B - -matchlen_match8_repeat_extend_encodeBlockAsm12B: - CMPL R9, $0x08 - JB matchlen_match4_repeat_extend_encodeBlockAsm12B - MOVQ (R10)(R12*1), R11 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm12B - LEAL -8(R9), R9 - LEAL 8(R12), R12 - JMP matchlen_match4_repeat_extend_encodeBlockAsm12B - -matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B - -matchlen_match4_repeat_extend_encodeBlockAsm12B: - CMPL R9, $0x04 - JB matchlen_match2_repeat_extend_encodeBlockAsm12B - MOVL (R10)(R12*1), R11 - CMPL (SI)(R12*1), R11 - JNE matchlen_match2_repeat_extend_encodeBlockAsm12B - LEAL -4(R9), R9 - LEAL 4(R12), R12 - -matchlen_match2_repeat_extend_encodeBlockAsm12B: - CMPL R9, $0x01 - JE matchlen_match1_repeat_extend_encodeBlockAsm12B - JB repeat_extend_forward_end_encodeBlockAsm12B - MOVW (R10)(R12*1), R11 - CMPW (SI)(R12*1), R11 - JNE matchlen_match1_repeat_extend_encodeBlockAsm12B - LEAL 2(R12), R12 - SUBL $0x02, R9 - JZ repeat_extend_forward_end_encodeBlockAsm12B - -matchlen_match1_repeat_extend_encodeBlockAsm12B: - MOVB (R10)(R12*1), R11 - CMPB (SI)(R12*1), R11 - JNE repeat_extend_forward_end_encodeBlockAsm12B - LEAL 1(R12), R12 - -repeat_extend_forward_end_encodeBlockAsm12B: - ADDL R12, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - TESTL R8, R8 - JZ repeat_as_copy_encodeBlockAsm12B - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_match_repeat_encodeBlockAsm12B - CMPL R8, $0x0c - JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm12B - CMPL DI, $0x00000800 - JB repeat_two_offset_match_repeat_encodeBlockAsm12B - -cant_repeat_two_offset_match_repeat_encodeBlockAsm12B: - CMPL SI, $0x00000104 - JB repeat_three_match_repeat_encodeBlockAsm12B - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_three_match_repeat_encodeBlockAsm12B: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_match_repeat_encodeBlockAsm12B: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_offset_match_repeat_encodeBlockAsm12B: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_as_copy_encodeBlockAsm12B: - // emitCopy - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B - CMPL DI, $0x00000800 - JAE long_offset_short_repeat_as_copy_encodeBlockAsm12B - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - SUBL $0x08, SI - - // emitRepeat - LEAL -4(SI), SI - JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -long_offset_short_repeat_as_copy_encodeBlockAsm12B: - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm12B - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm12B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm12B - -emit_copy_three_repeat_as_copy_encodeBlockAsm12B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeBlockAsm12B: - MOVL DX, 12(SP) - JMP search_loop_encodeBlockAsm12B - -no_repeat_found_encodeBlockAsm12B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBlockAsm12B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeBlockAsm12B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeBlockAsm12B - MOVL 20(SP), DX - JMP search_loop_encodeBlockAsm12B - -candidate3_match_encodeBlockAsm12B: - ADDL $0x02, DX - JMP candidate_match_encodeBlockAsm12B - -candidate2_match_encodeBlockAsm12B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeBlockAsm12B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBlockAsm12B - -match_extend_back_loop_encodeBlockAsm12B: - CMPL DX, DI - JBE match_extend_back_end_encodeBlockAsm12B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBlockAsm12B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBlockAsm12B - JMP match_extend_back_loop_encodeBlockAsm12B - -match_extend_back_end_encodeBlockAsm12B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBlockAsm12B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeBlockAsm12B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeBlockAsm12B - JB three_bytes_match_emit_encodeBlockAsm12B - -three_bytes_match_emit_encodeBlockAsm12B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBlockAsm12B - -two_bytes_match_emit_encodeBlockAsm12B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeBlockAsm12B - JMP memmove_long_match_emit_encodeBlockAsm12B - -one_byte_match_emit_encodeBlockAsm12B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBlockAsm12B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeBlockAsm12B - -emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm12B - -emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm12B - -emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBlockAsm12B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeBlockAsm12B - -memmove_long_match_emit_encodeBlockAsm12B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeBlockAsm12B: -match_nolit_loop_encodeBlockAsm12B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeBlockAsm12B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeBlockAsm12B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm12B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeBlockAsm12B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B - -matchlen_bsf_16match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm12B - -matchlen_match8_match_nolit_encodeBlockAsm12B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeBlockAsm12B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm12B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeBlockAsm12B - -matchlen_bsf_8_match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm12B - -matchlen_match4_match_nolit_encodeBlockAsm12B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeBlockAsm12B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeBlockAsm12B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeBlockAsm12B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeBlockAsm12B - JB match_nolit_end_encodeBlockAsm12B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeBlockAsm12B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeBlockAsm12B - -matchlen_match1_match_nolit_encodeBlockAsm12B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeBlockAsm12B - LEAL 1(R10), R10 - -match_nolit_end_encodeBlockAsm12B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBlockAsm12B - CMPL SI, $0x00000800 - JAE long_offset_short_match_nolit_encodeBlockAsm12B - MOVL $0x00000001, DI - LEAL 16(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -long_offset_short_match_nolit_encodeBlockAsm12B: - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - - // emitRepeat - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -two_byte_offset_short_match_nolit_encodeBlockAsm12B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeBlockAsm12B - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBlockAsm12B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm12B - -emit_copy_three_match_nolit_encodeBlockAsm12B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeBlockAsm12B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBlockAsm12B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBlockAsm12B: - MOVQ $0x000000cf1bbcdcbb, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x18, R8 - IMULQ R9, R8 - SHRQ $0x34, R8 - SHLQ $0x18, SI - IMULQ R9, SI - SHRQ $0x34, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeBlockAsm12B - INCL DX - JMP search_loop_encodeBlockAsm12B - -emit_remainder_encodeBlockAsm12B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBlockAsm12B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBlockAsm12B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBlockAsm12B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBlockAsm12B - JB three_bytes_emit_remainder_encodeBlockAsm12B - -three_bytes_emit_remainder_encodeBlockAsm12B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBlockAsm12B - -two_bytes_emit_remainder_encodeBlockAsm12B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBlockAsm12B - JMP memmove_long_emit_remainder_encodeBlockAsm12B - -one_byte_emit_remainder_encodeBlockAsm12B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBlockAsm12B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBlockAsm12B - -memmove_long_emit_remainder_encodeBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBlockAsm12B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBlockAsm10B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000020, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBlockAsm10B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBlockAsm10B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBlockAsm10B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBlockAsm10B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x9e3779b1, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ R9, R11 - SHRQ $0x36, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeBlockAsm10B - LEAL 1(DX), DI - MOVL 12(SP), R8 - MOVL DI, SI - SUBL 16(SP), SI - JZ repeat_extend_back_end_encodeBlockAsm10B - -repeat_extend_back_loop_encodeBlockAsm10B: - CMPL DI, R8 - JBE repeat_extend_back_end_encodeBlockAsm10B - MOVB -1(BX)(SI*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeBlockAsm10B - LEAL -1(DI), DI - DECL SI - JNZ repeat_extend_back_loop_encodeBlockAsm10B - -repeat_extend_back_end_encodeBlockAsm10B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeBlockAsm10B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeBlockAsm10B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeBlockAsm10B - JB three_bytes_repeat_emit_encodeBlockAsm10B - -three_bytes_repeat_emit_encodeBlockAsm10B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeBlockAsm10B - -two_bytes_repeat_emit_encodeBlockAsm10B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeBlockAsm10B - JMP memmove_long_repeat_emit_encodeBlockAsm10B - -one_byte_repeat_emit_encodeBlockAsm10B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_repeat_emit_encodeBlockAsm10B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeBlockAsm10B - -memmove_long_repeat_emit_encodeBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R12 - SHRQ $0x05, R12 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R13*1), R11 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R13*1), X4 - MOVOU -16(R10)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R9, R13 - JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeBlockAsm10B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R9 - SUBL DX, R9 - LEAQ (BX)(DX*1), R10 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_repeat_extend_encodeBlockAsm10B: - CMPL R9, $0x10 - JB matchlen_match8_repeat_extend_encodeBlockAsm10B - MOVQ (R10)(R12*1), R11 - MOVQ 8(R10)(R12*1), R13 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm10B - XORQ 8(SI)(R12*1), R13 - JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm10B - LEAL -16(R9), R9 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B - -matchlen_bsf_16repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B - -matchlen_match8_repeat_extend_encodeBlockAsm10B: - CMPL R9, $0x08 - JB matchlen_match4_repeat_extend_encodeBlockAsm10B - MOVQ (R10)(R12*1), R11 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm10B - LEAL -8(R9), R9 - LEAL 8(R12), R12 - JMP matchlen_match4_repeat_extend_encodeBlockAsm10B - -matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B - -matchlen_match4_repeat_extend_encodeBlockAsm10B: - CMPL R9, $0x04 - JB matchlen_match2_repeat_extend_encodeBlockAsm10B - MOVL (R10)(R12*1), R11 - CMPL (SI)(R12*1), R11 - JNE matchlen_match2_repeat_extend_encodeBlockAsm10B - LEAL -4(R9), R9 - LEAL 4(R12), R12 - -matchlen_match2_repeat_extend_encodeBlockAsm10B: - CMPL R9, $0x01 - JE matchlen_match1_repeat_extend_encodeBlockAsm10B - JB repeat_extend_forward_end_encodeBlockAsm10B - MOVW (R10)(R12*1), R11 - CMPW (SI)(R12*1), R11 - JNE matchlen_match1_repeat_extend_encodeBlockAsm10B - LEAL 2(R12), R12 - SUBL $0x02, R9 - JZ repeat_extend_forward_end_encodeBlockAsm10B - -matchlen_match1_repeat_extend_encodeBlockAsm10B: - MOVB (R10)(R12*1), R11 - CMPB (SI)(R12*1), R11 - JNE repeat_extend_forward_end_encodeBlockAsm10B - LEAL 1(R12), R12 - -repeat_extend_forward_end_encodeBlockAsm10B: - ADDL R12, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - TESTL R8, R8 - JZ repeat_as_copy_encodeBlockAsm10B - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_match_repeat_encodeBlockAsm10B - CMPL R8, $0x0c - JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm10B - CMPL DI, $0x00000800 - JB repeat_two_offset_match_repeat_encodeBlockAsm10B - -cant_repeat_two_offset_match_repeat_encodeBlockAsm10B: - CMPL SI, $0x00000104 - JB repeat_three_match_repeat_encodeBlockAsm10B - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_three_match_repeat_encodeBlockAsm10B: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_match_repeat_encodeBlockAsm10B: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_offset_match_repeat_encodeBlockAsm10B: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_as_copy_encodeBlockAsm10B: - // emitCopy - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B - CMPL DI, $0x00000800 - JAE long_offset_short_repeat_as_copy_encodeBlockAsm10B - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - SUBL $0x08, SI - - // emitRepeat - LEAL -4(SI), SI - JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -long_offset_short_repeat_as_copy_encodeBlockAsm10B: - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - - // emitRepeat - MOVL SI, R8 - LEAL -4(SI), SI - CMPL R8, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short - CMPL DI, $0x00000800 - JB repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm10B - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm10B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm10B - -emit_copy_three_repeat_as_copy_encodeBlockAsm10B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeBlockAsm10B: - MOVL DX, 12(SP) - JMP search_loop_encodeBlockAsm10B - -no_repeat_found_encodeBlockAsm10B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBlockAsm10B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeBlockAsm10B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeBlockAsm10B - MOVL 20(SP), DX - JMP search_loop_encodeBlockAsm10B - -candidate3_match_encodeBlockAsm10B: - ADDL $0x02, DX - JMP candidate_match_encodeBlockAsm10B - -candidate2_match_encodeBlockAsm10B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeBlockAsm10B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBlockAsm10B - -match_extend_back_loop_encodeBlockAsm10B: - CMPL DX, DI - JBE match_extend_back_end_encodeBlockAsm10B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBlockAsm10B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBlockAsm10B - JMP match_extend_back_loop_encodeBlockAsm10B - -match_extend_back_end_encodeBlockAsm10B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBlockAsm10B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeBlockAsm10B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeBlockAsm10B - JB three_bytes_match_emit_encodeBlockAsm10B - -three_bytes_match_emit_encodeBlockAsm10B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBlockAsm10B - -two_bytes_match_emit_encodeBlockAsm10B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeBlockAsm10B - JMP memmove_long_match_emit_encodeBlockAsm10B - -one_byte_match_emit_encodeBlockAsm10B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBlockAsm10B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeBlockAsm10B - -emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm10B - -emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm10B - -emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBlockAsm10B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeBlockAsm10B - -memmove_long_match_emit_encodeBlockAsm10B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeBlockAsm10B: -match_nolit_loop_encodeBlockAsm10B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeBlockAsm10B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeBlockAsm10B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm10B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeBlockAsm10B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B - -matchlen_bsf_16match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm10B - -matchlen_match8_match_nolit_encodeBlockAsm10B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeBlockAsm10B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm10B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeBlockAsm10B - -matchlen_bsf_8_match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm10B - -matchlen_match4_match_nolit_encodeBlockAsm10B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeBlockAsm10B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeBlockAsm10B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeBlockAsm10B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeBlockAsm10B - JB match_nolit_end_encodeBlockAsm10B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeBlockAsm10B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeBlockAsm10B - -matchlen_match1_match_nolit_encodeBlockAsm10B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeBlockAsm10B - LEAL 1(R10), R10 - -match_nolit_end_encodeBlockAsm10B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBlockAsm10B - CMPL SI, $0x00000800 - JAE long_offset_short_match_nolit_encodeBlockAsm10B - MOVL $0x00000001, DI - LEAL 16(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -long_offset_short_match_nolit_encodeBlockAsm10B: - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - - // emitRepeat - MOVL R10, DI - LEAL -4(R10), R10 - CMPL DI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short - CMPL SI, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -two_byte_offset_short_match_nolit_encodeBlockAsm10B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeBlockAsm10B - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBlockAsm10B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm10B - -emit_copy_three_match_nolit_encodeBlockAsm10B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeBlockAsm10B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBlockAsm10B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBlockAsm10B: - MOVQ $0x9e3779b1, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x20, R8 - IMULQ R9, R8 - SHRQ $0x36, R8 - SHLQ $0x20, SI - IMULQ R9, SI - SHRQ $0x36, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeBlockAsm10B - INCL DX - JMP search_loop_encodeBlockAsm10B - -emit_remainder_encodeBlockAsm10B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBlockAsm10B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBlockAsm10B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBlockAsm10B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBlockAsm10B - JB three_bytes_emit_remainder_encodeBlockAsm10B - -three_bytes_emit_remainder_encodeBlockAsm10B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBlockAsm10B - -two_bytes_emit_remainder_encodeBlockAsm10B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBlockAsm10B - JMP memmove_long_emit_remainder_encodeBlockAsm10B - -one_byte_emit_remainder_encodeBlockAsm10B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBlockAsm10B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBlockAsm10B - -memmove_long_emit_remainder_encodeBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBlockAsm10B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBlockAsm8B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000008, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBlockAsm8B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBlockAsm8B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBlockAsm8B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x04, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBlockAsm8B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x9e3779b1, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x38, R10 - SHLQ $0x20, R11 - IMULQ R9, R11 - SHRQ $0x38, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x38, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeBlockAsm8B - LEAL 1(DX), DI - MOVL 12(SP), R8 - MOVL DI, SI - SUBL 16(SP), SI - JZ repeat_extend_back_end_encodeBlockAsm8B - -repeat_extend_back_loop_encodeBlockAsm8B: - CMPL DI, R8 - JBE repeat_extend_back_end_encodeBlockAsm8B - MOVB -1(BX)(SI*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeBlockAsm8B - LEAL -1(DI), DI - DECL SI - JNZ repeat_extend_back_loop_encodeBlockAsm8B - -repeat_extend_back_end_encodeBlockAsm8B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeBlockAsm8B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeBlockAsm8B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeBlockAsm8B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeBlockAsm8B - JB three_bytes_repeat_emit_encodeBlockAsm8B - -three_bytes_repeat_emit_encodeBlockAsm8B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeBlockAsm8B - -two_bytes_repeat_emit_encodeBlockAsm8B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeBlockAsm8B - JMP memmove_long_repeat_emit_encodeBlockAsm8B - -one_byte_repeat_emit_encodeBlockAsm8B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_repeat_emit_encodeBlockAsm8B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeBlockAsm8B - -memmove_long_repeat_emit_encodeBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R12 - SHRQ $0x05, R12 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R13*1), R11 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R13*1), X4 - MOVOU -16(R10)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R9, R13 - JAE emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeBlockAsm8B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R9 - SUBL DX, R9 - LEAQ (BX)(DX*1), R10 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_repeat_extend_encodeBlockAsm8B: - CMPL R9, $0x10 - JB matchlen_match8_repeat_extend_encodeBlockAsm8B - MOVQ (R10)(R12*1), R11 - MOVQ 8(R10)(R12*1), R13 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm8B - XORQ 8(SI)(R12*1), R13 - JNZ matchlen_bsf_16repeat_extend_encodeBlockAsm8B - LEAL -16(R9), R9 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B - -matchlen_bsf_16repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B - -matchlen_match8_repeat_extend_encodeBlockAsm8B: - CMPL R9, $0x08 - JB matchlen_match4_repeat_extend_encodeBlockAsm8B - MOVQ (R10)(R12*1), R11 - XORQ (SI)(R12*1), R11 - JNZ matchlen_bsf_8_repeat_extend_encodeBlockAsm8B - LEAL -8(R9), R9 - LEAL 8(R12), R12 - JMP matchlen_match4_repeat_extend_encodeBlockAsm8B - -matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B - -matchlen_match4_repeat_extend_encodeBlockAsm8B: - CMPL R9, $0x04 - JB matchlen_match2_repeat_extend_encodeBlockAsm8B - MOVL (R10)(R12*1), R11 - CMPL (SI)(R12*1), R11 - JNE matchlen_match2_repeat_extend_encodeBlockAsm8B - LEAL -4(R9), R9 - LEAL 4(R12), R12 - -matchlen_match2_repeat_extend_encodeBlockAsm8B: - CMPL R9, $0x01 - JE matchlen_match1_repeat_extend_encodeBlockAsm8B - JB repeat_extend_forward_end_encodeBlockAsm8B - MOVW (R10)(R12*1), R11 - CMPW (SI)(R12*1), R11 - JNE matchlen_match1_repeat_extend_encodeBlockAsm8B - LEAL 2(R12), R12 - SUBL $0x02, R9 - JZ repeat_extend_forward_end_encodeBlockAsm8B - -matchlen_match1_repeat_extend_encodeBlockAsm8B: - MOVB (R10)(R12*1), R11 - CMPB (SI)(R12*1), R11 - JNE repeat_extend_forward_end_encodeBlockAsm8B - LEAL 1(R12), R12 - -repeat_extend_forward_end_encodeBlockAsm8B: - ADDL R12, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - TESTL R8, R8 - JZ repeat_as_copy_encodeBlockAsm8B - - // emitRepeat - MOVL SI, DI - LEAL -4(SI), SI - CMPL DI, $0x08 - JBE repeat_two_match_repeat_encodeBlockAsm8B - CMPL DI, $0x0c - JAE cant_repeat_two_offset_match_repeat_encodeBlockAsm8B - -cant_repeat_two_offset_match_repeat_encodeBlockAsm8B: - CMPL SI, $0x00000104 - JB repeat_three_match_repeat_encodeBlockAsm8B - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_three_match_repeat_encodeBlockAsm8B: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_two_match_repeat_encodeBlockAsm8B: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_as_copy_encodeBlockAsm8B: - // emitCopy - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B - CMPL DI, $0x00000800 - JAE long_offset_short_repeat_as_copy_encodeBlockAsm8B - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - SUBL $0x08, SI - - // emitRepeat - LEAL -4(SI), SI - JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b - MOVL SI, DI - LEAL -4(SI), SI - CMPL DI, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b - CMPL DI, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - -long_offset_short_repeat_as_copy_encodeBlockAsm8B: - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - - // emitRepeat - MOVL SI, DI - LEAL -4(SI), SI - CMPL DI, $0x08 - JBE repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short - CMPL DI, $0x0c - JAE cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short - -cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: - CMPL SI, $0x00000104 - JB repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short - LEAL -256(SI), SI - MOVW $0x0019, (CX) - MOVW SI, 2(CX) - ADDQ $0x04, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: - LEAL -4(SI), SI - MOVW $0x0015, (CX) - MOVB SI, 2(CX) - ADDQ $0x03, CX - JMP repeat_end_emit_encodeBlockAsm8B - -repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: - SHLL $0x02, SI - ORL $0x01, SI - MOVW SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - XORQ R8, R8 - LEAL 1(R8)(SI*4), SI - MOVB DI, 1(CX) - SARL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - -two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeBlockAsm8B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeBlockAsm8B - -emit_copy_three_repeat_as_copy_encodeBlockAsm8B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeBlockAsm8B: - MOVL DX, 12(SP) - JMP search_loop_encodeBlockAsm8B - -no_repeat_found_encodeBlockAsm8B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBlockAsm8B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeBlockAsm8B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeBlockAsm8B - MOVL 20(SP), DX - JMP search_loop_encodeBlockAsm8B - -candidate3_match_encodeBlockAsm8B: - ADDL $0x02, DX - JMP candidate_match_encodeBlockAsm8B - -candidate2_match_encodeBlockAsm8B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeBlockAsm8B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBlockAsm8B - -match_extend_back_loop_encodeBlockAsm8B: - CMPL DX, DI - JBE match_extend_back_end_encodeBlockAsm8B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBlockAsm8B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBlockAsm8B - JMP match_extend_back_loop_encodeBlockAsm8B - -match_extend_back_end_encodeBlockAsm8B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBlockAsm8B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeBlockAsm8B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeBlockAsm8B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeBlockAsm8B - JB three_bytes_match_emit_encodeBlockAsm8B - -three_bytes_match_emit_encodeBlockAsm8B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBlockAsm8B - -two_bytes_match_emit_encodeBlockAsm8B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeBlockAsm8B - JMP memmove_long_match_emit_encodeBlockAsm8B - -one_byte_match_emit_encodeBlockAsm8B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBlockAsm8B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeBlockAsm8B - -emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm8B - -emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBlockAsm8B - -emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBlockAsm8B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeBlockAsm8B - -memmove_long_match_emit_encodeBlockAsm8B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeBlockAsm8B: -match_nolit_loop_encodeBlockAsm8B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeBlockAsm8B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeBlockAsm8B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm8B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeBlockAsm8B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B - -matchlen_bsf_16match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm8B - -matchlen_match8_match_nolit_encodeBlockAsm8B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeBlockAsm8B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeBlockAsm8B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeBlockAsm8B - -matchlen_bsf_8_match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm8B - -matchlen_match4_match_nolit_encodeBlockAsm8B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeBlockAsm8B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeBlockAsm8B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeBlockAsm8B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeBlockAsm8B - JB match_nolit_end_encodeBlockAsm8B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeBlockAsm8B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeBlockAsm8B - -matchlen_match1_match_nolit_encodeBlockAsm8B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeBlockAsm8B - LEAL 1(R10), R10 - -match_nolit_end_encodeBlockAsm8B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBlockAsm8B - CMPL SI, $0x00000800 - JAE long_offset_short_match_nolit_encodeBlockAsm8B - MOVL $0x00000001, DI - LEAL 16(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b - MOVL R10, SI - LEAL -4(R10), R10 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -long_offset_short_match_nolit_encodeBlockAsm8B: - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - - // emitRepeat - MOVL R10, SI - LEAL -4(R10), R10 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short - LEAL -256(R10), R10 - MOVW $0x0019, (CX) - MOVW R10, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (CX) - MOVB R10, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - XORQ DI, DI - LEAL 1(DI)(R10*4), R10 - MOVB SI, 1(CX) - SARL $0x08, SI - SHLL $0x05, SI - ORL SI, R10 - MOVB R10, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -two_byte_offset_short_match_nolit_encodeBlockAsm8B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeBlockAsm8B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBlockAsm8B - -emit_copy_three_match_nolit_encodeBlockAsm8B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeBlockAsm8B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBlockAsm8B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBlockAsm8B: - MOVQ $0x9e3779b1, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x20, R8 - IMULQ R9, R8 - SHRQ $0x38, R8 - SHLQ $0x20, SI - IMULQ R9, SI - SHRQ $0x38, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeBlockAsm8B - INCL DX - JMP search_loop_encodeBlockAsm8B - -emit_remainder_encodeBlockAsm8B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBlockAsm8B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBlockAsm8B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBlockAsm8B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBlockAsm8B - JB three_bytes_emit_remainder_encodeBlockAsm8B - -three_bytes_emit_remainder_encodeBlockAsm8B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBlockAsm8B - -two_bytes_emit_remainder_encodeBlockAsm8B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBlockAsm8B - JMP memmove_long_emit_remainder_encodeBlockAsm8B - -one_byte_emit_remainder_encodeBlockAsm8B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBlockAsm8B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBlockAsm8B - -memmove_long_emit_remainder_encodeBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBlockAsm8B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBetterBlockAsm(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00001200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBetterBlockAsm: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBetterBlockAsm - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -6(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBetterBlockAsm: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x07, SI - CMPL SI, $0x63 - JBE check_maxskip_ok_encodeBetterBlockAsm - LEAL 100(DX), SI - JMP check_maxskip_cont_encodeBetterBlockAsm - -check_maxskip_ok_encodeBetterBlockAsm: - LEAL 1(DX)(SI*1), SI - -check_maxskip_cont_encodeBetterBlockAsm: - CMPL SI, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x00cf1bbcdcbfa563, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL 524288(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 524288(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeBetterBlockAsm - CMPQ R11, DI - JNE no_short_found_encodeBetterBlockAsm - MOVL R8, SI - JMP candidate_match_encodeBetterBlockAsm - -no_short_found_encodeBetterBlockAsm: - CMPL R10, DI - JEQ candidate_match_encodeBetterBlockAsm - CMPL R11, DI - JEQ candidateS_match_encodeBetterBlockAsm - MOVL 20(SP), DX - JMP search_loop_encodeBetterBlockAsm - -candidateS_match_encodeBetterBlockAsm: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBetterBlockAsm - DECL DX - MOVL R8, SI - -candidate_match_encodeBetterBlockAsm: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBetterBlockAsm - -match_extend_back_loop_encodeBetterBlockAsm: - CMPL DX, DI - JBE match_extend_back_end_encodeBetterBlockAsm - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBetterBlockAsm - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBetterBlockAsm - JMP match_extend_back_loop_encodeBetterBlockAsm - -match_extend_back_end_encodeBetterBlockAsm: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 5(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBetterBlockAsm: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeBetterBlockAsm: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeBetterBlockAsm - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm - -matchlen_bsf_16match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm - -matchlen_match8_match_nolit_encodeBetterBlockAsm: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeBetterBlockAsm - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeBetterBlockAsm - -matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm - -matchlen_match4_match_nolit_encodeBetterBlockAsm: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeBetterBlockAsm - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeBetterBlockAsm - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeBetterBlockAsm: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeBetterBlockAsm - JB match_nolit_end_encodeBetterBlockAsm - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeBetterBlockAsm - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeBetterBlockAsm - -matchlen_match1_match_nolit_encodeBetterBlockAsm: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeBetterBlockAsm - LEAL 1(R12), R12 - -match_nolit_end_encodeBetterBlockAsm: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL 16(SP), R8 - JEQ match_is_repeat_encodeBetterBlockAsm - CMPL R12, $0x01 - JA match_length_ok_encodeBetterBlockAsm - CMPL R8, $0x0000ffff - JBE match_length_ok_encodeBetterBlockAsm - MOVL 20(SP), DX - INCL DX - JMP search_loop_encodeBetterBlockAsm - -match_length_ok_encodeBetterBlockAsm: - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeBetterBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeBetterBlockAsm - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeBetterBlockAsm - CMPL SI, $0x00010000 - JB three_bytes_match_emit_encodeBetterBlockAsm - CMPL SI, $0x01000000 - JB four_bytes_match_emit_encodeBetterBlockAsm - MOVB $0xfc, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm - -four_bytes_match_emit_encodeBetterBlockAsm: - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm - -three_bytes_match_emit_encodeBetterBlockAsm: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm - -two_bytes_match_emit_encodeBetterBlockAsm: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeBetterBlockAsm - JMP memmove_long_match_emit_encodeBetterBlockAsm - -one_byte_match_emit_encodeBetterBlockAsm: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBetterBlockAsm: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeBetterBlockAsm - -memmove_long_match_emit_encodeBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeBetterBlockAsm: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R8, $0x00010000 - JB two_byte_offset_match_nolit_encodeBetterBlockAsm - CMPL R12, $0x40 - JBE four_bytes_remain_match_nolit_encodeBetterBlockAsm - MOVB $0xff, (CX) - MOVL R8, 1(CX) - LEAL -64(R12), R12 - ADDQ $0x05, CX - CMPL R12, $0x04 - JB four_bytes_remain_match_nolit_encodeBetterBlockAsm - - // emitRepeat -emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy: - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy - CMPL R12, $0x0100ffff - JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy - LEAL -16842747(R12), R12 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy - -repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy: - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -four_bytes_remain_match_nolit_encodeBetterBlockAsm: - TESTL R12, R12 - JZ match_nolit_emitcopy_end_encodeBetterBlockAsm - XORL SI, SI - LEAL -1(SI)(R12*4), R12 - MOVB R12, (CX) - MOVL R8, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -two_byte_offset_match_nolit_encodeBetterBlockAsm: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm - CMPL R8, $0x00000800 - JAE long_offset_short_match_nolit_encodeBetterBlockAsm - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB R8, 1(CX) - MOVL R8, R9 - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, SI - MOVB SI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R12 - - // emitRepeat - LEAL -4(R12), R12 - JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - -emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - CMPL R12, $0x0100ffff - JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - LEAL -16842747(R12), R12 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b - -repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -long_offset_short_match_nolit_encodeBetterBlockAsm: - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - - // emitRepeat -emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short: - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short - CMPL R12, $0x0100ffff - JB repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short - LEAL -16842747(R12), R12 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short - -repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short: - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -two_byte_offset_short_match_nolit_encodeBetterBlockAsm: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -emit_copy_three_match_nolit_encodeBetterBlockAsm: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -match_is_repeat_encodeBetterBlockAsm: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_repeat_encodeBetterBlockAsm - CMPL SI, $0x00000100 - JB two_bytes_match_emit_repeat_encodeBetterBlockAsm - CMPL SI, $0x00010000 - JB three_bytes_match_emit_repeat_encodeBetterBlockAsm - CMPL SI, $0x01000000 - JB four_bytes_match_emit_repeat_encodeBetterBlockAsm - MOVB $0xfc, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm - -four_bytes_match_emit_repeat_encodeBetterBlockAsm: - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm - -three_bytes_match_emit_repeat_encodeBetterBlockAsm: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm - -two_bytes_match_emit_repeat_encodeBetterBlockAsm: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_repeat_encodeBetterBlockAsm - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm - -one_byte_match_emit_repeat_encodeBetterBlockAsm: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_repeat_encodeBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64 - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm: - MOVQ SI, CX - JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm - -memmove_long_match_emit_repeat_encodeBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_repeat_encodeBetterBlockAsm: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitRepeat -emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm: - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm - -cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_repeat_encodeBetterBlockAsm - CMPL R12, $0x0100ffff - JB repeat_five_match_nolit_repeat_encodeBetterBlockAsm - LEAL -16842747(R12), R12 - MOVL $0xfffb001d, (CX) - MOVB $0xff, 4(CX) - ADDQ $0x05, CX - JMP emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm - -repeat_five_match_nolit_repeat_encodeBetterBlockAsm: - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_four_match_nolit_repeat_encodeBetterBlockAsm: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_three_match_nolit_repeat_encodeBetterBlockAsm: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_match_nolit_repeat_encodeBetterBlockAsm: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm - -repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - -match_nolit_emitcopy_end_encodeBetterBlockAsm: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBetterBlockAsm: - MOVQ $0x00cf1bbcdcbfa563, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x32, R11 - SHLQ $0x08, R12 - IMULQ SI, R12 - SHRQ $0x2f, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x32, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 524288(AX)(R11*4) - MOVL R14, 524288(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeBetterBlockAsm: - CMPQ R8, R9 - JAE search_loop_encodeBetterBlockAsm - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x08, R11 - IMULQ SI, R11 - SHRQ $0x2f, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeBetterBlockAsm - -emit_remainder_encodeBetterBlockAsm: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 5(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBetterBlockAsm: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBetterBlockAsm - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBetterBlockAsm - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeBetterBlockAsm - CMPL DX, $0x01000000 - JB four_bytes_emit_remainder_encodeBetterBlockAsm - MOVB $0xfc, (CX) - MOVL DX, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm - -four_bytes_emit_remainder_encodeBetterBlockAsm: - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm - -three_bytes_emit_remainder_encodeBetterBlockAsm: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm - -two_bytes_emit_remainder_encodeBetterBlockAsm: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBetterBlockAsm - JMP memmove_long_emit_remainder_encodeBetterBlockAsm - -one_byte_emit_remainder_encodeBetterBlockAsm: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBetterBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBetterBlockAsm: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm - -memmove_long_emit_remainder_encodeBetterBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBetterBlockAsm: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBetterBlockAsm4MB(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00001200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBetterBlockAsm4MB: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBetterBlockAsm4MB - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -6(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBetterBlockAsm4MB: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x07, SI - CMPL SI, $0x63 - JBE check_maxskip_ok_encodeBetterBlockAsm4MB - LEAL 100(DX), SI - JMP check_maxskip_cont_encodeBetterBlockAsm4MB - -check_maxskip_ok_encodeBetterBlockAsm4MB: - LEAL 1(DX)(SI*1), SI - -check_maxskip_cont_encodeBetterBlockAsm4MB: - CMPL SI, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm4MB - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x00cf1bbcdcbfa563, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL 524288(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 524288(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeBetterBlockAsm4MB - CMPQ R11, DI - JNE no_short_found_encodeBetterBlockAsm4MB - MOVL R8, SI - JMP candidate_match_encodeBetterBlockAsm4MB - -no_short_found_encodeBetterBlockAsm4MB: - CMPL R10, DI - JEQ candidate_match_encodeBetterBlockAsm4MB - CMPL R11, DI - JEQ candidateS_match_encodeBetterBlockAsm4MB - MOVL 20(SP), DX - JMP search_loop_encodeBetterBlockAsm4MB - -candidateS_match_encodeBetterBlockAsm4MB: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBetterBlockAsm4MB - DECL DX - MOVL R8, SI - -candidate_match_encodeBetterBlockAsm4MB: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBetterBlockAsm4MB - -match_extend_back_loop_encodeBetterBlockAsm4MB: - CMPL DX, DI - JBE match_extend_back_end_encodeBetterBlockAsm4MB - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBetterBlockAsm4MB - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBetterBlockAsm4MB - JMP match_extend_back_loop_encodeBetterBlockAsm4MB - -match_extend_back_end_encodeBetterBlockAsm4MB: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 4(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBetterBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBetterBlockAsm4MB: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB - -matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB - -matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB - -matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB - -matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeBetterBlockAsm4MB - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeBetterBlockAsm4MB - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeBetterBlockAsm4MB: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB - JB match_nolit_end_encodeBetterBlockAsm4MB - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeBetterBlockAsm4MB - -matchlen_match1_match_nolit_encodeBetterBlockAsm4MB: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeBetterBlockAsm4MB - LEAL 1(R12), R12 - -match_nolit_end_encodeBetterBlockAsm4MB: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL 16(SP), R8 - JEQ match_is_repeat_encodeBetterBlockAsm4MB - CMPL R12, $0x01 - JA match_length_ok_encodeBetterBlockAsm4MB - CMPL R8, $0x0000ffff - JBE match_length_ok_encodeBetterBlockAsm4MB - MOVL 20(SP), DX - INCL DX - JMP search_loop_encodeBetterBlockAsm4MB - -match_length_ok_encodeBetterBlockAsm4MB: - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeBetterBlockAsm4MB - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeBetterBlockAsm4MB - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeBetterBlockAsm4MB - CMPL SI, $0x00010000 - JB three_bytes_match_emit_encodeBetterBlockAsm4MB - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm4MB - -three_bytes_match_emit_encodeBetterBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm4MB - -two_bytes_match_emit_encodeBetterBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeBetterBlockAsm4MB - JMP memmove_long_match_emit_encodeBetterBlockAsm4MB - -one_byte_match_emit_encodeBetterBlockAsm4MB: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBetterBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBetterBlockAsm4MB: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeBetterBlockAsm4MB - -memmove_long_match_emit_encodeBetterBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeBetterBlockAsm4MB: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R8, $0x00010000 - JB two_byte_offset_match_nolit_encodeBetterBlockAsm4MB - CMPL R12, $0x40 - JBE four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB - MOVB $0xff, (CX) - MOVL R8, 1(CX) - LEAL -64(R12), R12 - ADDQ $0x05, CX - CMPL R12, $0x04 - JB four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB: - TESTL R12, R12 - JZ match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - XORL SI, SI - LEAL -1(SI)(R12*4), R12 - MOVB R12, (CX) - MOVL R8, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -two_byte_offset_match_nolit_encodeBetterBlockAsm4MB: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB - CMPL R8, $0x00000800 - JAE long_offset_short_match_nolit_encodeBetterBlockAsm4MB - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R12 - - // emitRepeat - LEAL -4(R12), R12 - JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -long_offset_short_match_nolit_encodeBetterBlockAsm4MB: - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm4MB - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm4MB - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -emit_copy_three_match_nolit_encodeBetterBlockAsm4MB: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -match_is_repeat_encodeBetterBlockAsm4MB: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_repeat_encodeBetterBlockAsm4MB - CMPL SI, $0x00000100 - JB two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB - CMPL SI, $0x00010000 - JB three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB - -three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB - -two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_repeat_encodeBetterBlockAsm4MB - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB - -one_byte_match_emit_repeat_encodeBetterBlockAsm4MB: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_repeat_encodeBetterBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB: - MOVQ SI, CX - JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB - -memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB - -cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB - CMPL R12, $0x00010100 - JB repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB - LEAL -65536(R12), R12 - MOVL R12, R8 - MOVW $0x001d, (CX) - MOVW R12, 2(CX) - SARL $0x10, R8 - MOVB R8, 4(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB: - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB - -repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - -match_nolit_emitcopy_end_encodeBetterBlockAsm4MB: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm4MB - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBetterBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBetterBlockAsm4MB: - MOVQ $0x00cf1bbcdcbfa563, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x32, R11 - SHLQ $0x08, R12 - IMULQ SI, R12 - SHRQ $0x2f, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x32, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 524288(AX)(R11*4) - MOVL R14, 524288(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeBetterBlockAsm4MB: - CMPQ R8, R9 - JAE search_loop_encodeBetterBlockAsm4MB - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x08, R11 - IMULQ SI, R11 - SHRQ $0x2f, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeBetterBlockAsm4MB - -emit_remainder_encodeBetterBlockAsm4MB: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 4(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBetterBlockAsm4MB - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBetterBlockAsm4MB: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBetterBlockAsm4MB - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBetterBlockAsm4MB - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeBetterBlockAsm4MB - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB - -three_bytes_emit_remainder_encodeBetterBlockAsm4MB: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB - -two_bytes_emit_remainder_encodeBetterBlockAsm4MB: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBetterBlockAsm4MB - JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB - -one_byte_emit_remainder_encodeBetterBlockAsm4MB: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBetterBlockAsm4MB: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB - -memmove_long_emit_remainder_encodeBetterBlockAsm4MB: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBetterBlockAsm12B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000280, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBetterBlockAsm12B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBetterBlockAsm12B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -6(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBetterBlockAsm12B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm12B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x34, R11 - MOVL (AX)(R10*4), SI - MOVL 65536(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 65536(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeBetterBlockAsm12B - CMPQ R11, DI - JNE no_short_found_encodeBetterBlockAsm12B - MOVL R8, SI - JMP candidate_match_encodeBetterBlockAsm12B - -no_short_found_encodeBetterBlockAsm12B: - CMPL R10, DI - JEQ candidate_match_encodeBetterBlockAsm12B - CMPL R11, DI - JEQ candidateS_match_encodeBetterBlockAsm12B - MOVL 20(SP), DX - JMP search_loop_encodeBetterBlockAsm12B - -candidateS_match_encodeBetterBlockAsm12B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBetterBlockAsm12B - DECL DX - MOVL R8, SI - -candidate_match_encodeBetterBlockAsm12B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBetterBlockAsm12B - -match_extend_back_loop_encodeBetterBlockAsm12B: - CMPL DX, DI - JBE match_extend_back_end_encodeBetterBlockAsm12B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBetterBlockAsm12B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBetterBlockAsm12B - JMP match_extend_back_loop_encodeBetterBlockAsm12B - -match_extend_back_end_encodeBetterBlockAsm12B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBetterBlockAsm12B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeBetterBlockAsm12B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B - -matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B - -matchlen_match8_match_nolit_encodeBetterBlockAsm12B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeBetterBlockAsm12B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B - -matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B - -matchlen_match4_match_nolit_encodeBetterBlockAsm12B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeBetterBlockAsm12B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeBetterBlockAsm12B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeBetterBlockAsm12B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeBetterBlockAsm12B - JB match_nolit_end_encodeBetterBlockAsm12B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeBetterBlockAsm12B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeBetterBlockAsm12B - -matchlen_match1_match_nolit_encodeBetterBlockAsm12B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeBetterBlockAsm12B - LEAL 1(R12), R12 - -match_nolit_end_encodeBetterBlockAsm12B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL 16(SP), R8 - JEQ match_is_repeat_encodeBetterBlockAsm12B - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeBetterBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeBetterBlockAsm12B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeBetterBlockAsm12B - JB three_bytes_match_emit_encodeBetterBlockAsm12B - -three_bytes_match_emit_encodeBetterBlockAsm12B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm12B - -two_bytes_match_emit_encodeBetterBlockAsm12B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeBetterBlockAsm12B - JMP memmove_long_match_emit_encodeBetterBlockAsm12B - -one_byte_match_emit_encodeBetterBlockAsm12B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBetterBlockAsm12B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeBetterBlockAsm12B - -memmove_long_match_emit_encodeBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeBetterBlockAsm12B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B - CMPL R8, $0x00000800 - JAE long_offset_short_match_nolit_encodeBetterBlockAsm12B - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R12 - - // emitRepeat - LEAL -4(R12), R12 - JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -long_offset_short_match_nolit_encodeBetterBlockAsm12B: - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm12B - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm12B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -emit_copy_three_match_nolit_encodeBetterBlockAsm12B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -match_is_repeat_encodeBetterBlockAsm12B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_repeat_encodeBetterBlockAsm12B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_repeat_encodeBetterBlockAsm12B - JB three_bytes_match_emit_repeat_encodeBetterBlockAsm12B - -three_bytes_match_emit_repeat_encodeBetterBlockAsm12B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B - -two_bytes_match_emit_repeat_encodeBetterBlockAsm12B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_repeat_encodeBetterBlockAsm12B - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B - -one_byte_match_emit_repeat_encodeBetterBlockAsm12B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_repeat_encodeBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B - -memmove_long_match_emit_repeat_encodeBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B - -cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B - -repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - -match_nolit_emitcopy_end_encodeBetterBlockAsm12B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm12B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBetterBlockAsm12B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x32, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x34, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x32, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x34, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 65536(AX)(R11*4) - MOVL R14, 65536(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeBetterBlockAsm12B: - CMPQ R8, R9 - JAE search_loop_encodeBetterBlockAsm12B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x32, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeBetterBlockAsm12B - -emit_remainder_encodeBetterBlockAsm12B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBetterBlockAsm12B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm12B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBetterBlockAsm12B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBetterBlockAsm12B - JB three_bytes_emit_remainder_encodeBetterBlockAsm12B - -three_bytes_emit_remainder_encodeBetterBlockAsm12B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B - -two_bytes_emit_remainder_encodeBetterBlockAsm12B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBetterBlockAsm12B - JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B - -one_byte_emit_remainder_encodeBetterBlockAsm12B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBetterBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm12B - -memmove_long_emit_remainder_encodeBetterBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBetterBlockAsm12B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBetterBlockAsm10B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x000000a0, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBetterBlockAsm10B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBetterBlockAsm10B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -6(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBetterBlockAsm10B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm10B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x36, R11 - MOVL (AX)(R10*4), SI - MOVL 16384(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 16384(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeBetterBlockAsm10B - CMPQ R11, DI - JNE no_short_found_encodeBetterBlockAsm10B - MOVL R8, SI - JMP candidate_match_encodeBetterBlockAsm10B - -no_short_found_encodeBetterBlockAsm10B: - CMPL R10, DI - JEQ candidate_match_encodeBetterBlockAsm10B - CMPL R11, DI - JEQ candidateS_match_encodeBetterBlockAsm10B - MOVL 20(SP), DX - JMP search_loop_encodeBetterBlockAsm10B - -candidateS_match_encodeBetterBlockAsm10B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBetterBlockAsm10B - DECL DX - MOVL R8, SI - -candidate_match_encodeBetterBlockAsm10B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBetterBlockAsm10B - -match_extend_back_loop_encodeBetterBlockAsm10B: - CMPL DX, DI - JBE match_extend_back_end_encodeBetterBlockAsm10B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBetterBlockAsm10B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBetterBlockAsm10B - JMP match_extend_back_loop_encodeBetterBlockAsm10B - -match_extend_back_end_encodeBetterBlockAsm10B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBetterBlockAsm10B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeBetterBlockAsm10B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B - -matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B - -matchlen_match8_match_nolit_encodeBetterBlockAsm10B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeBetterBlockAsm10B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B - -matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B - -matchlen_match4_match_nolit_encodeBetterBlockAsm10B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeBetterBlockAsm10B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeBetterBlockAsm10B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeBetterBlockAsm10B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeBetterBlockAsm10B - JB match_nolit_end_encodeBetterBlockAsm10B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeBetterBlockAsm10B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeBetterBlockAsm10B - -matchlen_match1_match_nolit_encodeBetterBlockAsm10B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeBetterBlockAsm10B - LEAL 1(R12), R12 - -match_nolit_end_encodeBetterBlockAsm10B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL 16(SP), R8 - JEQ match_is_repeat_encodeBetterBlockAsm10B - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeBetterBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeBetterBlockAsm10B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeBetterBlockAsm10B - JB three_bytes_match_emit_encodeBetterBlockAsm10B - -three_bytes_match_emit_encodeBetterBlockAsm10B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm10B - -two_bytes_match_emit_encodeBetterBlockAsm10B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeBetterBlockAsm10B - JMP memmove_long_match_emit_encodeBetterBlockAsm10B - -one_byte_match_emit_encodeBetterBlockAsm10B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBetterBlockAsm10B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeBetterBlockAsm10B - -memmove_long_match_emit_encodeBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeBetterBlockAsm10B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B - CMPL R8, $0x00000800 - JAE long_offset_short_match_nolit_encodeBetterBlockAsm10B - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R12 - - // emitRepeat - LEAL -4(R12), R12 - JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -long_offset_short_match_nolit_encodeBetterBlockAsm10B: - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm10B - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm10B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -emit_copy_three_match_nolit_encodeBetterBlockAsm10B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -match_is_repeat_encodeBetterBlockAsm10B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_repeat_encodeBetterBlockAsm10B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_repeat_encodeBetterBlockAsm10B - JB three_bytes_match_emit_repeat_encodeBetterBlockAsm10B - -three_bytes_match_emit_repeat_encodeBetterBlockAsm10B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B - -two_bytes_match_emit_repeat_encodeBetterBlockAsm10B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_repeat_encodeBetterBlockAsm10B - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B - -one_byte_match_emit_repeat_encodeBetterBlockAsm10B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_repeat_encodeBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B - -memmove_long_match_emit_repeat_encodeBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B - CMPL R8, $0x00000800 - JB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B - -cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B - -repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - -match_nolit_emitcopy_end_encodeBetterBlockAsm10B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm10B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBetterBlockAsm10B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x34, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x36, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x34, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x36, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 16384(AX)(R11*4) - MOVL R14, 16384(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeBetterBlockAsm10B: - CMPQ R8, R9 - JAE search_loop_encodeBetterBlockAsm10B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x34, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x34, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeBetterBlockAsm10B - -emit_remainder_encodeBetterBlockAsm10B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBetterBlockAsm10B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm10B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBetterBlockAsm10B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBetterBlockAsm10B - JB three_bytes_emit_remainder_encodeBetterBlockAsm10B - -three_bytes_emit_remainder_encodeBetterBlockAsm10B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B - -two_bytes_emit_remainder_encodeBetterBlockAsm10B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBetterBlockAsm10B - JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B - -one_byte_emit_remainder_encodeBetterBlockAsm10B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBetterBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm10B - -memmove_long_emit_remainder_encodeBetterBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBetterBlockAsm10B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeBetterBlockAsm8B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000028, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeBetterBlockAsm8B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeBetterBlockAsm8B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -6(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeBetterBlockAsm8B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x04, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm8B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x38, R11 - MOVL (AX)(R10*4), SI - MOVL 4096(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 4096(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeBetterBlockAsm8B - CMPQ R11, DI - JNE no_short_found_encodeBetterBlockAsm8B - MOVL R8, SI - JMP candidate_match_encodeBetterBlockAsm8B - -no_short_found_encodeBetterBlockAsm8B: - CMPL R10, DI - JEQ candidate_match_encodeBetterBlockAsm8B - CMPL R11, DI - JEQ candidateS_match_encodeBetterBlockAsm8B - MOVL 20(SP), DX - JMP search_loop_encodeBetterBlockAsm8B - -candidateS_match_encodeBetterBlockAsm8B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeBetterBlockAsm8B - DECL DX - MOVL R8, SI - -candidate_match_encodeBetterBlockAsm8B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeBetterBlockAsm8B - -match_extend_back_loop_encodeBetterBlockAsm8B: - CMPL DX, DI - JBE match_extend_back_end_encodeBetterBlockAsm8B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeBetterBlockAsm8B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeBetterBlockAsm8B - JMP match_extend_back_loop_encodeBetterBlockAsm8B - -match_extend_back_end_encodeBetterBlockAsm8B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeBetterBlockAsm8B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeBetterBlockAsm8B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B - -matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B - -matchlen_match8_match_nolit_encodeBetterBlockAsm8B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeBetterBlockAsm8B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B - -matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B - -matchlen_match4_match_nolit_encodeBetterBlockAsm8B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeBetterBlockAsm8B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeBetterBlockAsm8B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeBetterBlockAsm8B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeBetterBlockAsm8B - JB match_nolit_end_encodeBetterBlockAsm8B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeBetterBlockAsm8B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeBetterBlockAsm8B - -matchlen_match1_match_nolit_encodeBetterBlockAsm8B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeBetterBlockAsm8B - LEAL 1(R12), R12 - -match_nolit_end_encodeBetterBlockAsm8B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL 16(SP), R8 - JEQ match_is_repeat_encodeBetterBlockAsm8B - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeBetterBlockAsm8B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeBetterBlockAsm8B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeBetterBlockAsm8B - JB three_bytes_match_emit_encodeBetterBlockAsm8B - -three_bytes_match_emit_encodeBetterBlockAsm8B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeBetterBlockAsm8B - -two_bytes_match_emit_encodeBetterBlockAsm8B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeBetterBlockAsm8B - JMP memmove_long_match_emit_encodeBetterBlockAsm8B - -one_byte_match_emit_encodeBetterBlockAsm8B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeBetterBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x04 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4 - CMPQ R9, $0x08 - JB emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4: - MOVL (R10), R11 - MOVL R11, (CX) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7: - MOVL (R10), R11 - MOVL -4(R10)(R9*1), R10 - MOVL R11, (CX) - MOVL R10, -4(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeBetterBlockAsm8B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeBetterBlockAsm8B - -memmove_long_match_emit_encodeBetterBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeBetterBlockAsm8B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B - CMPL R8, $0x00000800 - JAE long_offset_short_match_nolit_encodeBetterBlockAsm8B - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - SUBL $0x08, R12 - - // emitRepeat - LEAL -4(R12), R12 - JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -long_offset_short_match_nolit_encodeBetterBlockAsm8B: - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short - -cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeBetterBlockAsm8B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -emit_copy_three_match_nolit_encodeBetterBlockAsm8B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -match_is_repeat_encodeBetterBlockAsm8B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_match_emit_repeat_encodeBetterBlockAsm8B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_repeat_encodeBetterBlockAsm8B - JB three_bytes_match_emit_repeat_encodeBetterBlockAsm8B - -three_bytes_match_emit_repeat_encodeBetterBlockAsm8B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B - -two_bytes_match_emit_repeat_encodeBetterBlockAsm8B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_repeat_encodeBetterBlockAsm8B - JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B - -one_byte_match_emit_repeat_encodeBetterBlockAsm8B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_repeat_encodeBetterBlockAsm8B: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x04 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4 - CMPQ R8, $0x08 - JB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7 - CMPQ R8, $0x10 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4: - MOVL (R9), R10 - MOVL R10, (CX) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7: - MOVL (R9), R10 - MOVL -4(R9)(R8*1), R9 - MOVL R10, (CX) - MOVL R9, -4(CX)(R8*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B - -emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B - -memmove_long_match_emit_repeat_encodeBetterBlockAsm8B: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R13 - SUBQ R10, R13 - DECQ R11 - JA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(R9)(R13*1), R10 - LEAQ -32(CX)(R13*1), R14 - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R10 - ADDQ $0x20, R13 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(R9)(R13*1), X4 - MOVOU -16(R9)(R13*1), X5 - MOVOA X4, -32(CX)(R13*1) - MOVOA X5, -16(CX)(R13*1) - ADDQ $0x20, R13 - CMPQ R8, R13 - JAE emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitRepeat - MOVL R12, SI - LEAL -4(R12), R12 - CMPL SI, $0x08 - JBE repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B - CMPL SI, $0x0c - JAE cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B - -cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B: - CMPL R12, $0x00000104 - JB repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B - LEAL -256(R12), R12 - MOVW $0x0019, (CX) - MOVW R12, 2(CX) - ADDQ $0x04, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B: - LEAL -4(R12), R12 - MOVW $0x0015, (CX) - MOVB R12, 2(CX) - ADDQ $0x03, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - -repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B: - SHLL $0x02, R12 - ORL $0x01, R12 - MOVW R12, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B - XORQ SI, SI - LEAL 1(SI)(R12*4), R12 - MOVB R8, 1(CX) - SARL $0x08, R8 - SHLL $0x05, R8 - ORL R8, R12 - MOVB R12, (CX) - ADDQ $0x02, CX - -match_nolit_emitcopy_end_encodeBetterBlockAsm8B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeBetterBlockAsm8B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeBetterBlockAsm8B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x38, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x36, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x38, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 4096(AX)(R11*4) - MOVL R14, 4096(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeBetterBlockAsm8B: - CMPQ R8, R9 - JAE search_loop_encodeBetterBlockAsm8B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x36, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x36, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeBetterBlockAsm8B - -emit_remainder_encodeBetterBlockAsm8B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeBetterBlockAsm8B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm8B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeBetterBlockAsm8B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeBetterBlockAsm8B - JB three_bytes_emit_remainder_encodeBetterBlockAsm8B - -three_bytes_emit_remainder_encodeBetterBlockAsm8B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B - -two_bytes_emit_remainder_encodeBetterBlockAsm8B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeBetterBlockAsm8B - JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B - -one_byte_emit_remainder_encodeBetterBlockAsm8B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeBetterBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm8B - -memmove_long_emit_remainder_encodeBetterBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeBetterBlockAsm8B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBlockAsm(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBlockAsm: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBlockAsm - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBlockAsm: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ R9, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeSnappyBlockAsm - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_encodeSnappyBlockAsm - -repeat_extend_back_loop_encodeSnappyBlockAsm: - CMPL DI, SI - JBE repeat_extend_back_end_encodeSnappyBlockAsm - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeSnappyBlockAsm - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_encodeSnappyBlockAsm - -repeat_extend_back_end_encodeSnappyBlockAsm: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 5(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeSnappyBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeSnappyBlockAsm: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeSnappyBlockAsm - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeSnappyBlockAsm - CMPL SI, $0x00010000 - JB three_bytes_repeat_emit_encodeSnappyBlockAsm - CMPL SI, $0x01000000 - JB four_bytes_repeat_emit_encodeSnappyBlockAsm - MOVB $0xfc, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm - -four_bytes_repeat_emit_encodeSnappyBlockAsm: - MOVL SI, R10 - SHRL $0x10, R10 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R10, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm - -three_bytes_repeat_emit_encodeSnappyBlockAsm: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm - -two_bytes_repeat_emit_encodeSnappyBlockAsm: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeSnappyBlockAsm - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm - -one_byte_repeat_emit_encodeSnappyBlockAsm: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeSnappyBlockAsm: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8: - MOVQ (R9), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_repeat_emit_encodeSnappyBlockAsm: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm - -memmove_long_repeat_emit_encodeSnappyBlockAsm: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(R9)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(R9)(R12*1), X4 - MOVOU -16(R9)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R8, R12 - JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeSnappyBlockAsm: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm - -matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm - -matchlen_match8_repeat_extend_encodeSnappyBlockAsm: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm - -matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm - -matchlen_match4_repeat_extend_encodeSnappyBlockAsm: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_encodeSnappyBlockAsm: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm - JB repeat_extend_forward_end_encodeSnappyBlockAsm - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_encodeSnappyBlockAsm - -matchlen_match1_repeat_extend_encodeSnappyBlockAsm: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_encodeSnappyBlockAsm - LEAL 1(R11), R11 - -repeat_extend_forward_end_encodeSnappyBlockAsm: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy - CMPL DI, $0x00010000 - JB two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm - -four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm: - CMPL SI, $0x40 - JBE four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm - MOVB $0xff, (CX) - MOVL DI, 1(CX) - LEAL -64(SI), SI - ADDQ $0x05, CX - CMPL SI, $0x04 - JB four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm - JMP four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm - -four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm: - TESTL SI, SI - JZ repeat_end_emit_encodeSnappyBlockAsm - XORL R8, R8 - LEAL -1(R8)(SI*4), SI - MOVB SI, (CX) - MOVL DI, 1(CX) - ADDQ $0x05, CX - JMP repeat_end_emit_encodeSnappyBlockAsm - -two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm - -two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeSnappyBlockAsm - -emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeSnappyBlockAsm: - MOVL DX, 12(SP) - JMP search_loop_encodeSnappyBlockAsm - -no_repeat_found_encodeSnappyBlockAsm: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBlockAsm - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeSnappyBlockAsm - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeSnappyBlockAsm - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBlockAsm - -candidate3_match_encodeSnappyBlockAsm: - ADDL $0x02, DX - JMP candidate_match_encodeSnappyBlockAsm - -candidate2_match_encodeSnappyBlockAsm: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeSnappyBlockAsm: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBlockAsm - -match_extend_back_loop_encodeSnappyBlockAsm: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBlockAsm - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBlockAsm - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBlockAsm - JMP match_extend_back_loop_encodeSnappyBlockAsm - -match_extend_back_end_encodeSnappyBlockAsm: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 5(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBlockAsm: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeSnappyBlockAsm - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBlockAsm - CMPL R8, $0x00010000 - JB three_bytes_match_emit_encodeSnappyBlockAsm - CMPL R8, $0x01000000 - JB four_bytes_match_emit_encodeSnappyBlockAsm - MOVB $0xfc, (CX) - MOVL R8, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm - -four_bytes_match_emit_encodeSnappyBlockAsm: - MOVL R8, R10 - SHRL $0x10, R10 - MOVB $0xf8, (CX) - MOVW R8, 1(CX) - MOVB R10, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm - -three_bytes_match_emit_encodeSnappyBlockAsm: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm - -two_bytes_match_emit_encodeSnappyBlockAsm: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeSnappyBlockAsm - JMP memmove_long_match_emit_encodeSnappyBlockAsm - -one_byte_match_emit_encodeSnappyBlockAsm: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBlockAsm: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBlockAsm: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeSnappyBlockAsm - -memmove_long_match_emit_encodeSnappyBlockAsm: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeSnappyBlockAsm: -match_nolit_loop_encodeSnappyBlockAsm: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBlockAsm - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm - -matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm - -matchlen_match8_match_nolit_encodeSnappyBlockAsm: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBlockAsm - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm - -matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm - -matchlen_match4_match_nolit_encodeSnappyBlockAsm: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBlockAsm - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeSnappyBlockAsm: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBlockAsm - JB match_nolit_end_encodeSnappyBlockAsm - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeSnappyBlockAsm - -matchlen_match1_match_nolit_encodeSnappyBlockAsm: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeSnappyBlockAsm - LEAL 1(R10), R10 - -match_nolit_end_encodeSnappyBlockAsm: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL SI, $0x00010000 - JB two_byte_offset_match_nolit_encodeSnappyBlockAsm - -four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm: - CMPL R10, $0x40 - JBE four_bytes_remain_match_nolit_encodeSnappyBlockAsm - MOVB $0xff, (CX) - MOVL SI, 1(CX) - LEAL -64(R10), R10 - ADDQ $0x05, CX - CMPL R10, $0x04 - JB four_bytes_remain_match_nolit_encodeSnappyBlockAsm - JMP four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm - -four_bytes_remain_match_nolit_encodeSnappyBlockAsm: - TESTL R10, R10 - JZ match_nolit_emitcopy_end_encodeSnappyBlockAsm - XORL DI, DI - LEAL -1(DI)(R10*4), R10 - MOVB R10, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm - -two_byte_offset_match_nolit_encodeSnappyBlockAsm: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm - -two_byte_offset_short_match_nolit_encodeSnappyBlockAsm: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm - -emit_copy_three_match_nolit_encodeSnappyBlockAsm: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBlockAsm: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBlockAsm: - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x10, R8 - IMULQ R9, R8 - SHRQ $0x32, R8 - SHLQ $0x10, SI - IMULQ R9, SI - SHRQ $0x32, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeSnappyBlockAsm - INCL DX - JMP search_loop_encodeSnappyBlockAsm - -emit_remainder_encodeSnappyBlockAsm: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 5(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBlockAsm: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBlockAsm - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBlockAsm - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeSnappyBlockAsm - CMPL DX, $0x01000000 - JB four_bytes_emit_remainder_encodeSnappyBlockAsm - MOVB $0xfc, (CX) - MOVL DX, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm - -four_bytes_emit_remainder_encodeSnappyBlockAsm: - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm - -three_bytes_emit_remainder_encodeSnappyBlockAsm: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm - -two_bytes_emit_remainder_encodeSnappyBlockAsm: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBlockAsm - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm - -one_byte_emit_remainder_encodeSnappyBlockAsm: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBlockAsm: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm - -memmove_long_emit_remainder_encodeSnappyBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBlockAsm: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBlockAsm64K(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBlockAsm64K: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBlockAsm64K - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBlockAsm64K: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm64K - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ R9, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeSnappyBlockAsm64K - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_encodeSnappyBlockAsm64K - -repeat_extend_back_loop_encodeSnappyBlockAsm64K: - CMPL DI, SI - JBE repeat_extend_back_end_encodeSnappyBlockAsm64K - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeSnappyBlockAsm64K - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_encodeSnappyBlockAsm64K - -repeat_extend_back_end_encodeSnappyBlockAsm64K: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeSnappyBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeSnappyBlockAsm64K: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeSnappyBlockAsm64K - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeSnappyBlockAsm64K - JB three_bytes_repeat_emit_encodeSnappyBlockAsm64K - -three_bytes_repeat_emit_encodeSnappyBlockAsm64K: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K - -two_bytes_repeat_emit_encodeSnappyBlockAsm64K: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeSnappyBlockAsm64K - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K - -one_byte_repeat_emit_encodeSnappyBlockAsm64K: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeSnappyBlockAsm64K: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8: - MOVQ (R9), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K - -memmove_long_repeat_emit_encodeSnappyBlockAsm64K: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - LEAQ -32(R9)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32: - MOVOU -32(R9)(R12*1), X4 - MOVOU -16(R9)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R8, R12 - JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K - -matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K - -matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K - -matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K - -matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K - JB repeat_extend_forward_end_encodeSnappyBlockAsm64K - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_encodeSnappyBlockAsm64K - -matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_encodeSnappyBlockAsm64K - LEAL 1(R11), R11 - -repeat_extend_forward_end_encodeSnappyBlockAsm64K: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy -two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K - -two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeSnappyBlockAsm64K - -emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeSnappyBlockAsm64K: - MOVL DX, 12(SP) - JMP search_loop_encodeSnappyBlockAsm64K - -no_repeat_found_encodeSnappyBlockAsm64K: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBlockAsm64K - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeSnappyBlockAsm64K - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeSnappyBlockAsm64K - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBlockAsm64K - -candidate3_match_encodeSnappyBlockAsm64K: - ADDL $0x02, DX - JMP candidate_match_encodeSnappyBlockAsm64K - -candidate2_match_encodeSnappyBlockAsm64K: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeSnappyBlockAsm64K: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBlockAsm64K - -match_extend_back_loop_encodeSnappyBlockAsm64K: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBlockAsm64K - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBlockAsm64K - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBlockAsm64K - JMP match_extend_back_loop_encodeSnappyBlockAsm64K - -match_extend_back_end_encodeSnappyBlockAsm64K: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBlockAsm64K: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm64K - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeSnappyBlockAsm64K - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBlockAsm64K - JB three_bytes_match_emit_encodeSnappyBlockAsm64K - -three_bytes_match_emit_encodeSnappyBlockAsm64K: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm64K - -two_bytes_match_emit_encodeSnappyBlockAsm64K: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeSnappyBlockAsm64K - JMP memmove_long_match_emit_encodeSnappyBlockAsm64K - -one_byte_match_emit_encodeSnappyBlockAsm64K: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBlockAsm64K: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBlockAsm64K: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeSnappyBlockAsm64K - -memmove_long_match_emit_encodeSnappyBlockAsm64K: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeSnappyBlockAsm64K: -match_nolit_loop_encodeSnappyBlockAsm64K: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBlockAsm64K - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K - -matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K - -matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBlockAsm64K - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K - -matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K - -matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBlockAsm64K - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm64K - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeSnappyBlockAsm64K: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K - JB match_nolit_end_encodeSnappyBlockAsm64K - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeSnappyBlockAsm64K - -matchlen_match1_match_nolit_encodeSnappyBlockAsm64K: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeSnappyBlockAsm64K - LEAL 1(R10), R10 - -match_nolit_end_encodeSnappyBlockAsm64K: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBlockAsm64K: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm64K - -two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm64K - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm64K - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm64K - -emit_copy_three_match_nolit_encodeSnappyBlockAsm64K: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBlockAsm64K: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm64K - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBlockAsm64K: - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x10, R8 - IMULQ R9, R8 - SHRQ $0x32, R8 - SHLQ $0x10, SI - IMULQ R9, SI - SHRQ $0x32, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeSnappyBlockAsm64K - INCL DX - JMP search_loop_encodeSnappyBlockAsm64K - -emit_remainder_encodeSnappyBlockAsm64K: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBlockAsm64K: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBlockAsm64K - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBlockAsm64K - JB three_bytes_emit_remainder_encodeSnappyBlockAsm64K - -three_bytes_emit_remainder_encodeSnappyBlockAsm64K: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K - -two_bytes_emit_remainder_encodeSnappyBlockAsm64K: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBlockAsm64K - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K - -one_byte_emit_remainder_encodeSnappyBlockAsm64K: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBlockAsm64K: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K - -memmove_long_emit_remainder_encodeSnappyBlockAsm64K: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBlockAsm12B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000080, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBlockAsm12B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBlockAsm12B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBlockAsm12B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm12B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x000000cf1bbcdcbb, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x18, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - SHLQ $0x18, R11 - IMULQ R9, R11 - SHRQ $0x34, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x18, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeSnappyBlockAsm12B - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_encodeSnappyBlockAsm12B - -repeat_extend_back_loop_encodeSnappyBlockAsm12B: - CMPL DI, SI - JBE repeat_extend_back_end_encodeSnappyBlockAsm12B - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeSnappyBlockAsm12B - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_encodeSnappyBlockAsm12B - -repeat_extend_back_end_encodeSnappyBlockAsm12B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeSnappyBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeSnappyBlockAsm12B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeSnappyBlockAsm12B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeSnappyBlockAsm12B - JB three_bytes_repeat_emit_encodeSnappyBlockAsm12B - -three_bytes_repeat_emit_encodeSnappyBlockAsm12B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B - -two_bytes_repeat_emit_encodeSnappyBlockAsm12B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeSnappyBlockAsm12B - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B - -one_byte_repeat_emit_encodeSnappyBlockAsm12B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeSnappyBlockAsm12B: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8: - MOVQ (R9), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B - -memmove_long_repeat_emit_encodeSnappyBlockAsm12B: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(R9)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(R9)(R12*1), X4 - MOVOU -16(R9)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R8, R12 - JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B - -matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B - -matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B - -matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B - -matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B - JB repeat_extend_forward_end_encodeSnappyBlockAsm12B - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_encodeSnappyBlockAsm12B - -matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_encodeSnappyBlockAsm12B - LEAL 1(R11), R11 - -repeat_extend_forward_end_encodeSnappyBlockAsm12B: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy -two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B - -two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeSnappyBlockAsm12B - -emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeSnappyBlockAsm12B: - MOVL DX, 12(SP) - JMP search_loop_encodeSnappyBlockAsm12B - -no_repeat_found_encodeSnappyBlockAsm12B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBlockAsm12B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeSnappyBlockAsm12B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeSnappyBlockAsm12B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBlockAsm12B - -candidate3_match_encodeSnappyBlockAsm12B: - ADDL $0x02, DX - JMP candidate_match_encodeSnappyBlockAsm12B - -candidate2_match_encodeSnappyBlockAsm12B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeSnappyBlockAsm12B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBlockAsm12B - -match_extend_back_loop_encodeSnappyBlockAsm12B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBlockAsm12B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBlockAsm12B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBlockAsm12B - JMP match_extend_back_loop_encodeSnappyBlockAsm12B - -match_extend_back_end_encodeSnappyBlockAsm12B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBlockAsm12B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeSnappyBlockAsm12B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBlockAsm12B - JB three_bytes_match_emit_encodeSnappyBlockAsm12B - -three_bytes_match_emit_encodeSnappyBlockAsm12B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm12B - -two_bytes_match_emit_encodeSnappyBlockAsm12B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeSnappyBlockAsm12B - JMP memmove_long_match_emit_encodeSnappyBlockAsm12B - -one_byte_match_emit_encodeSnappyBlockAsm12B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBlockAsm12B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBlockAsm12B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeSnappyBlockAsm12B - -memmove_long_match_emit_encodeSnappyBlockAsm12B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeSnappyBlockAsm12B: -match_nolit_loop_encodeSnappyBlockAsm12B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBlockAsm12B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B - -matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B - -matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBlockAsm12B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B - -matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B - -matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBlockAsm12B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm12B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeSnappyBlockAsm12B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B - JB match_nolit_end_encodeSnappyBlockAsm12B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeSnappyBlockAsm12B - -matchlen_match1_match_nolit_encodeSnappyBlockAsm12B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeSnappyBlockAsm12B - LEAL 1(R10), R10 - -match_nolit_end_encodeSnappyBlockAsm12B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBlockAsm12B: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm12B - -two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm12B - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm12B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm12B - -emit_copy_three_match_nolit_encodeSnappyBlockAsm12B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBlockAsm12B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm12B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBlockAsm12B: - MOVQ $0x000000cf1bbcdcbb, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x18, R8 - IMULQ R9, R8 - SHRQ $0x34, R8 - SHLQ $0x18, SI - IMULQ R9, SI - SHRQ $0x34, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeSnappyBlockAsm12B - INCL DX - JMP search_loop_encodeSnappyBlockAsm12B - -emit_remainder_encodeSnappyBlockAsm12B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBlockAsm12B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBlockAsm12B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBlockAsm12B - JB three_bytes_emit_remainder_encodeSnappyBlockAsm12B - -three_bytes_emit_remainder_encodeSnappyBlockAsm12B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B - -two_bytes_emit_remainder_encodeSnappyBlockAsm12B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBlockAsm12B - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B - -one_byte_emit_remainder_encodeSnappyBlockAsm12B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B - -memmove_long_emit_remainder_encodeSnappyBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBlockAsm10B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000020, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBlockAsm10B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBlockAsm10B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBlockAsm10B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm10B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x9e3779b1, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ R9, R11 - SHRQ $0x36, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeSnappyBlockAsm10B - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_encodeSnappyBlockAsm10B - -repeat_extend_back_loop_encodeSnappyBlockAsm10B: - CMPL DI, SI - JBE repeat_extend_back_end_encodeSnappyBlockAsm10B - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeSnappyBlockAsm10B - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_encodeSnappyBlockAsm10B - -repeat_extend_back_end_encodeSnappyBlockAsm10B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeSnappyBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeSnappyBlockAsm10B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeSnappyBlockAsm10B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeSnappyBlockAsm10B - JB three_bytes_repeat_emit_encodeSnappyBlockAsm10B - -three_bytes_repeat_emit_encodeSnappyBlockAsm10B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B - -two_bytes_repeat_emit_encodeSnappyBlockAsm10B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeSnappyBlockAsm10B - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B - -one_byte_repeat_emit_encodeSnappyBlockAsm10B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeSnappyBlockAsm10B: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8: - MOVQ (R9), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B - -memmove_long_repeat_emit_encodeSnappyBlockAsm10B: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(R9)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(R9)(R12*1), X4 - MOVOU -16(R9)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R8, R12 - JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B - -matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B - -matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B - -matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B - -matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B - JB repeat_extend_forward_end_encodeSnappyBlockAsm10B - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_encodeSnappyBlockAsm10B - -matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_encodeSnappyBlockAsm10B - LEAL 1(R11), R11 - -repeat_extend_forward_end_encodeSnappyBlockAsm10B: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy -two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B - -two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeSnappyBlockAsm10B - -emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeSnappyBlockAsm10B: - MOVL DX, 12(SP) - JMP search_loop_encodeSnappyBlockAsm10B - -no_repeat_found_encodeSnappyBlockAsm10B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBlockAsm10B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeSnappyBlockAsm10B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeSnappyBlockAsm10B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBlockAsm10B - -candidate3_match_encodeSnappyBlockAsm10B: - ADDL $0x02, DX - JMP candidate_match_encodeSnappyBlockAsm10B - -candidate2_match_encodeSnappyBlockAsm10B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeSnappyBlockAsm10B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBlockAsm10B - -match_extend_back_loop_encodeSnappyBlockAsm10B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBlockAsm10B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBlockAsm10B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBlockAsm10B - JMP match_extend_back_loop_encodeSnappyBlockAsm10B - -match_extend_back_end_encodeSnappyBlockAsm10B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBlockAsm10B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeSnappyBlockAsm10B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBlockAsm10B - JB three_bytes_match_emit_encodeSnappyBlockAsm10B - -three_bytes_match_emit_encodeSnappyBlockAsm10B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm10B - -two_bytes_match_emit_encodeSnappyBlockAsm10B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeSnappyBlockAsm10B - JMP memmove_long_match_emit_encodeSnappyBlockAsm10B - -one_byte_match_emit_encodeSnappyBlockAsm10B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBlockAsm10B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBlockAsm10B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeSnappyBlockAsm10B - -memmove_long_match_emit_encodeSnappyBlockAsm10B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeSnappyBlockAsm10B: -match_nolit_loop_encodeSnappyBlockAsm10B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBlockAsm10B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B - -matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B - -matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBlockAsm10B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B - -matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B - -matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBlockAsm10B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm10B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeSnappyBlockAsm10B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B - JB match_nolit_end_encodeSnappyBlockAsm10B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeSnappyBlockAsm10B - -matchlen_match1_match_nolit_encodeSnappyBlockAsm10B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeSnappyBlockAsm10B - LEAL 1(R10), R10 - -match_nolit_end_encodeSnappyBlockAsm10B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBlockAsm10B: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm10B - -two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm10B - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm10B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm10B - -emit_copy_three_match_nolit_encodeSnappyBlockAsm10B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBlockAsm10B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm10B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBlockAsm10B: - MOVQ $0x9e3779b1, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x20, R8 - IMULQ R9, R8 - SHRQ $0x36, R8 - SHLQ $0x20, SI - IMULQ R9, SI - SHRQ $0x36, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeSnappyBlockAsm10B - INCL DX - JMP search_loop_encodeSnappyBlockAsm10B - -emit_remainder_encodeSnappyBlockAsm10B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBlockAsm10B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBlockAsm10B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBlockAsm10B - JB three_bytes_emit_remainder_encodeSnappyBlockAsm10B - -three_bytes_emit_remainder_encodeSnappyBlockAsm10B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B - -two_bytes_emit_remainder_encodeSnappyBlockAsm10B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBlockAsm10B - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B - -one_byte_emit_remainder_encodeSnappyBlockAsm10B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B - -memmove_long_emit_remainder_encodeSnappyBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBlockAsm8B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000008, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBlockAsm8B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBlockAsm8B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBlockAsm8B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x04, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm8B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x9e3779b1, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x38, R10 - SHLQ $0x20, R11 - IMULQ R9, R11 - SHRQ $0x38, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x38, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_encodeSnappyBlockAsm8B - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_encodeSnappyBlockAsm8B - -repeat_extend_back_loop_encodeSnappyBlockAsm8B: - CMPL DI, SI - JBE repeat_extend_back_end_encodeSnappyBlockAsm8B - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_encodeSnappyBlockAsm8B - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_encodeSnappyBlockAsm8B - -repeat_extend_back_end_encodeSnappyBlockAsm8B: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_encodeSnappyBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -repeat_dst_size_check_encodeSnappyBlockAsm8B: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_encodeSnappyBlockAsm8B - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_encodeSnappyBlockAsm8B - JB three_bytes_repeat_emit_encodeSnappyBlockAsm8B - -three_bytes_repeat_emit_encodeSnappyBlockAsm8B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B - -two_bytes_repeat_emit_encodeSnappyBlockAsm8B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_encodeSnappyBlockAsm8B - JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B - -one_byte_repeat_emit_encodeSnappyBlockAsm8B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_repeat_emit_encodeSnappyBlockAsm8B: - LEAQ (CX)(R8*1), SI - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8: - MOVQ (R9), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: - MOVQ (R9), R10 - MOVQ -8(R9)(R8*1), R9 - MOVQ R10, (CX) - MOVQ R9, -8(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: - MOVOU (R9), X0 - MOVOU -16(R9)(R8*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R8*1) - JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - -memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B: - MOVQ SI, CX - JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B - -memmove_long_repeat_emit_encodeSnappyBlockAsm8B: - LEAQ (CX)(R8*1), SI - - // genMemMoveLong - MOVOU (R9), X0 - MOVOU 16(R9), X1 - MOVOU -32(R9)(R8*1), X2 - MOVOU -16(R9)(R8*1), X3 - MOVQ R8, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(R9)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(R9)(R12*1), X4 - MOVOU -16(R9)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R8, R12 - JAE emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R8*1) - MOVOU X3, -16(CX)(R8*1) - MOVQ SI, CX - -emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B - -matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B - -matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B - -matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B - -matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B - JB repeat_extend_forward_end_encodeSnappyBlockAsm8B - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_encodeSnappyBlockAsm8B - -matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_encodeSnappyBlockAsm8B - LEAL 1(R11), R11 - -repeat_extend_forward_end_encodeSnappyBlockAsm8B: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy -two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B - MOVB $0xee, (CX) - MOVW DI, 1(CX) - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B - -two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B - LEAL -15(R8), R8 - MOVB DI, 1(CX) - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, R8 - MOVB R8, (CX) - ADDQ $0x02, CX - JMP repeat_end_emit_encodeSnappyBlockAsm8B - -emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B: - LEAL -2(R8), R8 - MOVB R8, (CX) - MOVW DI, 1(CX) - ADDQ $0x03, CX - -repeat_end_emit_encodeSnappyBlockAsm8B: - MOVL DX, 12(SP) - JMP search_loop_encodeSnappyBlockAsm8B - -no_repeat_found_encodeSnappyBlockAsm8B: - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBlockAsm8B - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_encodeSnappyBlockAsm8B - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_encodeSnappyBlockAsm8B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBlockAsm8B - -candidate3_match_encodeSnappyBlockAsm8B: - ADDL $0x02, DX - JMP candidate_match_encodeSnappyBlockAsm8B - -candidate2_match_encodeSnappyBlockAsm8B: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_encodeSnappyBlockAsm8B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBlockAsm8B - -match_extend_back_loop_encodeSnappyBlockAsm8B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBlockAsm8B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBlockAsm8B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBlockAsm8B - JMP match_extend_back_loop_encodeSnappyBlockAsm8B - -match_extend_back_end_encodeSnappyBlockAsm8B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBlockAsm8B: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_encodeSnappyBlockAsm8B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), R8 - CMPL R8, $0x3c - JB one_byte_match_emit_encodeSnappyBlockAsm8B - CMPL R8, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBlockAsm8B - JB three_bytes_match_emit_encodeSnappyBlockAsm8B - -three_bytes_match_emit_encodeSnappyBlockAsm8B: - MOVB $0xf4, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBlockAsm8B - -two_bytes_match_emit_encodeSnappyBlockAsm8B: - MOVB $0xf0, (CX) - MOVB R8, 1(CX) - ADDQ $0x02, CX - CMPL R8, $0x40 - JB memmove_match_emit_encodeSnappyBlockAsm8B - JMP memmove_long_match_emit_encodeSnappyBlockAsm8B - -one_byte_match_emit_encodeSnappyBlockAsm8B: - SHLB $0x02, R8 - MOVB R8, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBlockAsm8B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8: - MOVQ (DI), R10 - MOVQ R10, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: - MOVQ (DI), R10 - MOVQ -8(DI)(R9*1), DI - MOVQ R10, (CX) - MOVQ DI, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: - MOVOU (DI), X0 - MOVOU -16(DI)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBlockAsm8B: - MOVQ R8, CX - JMP emit_literal_done_match_emit_encodeSnappyBlockAsm8B - -memmove_long_match_emit_encodeSnappyBlockAsm8B: - LEAQ (CX)(R9*1), R8 - - // genMemMoveLong - MOVOU (DI), X0 - MOVOU 16(DI), X1 - MOVOU -32(DI)(R9*1), X2 - MOVOU -16(DI)(R9*1), X3 - MOVQ R9, R11 - SHRQ $0x05, R11 - MOVQ CX, R10 - ANDL $0x0000001f, R10 - MOVQ $0x00000040, R12 - SUBQ R10, R12 - DECQ R11 - JA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(DI)(R12*1), R10 - LEAQ -32(CX)(R12*1), R13 - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: - MOVOU (R10), X4 - MOVOU 16(R10), X5 - MOVOA X4, (R13) - MOVOA X5, 16(R13) - ADDQ $0x20, R13 - ADDQ $0x20, R10 - ADDQ $0x20, R12 - DECQ R11 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(DI)(R12*1), X4 - MOVOU -16(DI)(R12*1), X5 - MOVOA X4, -32(CX)(R12*1) - MOVOA X5, -16(CX)(R12*1) - ADDQ $0x20, R12 - CMPQ R9, R12 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ R8, CX - -emit_literal_done_match_emit_encodeSnappyBlockAsm8B: -match_nolit_loop_encodeSnappyBlockAsm8B: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBlockAsm8B - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B - -matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B - -matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBlockAsm8B - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B - -matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B - -matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBlockAsm8B - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_encodeSnappyBlockAsm8B - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_encodeSnappyBlockAsm8B: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B - JB match_nolit_end_encodeSnappyBlockAsm8B - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_encodeSnappyBlockAsm8B - -matchlen_match1_match_nolit_encodeSnappyBlockAsm8B: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_encodeSnappyBlockAsm8B - LEAL 1(R10), R10 - -match_nolit_end_encodeSnappyBlockAsm8B: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBlockAsm8B: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B - MOVB $0xee, (CX) - MOVW SI, 1(CX) - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm8B - -two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBlockAsm8B - LEAL -15(DI), DI - MOVB SI, 1(CX) - SHRL $0x08, SI - SHLL $0x05, SI - ORL SI, DI - MOVB DI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm8B - -emit_copy_three_match_nolit_encodeSnappyBlockAsm8B: - LEAL -2(DI), DI - MOVB DI, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBlockAsm8B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBlockAsm8B - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBlockAsm8B: - MOVQ $0x9e3779b1, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x20, R8 - IMULQ R9, R8 - SHRQ $0x38, R8 - SHLQ $0x20, SI - IMULQ R9, SI - SHRQ $0x38, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_encodeSnappyBlockAsm8B - INCL DX - JMP search_loop_encodeSnappyBlockAsm8B - -emit_remainder_encodeSnappyBlockAsm8B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBlockAsm8B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBlockAsm8B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBlockAsm8B - JB three_bytes_emit_remainder_encodeSnappyBlockAsm8B - -three_bytes_emit_remainder_encodeSnappyBlockAsm8B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B - -two_bytes_emit_remainder_encodeSnappyBlockAsm8B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBlockAsm8B - JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B - -one_byte_emit_remainder_encodeSnappyBlockAsm8B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B - -memmove_long_emit_remainder_encodeSnappyBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00001200, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBetterBlockAsm: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBetterBlockAsm - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBetterBlockAsm: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x07, SI - CMPL SI, $0x63 - JBE check_maxskip_ok_encodeSnappyBetterBlockAsm - LEAL 100(DX), SI - JMP check_maxskip_cont_encodeSnappyBetterBlockAsm - -check_maxskip_ok_encodeSnappyBetterBlockAsm: - LEAL 1(DX)(SI*1), SI - -check_maxskip_cont_encodeSnappyBetterBlockAsm: - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x00cf1bbcdcbfa563, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x32, R11 - MOVL (AX)(R10*4), SI - MOVL 524288(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 524288(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm - CMPQ R11, DI - JNE no_short_found_encodeSnappyBetterBlockAsm - MOVL R8, SI - JMP candidate_match_encodeSnappyBetterBlockAsm - -no_short_found_encodeSnappyBetterBlockAsm: - CMPL R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm - CMPL R11, DI - JEQ candidateS_match_encodeSnappyBetterBlockAsm - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBetterBlockAsm - -candidateS_match_encodeSnappyBetterBlockAsm: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x2f, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBetterBlockAsm - DECL DX - MOVL R8, SI - -candidate_match_encodeSnappyBetterBlockAsm: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm - -match_extend_back_loop_encodeSnappyBetterBlockAsm: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBetterBlockAsm - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBetterBlockAsm - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm - JMP match_extend_back_loop_encodeSnappyBetterBlockAsm - -match_extend_back_end_encodeSnappyBetterBlockAsm: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 5(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBetterBlockAsm: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm - -matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm - -matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm - -matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm - -matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm - JB match_nolit_end_encodeSnappyBetterBlockAsm - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeSnappyBetterBlockAsm - -matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeSnappyBetterBlockAsm - LEAL 1(R12), R12 - -match_nolit_end_encodeSnappyBetterBlockAsm: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - CMPL R12, $0x01 - JA match_length_ok_encodeSnappyBetterBlockAsm - CMPL R8, $0x0000ffff - JBE match_length_ok_encodeSnappyBetterBlockAsm - MOVL 20(SP), DX - INCL DX - JMP search_loop_encodeSnappyBetterBlockAsm - -match_length_ok_encodeSnappyBetterBlockAsm: - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeSnappyBetterBlockAsm - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBetterBlockAsm - CMPL SI, $0x00010000 - JB three_bytes_match_emit_encodeSnappyBetterBlockAsm - CMPL SI, $0x01000000 - JB four_bytes_match_emit_encodeSnappyBetterBlockAsm - MOVB $0xfc, (CX) - MOVL SI, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm - -four_bytes_match_emit_encodeSnappyBetterBlockAsm: - MOVL SI, R11 - SHRL $0x10, R11 - MOVB $0xf8, (CX) - MOVW SI, 1(CX) - MOVB R11, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm - -three_bytes_match_emit_encodeSnappyBetterBlockAsm: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm - -two_bytes_match_emit_encodeSnappyBetterBlockAsm: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeSnappyBetterBlockAsm - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm - -one_byte_match_emit_encodeSnappyBetterBlockAsm: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm - -memmove_long_match_emit_encodeSnappyBetterBlockAsm: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeSnappyBetterBlockAsm: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy - CMPL R8, $0x00010000 - JB two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm - -four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R12, $0x40 - JBE four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm - MOVB $0xff, (CX) - MOVL R8, 1(CX) - LEAL -64(R12), R12 - ADDQ $0x05, CX - CMPL R12, $0x04 - JB four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm - JMP four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm - -four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm: - TESTL R12, R12 - JZ match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm - XORL SI, SI - LEAL -1(SI)(R12*4), R12 - MOVB R12, (CX) - MOVL R8, 1(CX) - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm - -two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm - -two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm - -emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBetterBlockAsm: - MOVQ $0x00cf1bbcdcbfa563, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x32, R11 - SHLQ $0x08, R12 - IMULQ SI, R12 - SHRQ $0x2f, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x32, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 524288(AX)(R11*4) - MOVL R14, 524288(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeSnappyBetterBlockAsm: - CMPQ R8, R9 - JAE search_loop_encodeSnappyBetterBlockAsm - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x2f, R10 - SHLQ $0x08, R11 - IMULQ SI, R11 - SHRQ $0x2f, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeSnappyBetterBlockAsm - -emit_remainder_encodeSnappyBetterBlockAsm: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 5(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBetterBlockAsm - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBetterBlockAsm: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm - CMPL DX, $0x00010000 - JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm - CMPL DX, $0x01000000 - JB four_bytes_emit_remainder_encodeSnappyBetterBlockAsm - MOVB $0xfc, (CX) - MOVL DX, 1(CX) - ADDQ $0x05, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm - -four_bytes_emit_remainder_encodeSnappyBetterBlockAsm: - MOVL DX, BX - SHRL $0x10, BX - MOVB $0xf8, (CX) - MOVW DX, 1(CX) - MOVB BL, 3(CX) - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm - -three_bytes_emit_remainder_encodeSnappyBetterBlockAsm: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm - -two_bytes_emit_remainder_encodeSnappyBetterBlockAsm: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBetterBlockAsm - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm - -one_byte_emit_remainder_encodeSnappyBetterBlockAsm: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBetterBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm - -memmove_long_emit_remainder_encodeSnappyBetterBlockAsm: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000900, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBetterBlockAsm64K: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBetterBlockAsm64K - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBetterBlockAsm64K: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x07, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm64K - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x00cf1bbcdcbfa563, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x30, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x33, R11 - MOVL (AX)(R10*4), SI - MOVL 262144(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 262144(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm64K - CMPQ R11, DI - JNE no_short_found_encodeSnappyBetterBlockAsm64K - MOVL R8, SI - JMP candidate_match_encodeSnappyBetterBlockAsm64K - -no_short_found_encodeSnappyBetterBlockAsm64K: - CMPL R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm64K - CMPL R11, DI - JEQ candidateS_match_encodeSnappyBetterBlockAsm64K - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBetterBlockAsm64K - -candidateS_match_encodeSnappyBetterBlockAsm64K: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x08, R10 - IMULQ R9, R10 - SHRQ $0x30, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBetterBlockAsm64K - DECL DX - MOVL R8, SI - -candidate_match_encodeSnappyBetterBlockAsm64K: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm64K - -match_extend_back_loop_encodeSnappyBetterBlockAsm64K: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBetterBlockAsm64K - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBetterBlockAsm64K - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm64K - JMP match_extend_back_loop_encodeSnappyBetterBlockAsm64K - -match_extend_back_end_encodeSnappyBetterBlockAsm64K: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBetterBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBetterBlockAsm64K: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K - -matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K - -matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K - -matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K - -matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K - JB match_nolit_end_encodeSnappyBetterBlockAsm64K - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeSnappyBetterBlockAsm64K - -matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeSnappyBetterBlockAsm64K - LEAL 1(R12), R12 - -match_nolit_end_encodeSnappyBetterBlockAsm64K: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeSnappyBetterBlockAsm64K - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBetterBlockAsm64K - JB three_bytes_match_emit_encodeSnappyBetterBlockAsm64K - -three_bytes_match_emit_encodeSnappyBetterBlockAsm64K: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K - -two_bytes_match_emit_encodeSnappyBetterBlockAsm64K: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeSnappyBetterBlockAsm64K - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K - -one_byte_match_emit_encodeSnappyBetterBlockAsm64K: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBetterBlockAsm64K: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K - -memmove_long_match_emit_encodeSnappyBetterBlockAsm64K: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K - -two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K - -emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm64K - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K: - MOVQ $0x00cf1bbcdcbfa563, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x30, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x33, R11 - SHLQ $0x08, R12 - IMULQ SI, R12 - SHRQ $0x30, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x33, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 262144(AX)(R11*4) - MOVL R14, 262144(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeSnappyBetterBlockAsm64K: - CMPQ R8, R9 - JAE search_loop_encodeSnappyBetterBlockAsm64K - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x08, R10 - IMULQ SI, R10 - SHRQ $0x30, R10 - SHLQ $0x08, R11 - IMULQ SI, R11 - SHRQ $0x30, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeSnappyBetterBlockAsm64K - -emit_remainder_encodeSnappyBetterBlockAsm64K: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBetterBlockAsm64K - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBetterBlockAsm64K: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K - JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K - -three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K - -two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBetterBlockAsm64K - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K - -one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBetterBlockAsm64K: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K - -memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000280, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBetterBlockAsm12B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBetterBlockAsm12B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBetterBlockAsm12B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x06, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm12B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x34, R11 - MOVL (AX)(R10*4), SI - MOVL 65536(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 65536(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm12B - CMPQ R11, DI - JNE no_short_found_encodeSnappyBetterBlockAsm12B - MOVL R8, SI - JMP candidate_match_encodeSnappyBetterBlockAsm12B - -no_short_found_encodeSnappyBetterBlockAsm12B: - CMPL R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm12B - CMPL R11, DI - JEQ candidateS_match_encodeSnappyBetterBlockAsm12B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBetterBlockAsm12B - -candidateS_match_encodeSnappyBetterBlockAsm12B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x32, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBetterBlockAsm12B - DECL DX - MOVL R8, SI - -candidate_match_encodeSnappyBetterBlockAsm12B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm12B - -match_extend_back_loop_encodeSnappyBetterBlockAsm12B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBetterBlockAsm12B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBetterBlockAsm12B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm12B - JMP match_extend_back_loop_encodeSnappyBetterBlockAsm12B - -match_extend_back_end_encodeSnappyBetterBlockAsm12B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBetterBlockAsm12B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B - -matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B - -matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B - -matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B - -matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B - JB match_nolit_end_encodeSnappyBetterBlockAsm12B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeSnappyBetterBlockAsm12B - -matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeSnappyBetterBlockAsm12B - LEAL 1(R12), R12 - -match_nolit_end_encodeSnappyBetterBlockAsm12B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeSnappyBetterBlockAsm12B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBetterBlockAsm12B - JB three_bytes_match_emit_encodeSnappyBetterBlockAsm12B - -three_bytes_match_emit_encodeSnappyBetterBlockAsm12B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B - -two_bytes_match_emit_encodeSnappyBetterBlockAsm12B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeSnappyBetterBlockAsm12B - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B - -one_byte_match_emit_encodeSnappyBetterBlockAsm12B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B - -memmove_long_match_emit_encodeSnappyBetterBlockAsm12B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B - -two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B - -emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm12B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x32, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x34, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x32, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x34, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 65536(AX)(R11*4) - MOVL R14, 65536(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeSnappyBetterBlockAsm12B: - CMPQ R8, R9 - JAE search_loop_encodeSnappyBetterBlockAsm12B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x32, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x32, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeSnappyBetterBlockAsm12B - -emit_remainder_encodeSnappyBetterBlockAsm12B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBetterBlockAsm12B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBetterBlockAsm12B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B - JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B - -three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B - -two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBetterBlockAsm12B - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B - -one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBetterBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B - -memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x000000a0, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBetterBlockAsm10B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBetterBlockAsm10B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBetterBlockAsm10B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm10B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x36, R11 - MOVL (AX)(R10*4), SI - MOVL 16384(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 16384(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm10B - CMPQ R11, DI - JNE no_short_found_encodeSnappyBetterBlockAsm10B - MOVL R8, SI - JMP candidate_match_encodeSnappyBetterBlockAsm10B - -no_short_found_encodeSnappyBetterBlockAsm10B: - CMPL R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm10B - CMPL R11, DI - JEQ candidateS_match_encodeSnappyBetterBlockAsm10B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBetterBlockAsm10B - -candidateS_match_encodeSnappyBetterBlockAsm10B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x34, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBetterBlockAsm10B - DECL DX - MOVL R8, SI - -candidate_match_encodeSnappyBetterBlockAsm10B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm10B - -match_extend_back_loop_encodeSnappyBetterBlockAsm10B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBetterBlockAsm10B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBetterBlockAsm10B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm10B - JMP match_extend_back_loop_encodeSnappyBetterBlockAsm10B - -match_extend_back_end_encodeSnappyBetterBlockAsm10B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBetterBlockAsm10B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B - -matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B - -matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B - -matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B - -matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B - JB match_nolit_end_encodeSnappyBetterBlockAsm10B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeSnappyBetterBlockAsm10B - -matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeSnappyBetterBlockAsm10B - LEAL 1(R12), R12 - -match_nolit_end_encodeSnappyBetterBlockAsm10B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeSnappyBetterBlockAsm10B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBetterBlockAsm10B - JB three_bytes_match_emit_encodeSnappyBetterBlockAsm10B - -three_bytes_match_emit_encodeSnappyBetterBlockAsm10B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B - -two_bytes_match_emit_encodeSnappyBetterBlockAsm10B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeSnappyBetterBlockAsm10B - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B - -one_byte_match_emit_encodeSnappyBetterBlockAsm10B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B - -memmove_long_match_emit_encodeSnappyBetterBlockAsm10B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B - -two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B - CMPL R8, $0x00000800 - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B - -emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm10B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x34, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x36, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x34, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x36, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 16384(AX)(R11*4) - MOVL R14, 16384(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeSnappyBetterBlockAsm10B: - CMPQ R8, R9 - JAE search_loop_encodeSnappyBetterBlockAsm10B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x34, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x34, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeSnappyBetterBlockAsm10B - -emit_remainder_encodeSnappyBetterBlockAsm10B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBetterBlockAsm10B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBetterBlockAsm10B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B - JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B - -three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B - -two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBetterBlockAsm10B - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B - -one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBetterBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B - -memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int -// Requires: BMI, SSE2 -TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64 - MOVQ tmp+48(FP), AX - MOVQ dst_base+0(FP), CX - MOVQ $0x00000028, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_encodeSnappyBetterBlockAsm8B: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_encodeSnappyBetterBlockAsm8B - MOVL $0x00000000, 12(SP) - MOVQ src_len+32(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL $0x00000000, 16(SP) - MOVQ src_base+24(FP), BX - -search_loop_encodeSnappyBetterBlockAsm8B: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x04, SI - LEAL 1(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm8B - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ $0x9e3779b1, SI - MOVQ DI, R10 - MOVQ DI, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ SI, R11 - SHRQ $0x38, R11 - MOVL (AX)(R10*4), SI - MOVL 4096(AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - MOVL DX, 4096(AX)(R11*4) - MOVQ (BX)(SI*1), R10 - MOVQ (BX)(R8*1), R11 - CMPQ R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm8B - CMPQ R11, DI - JNE no_short_found_encodeSnappyBetterBlockAsm8B - MOVL R8, SI - JMP candidate_match_encodeSnappyBetterBlockAsm8B - -no_short_found_encodeSnappyBetterBlockAsm8B: - CMPL R10, DI - JEQ candidate_match_encodeSnappyBetterBlockAsm8B - CMPL R11, DI - JEQ candidateS_match_encodeSnappyBetterBlockAsm8B - MOVL 20(SP), DX - JMP search_loop_encodeSnappyBetterBlockAsm8B - -candidateS_match_encodeSnappyBetterBlockAsm8B: - SHRQ $0x08, DI - MOVQ DI, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x36, R10 - MOVL (AX)(R10*4), SI - INCL DX - MOVL DX, (AX)(R10*4) - CMPL (BX)(SI*1), DI - JEQ candidate_match_encodeSnappyBetterBlockAsm8B - DECL DX - MOVL R8, SI - -candidate_match_encodeSnappyBetterBlockAsm8B: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm8B - -match_extend_back_loop_encodeSnappyBetterBlockAsm8B: - CMPL DX, DI - JBE match_extend_back_end_encodeSnappyBetterBlockAsm8B - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_encodeSnappyBetterBlockAsm8B - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_encodeSnappyBetterBlockAsm8B - JMP match_extend_back_loop_encodeSnappyBetterBlockAsm8B - -match_extend_back_end_encodeSnappyBetterBlockAsm8B: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_encodeSnappyBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_dst_size_check_encodeSnappyBetterBlockAsm8B: - MOVL DX, DI - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+32(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), R10 - - // matchLen - XORL R12, R12 - -matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B: - CMPL R8, $0x10 - JB matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B - MOVQ (R9)(R12*1), R11 - MOVQ 8(R9)(R12*1), R13 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B - XORQ 8(R10)(R12*1), R13 - JNZ matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B - LEAL -16(R8), R8 - LEAL 16(R12), R12 - JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B - -matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B - -matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: - CMPL R8, $0x08 - JB matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B - MOVQ (R9)(R12*1), R11 - XORQ (R10)(R12*1), R11 - JNZ matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B - LEAL -8(R8), R8 - LEAL 8(R12), R12 - JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B - -matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B - -matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: - CMPL R8, $0x04 - JB matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B - MOVL (R9)(R12*1), R11 - CMPL (R10)(R12*1), R11 - JNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B - LEAL -4(R8), R8 - LEAL 4(R12), R12 - -matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B: - CMPL R8, $0x01 - JE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B - JB match_nolit_end_encodeSnappyBetterBlockAsm8B - MOVW (R9)(R12*1), R11 - CMPW (R10)(R12*1), R11 - JNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B - LEAL 2(R12), R12 - SUBL $0x02, R8 - JZ match_nolit_end_encodeSnappyBetterBlockAsm8B - -matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B: - MOVB (R9)(R12*1), R11 - CMPB (R10)(R12*1), R11 - JNE match_nolit_end_encodeSnappyBetterBlockAsm8B - LEAL 1(R12), R12 - -match_nolit_end_encodeSnappyBetterBlockAsm8B: - MOVL DX, R8 - SUBL SI, R8 - - // Check if repeat - MOVL R8, 16(SP) - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R10 - SUBL SI, R9 - LEAL -1(R9), SI - CMPL SI, $0x3c - JB one_byte_match_emit_encodeSnappyBetterBlockAsm8B - CMPL SI, $0x00000100 - JB two_bytes_match_emit_encodeSnappyBetterBlockAsm8B - JB three_bytes_match_emit_encodeSnappyBetterBlockAsm8B - -three_bytes_match_emit_encodeSnappyBetterBlockAsm8B: - MOVB $0xf4, (CX) - MOVW SI, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B - -two_bytes_match_emit_encodeSnappyBetterBlockAsm8B: - MOVB $0xf0, (CX) - MOVB SI, 1(CX) - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_match_emit_encodeSnappyBetterBlockAsm8B - JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B - -one_byte_match_emit_encodeSnappyBetterBlockAsm8B: - SHLB $0x02, SI - MOVB SI, (CX) - ADDQ $0x01, CX - -memmove_match_emit_encodeSnappyBetterBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8: - MOVQ (R10), R11 - MOVQ R11, (CX) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: - MOVQ (R10), R11 - MOVQ -8(R10)(R9*1), R10 - MOVQ R11, (CX) - MOVQ R10, -8(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: - MOVOU (R10), X0 - MOVOU -16(R10)(R9*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(R9*1) - JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - -memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B: - MOVQ SI, CX - JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B - -memmove_long_match_emit_encodeSnappyBetterBlockAsm8B: - LEAQ (CX)(R9*1), SI - - // genMemMoveLong - MOVOU (R10), X0 - MOVOU 16(R10), X1 - MOVOU -32(R10)(R9*1), X2 - MOVOU -16(R10)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ CX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R14 - SUBQ R11, R14 - DECQ R13 - JA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(R10)(R14*1), R11 - LEAQ -32(CX)(R14*1), R15 - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R11 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(R10)(R14*1), X4 - MOVOU -16(R10)(R14*1), X5 - MOVOA X4, -32(CX)(R14*1) - MOVOA X5, -16(CX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(R9*1) - MOVOU X3, -16(CX)(R9*1) - MOVQ SI, CX - -emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B: - ADDL R12, DX - ADDL $0x04, R12 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B: - CMPL R12, $0x40 - JBE two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B - MOVB $0xee, (CX) - MOVW R8, 1(CX) - LEAL -60(R12), R12 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B - -two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B: - MOVL R12, SI - SHLL $0x02, SI - CMPL R12, $0x0c - JAE emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B - LEAL -15(SI), SI - MOVB R8, 1(CX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, SI - MOVB SI, (CX) - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B - -emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B: - LEAL -2(SI), SI - MOVB SI, (CX) - MOVW R8, 1(CX) - ADDQ $0x03, CX - -match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B: - CMPL DX, 8(SP) - JAE emit_remainder_encodeSnappyBetterBlockAsm8B - CMPQ CX, (SP) - JB match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B: - MOVQ $0x0000cf1bbcdcbf9b, SI - MOVQ $0x9e3779b1, R8 - LEAQ 1(DI), DI - LEAQ -2(DX), R9 - MOVQ (BX)(DI*1), R10 - MOVQ 1(BX)(DI*1), R11 - MOVQ (BX)(R9*1), R12 - MOVQ 1(BX)(R9*1), R13 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x36, R10 - SHLQ $0x20, R11 - IMULQ R8, R11 - SHRQ $0x38, R11 - SHLQ $0x10, R12 - IMULQ SI, R12 - SHRQ $0x36, R12 - SHLQ $0x20, R13 - IMULQ R8, R13 - SHRQ $0x38, R13 - LEAQ 1(DI), R8 - LEAQ 1(R9), R14 - MOVL DI, (AX)(R10*4) - MOVL R9, (AX)(R12*4) - MOVL R8, 4096(AX)(R11*4) - MOVL R14, 4096(AX)(R13*4) - LEAQ 1(R9)(DI*1), R8 - SHRQ $0x01, R8 - ADDQ $0x01, DI - SUBQ $0x01, R9 - -index_loop_encodeSnappyBetterBlockAsm8B: - CMPQ R8, R9 - JAE search_loop_encodeSnappyBetterBlockAsm8B - MOVQ (BX)(DI*1), R10 - MOVQ (BX)(R8*1), R11 - SHLQ $0x10, R10 - IMULQ SI, R10 - SHRQ $0x36, R10 - SHLQ $0x10, R11 - IMULQ SI, R11 - SHRQ $0x36, R11 - MOVL DI, (AX)(R10*4) - MOVL R8, (AX)(R11*4) - ADDQ $0x02, DI - ADDQ $0x02, R8 - JMP index_loop_encodeSnappyBetterBlockAsm8B - -emit_remainder_encodeSnappyBetterBlockAsm8B: - MOVQ src_len+32(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_encodeSnappyBetterBlockAsm8B - MOVQ $0x00000000, ret+56(FP) - RET - -emit_remainder_ok_encodeSnappyBetterBlockAsm8B: - MOVQ src_len+32(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), DX - CMPL DX, $0x3c - JB one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B - CMPL DX, $0x00000100 - JB two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B - JB three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B - -three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: - MOVB $0xf4, (CX) - MOVW DX, 1(CX) - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B - -two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: - MOVB $0xf0, (CX) - MOVB DL, 1(CX) - ADDQ $0x02, CX - CMPL DX, $0x40 - JB memmove_emit_remainder_encodeSnappyBetterBlockAsm8B - JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B - -one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B: - SHLB $0x02, DL - MOVB DL, (CX) - ADDQ $0x01, CX - -memmove_emit_remainder_encodeSnappyBetterBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveShort - CMPQ BX, $0x03 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2 - JE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3 - CMPQ BX, $0x08 - JB emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7 - CMPQ BX, $0x10 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 - CMPQ BX, $0x20 - JBE emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 - JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2: - MOVB (AX), SI - MOVB -1(AX)(BX*1), AL - MOVB SI, (CX) - MOVB AL, -1(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3: - MOVW (AX), SI - MOVB 2(AX), AL - MOVW SI, (CX) - MOVB AL, 2(CX) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7: - MOVL (AX), SI - MOVL -4(AX)(BX*1), AX - MOVL SI, (CX) - MOVL AX, -4(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: - MOVQ (AX), SI - MOVQ -8(AX)(BX*1), AX - MOVQ SI, (CX) - MOVQ AX, -8(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: - MOVOU (AX), X0 - MOVOU -16(AX)(BX*1), X1 - MOVOU X0, (CX) - MOVOU X1, -16(CX)(BX*1) - JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B - -emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - -memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B: - MOVQ DX, CX - JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B - -memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B: - LEAQ (CX)(SI*1), DX - MOVL SI, BX - - // genMemMoveLong - MOVOU (AX), X0 - MOVOU 16(AX), X1 - MOVOU -32(AX)(BX*1), X2 - MOVOU -16(AX)(BX*1), X3 - MOVQ BX, DI - SHRQ $0x05, DI - MOVQ CX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32 - LEAQ -32(AX)(R8*1), SI - LEAQ -32(CX)(R8*1), R9 - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back - -emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32: - MOVOU -32(AX)(R8*1), X4 - MOVOU -16(AX)(R8*1), X5 - MOVOA X4, -32(CX)(R8*1) - MOVOA X5, -16(CX)(R8*1) - ADDQ $0x20, R8 - CMPQ BX, R8 - JAE emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32 - MOVOU X0, (CX) - MOVOU X1, 16(CX) - MOVOU X2, -32(CX)(BX*1) - MOVOU X3, -16(CX)(BX*1) - MOVQ DX, CX - -emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B: - MOVQ dst_base+0(FP), AX - SUBQ AX, CX - MOVQ CX, ret+56(FP) - RET - -// func calcBlockSize(src []byte, tmp *[32768]byte) int -// Requires: BMI, SSE2 -TEXT ·calcBlockSize(SB), $24-40 - MOVQ tmp+24(FP), AX - XORQ CX, CX - MOVQ $0x00000100, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_calcBlockSize: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_calcBlockSize - MOVL $0x00000000, 12(SP) - MOVQ src_len+8(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+0(FP), BX - -search_loop_calcBlockSize: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x05, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_calcBlockSize - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x33, R10 - SHLQ $0x10, R11 - IMULQ R9, R11 - SHRQ $0x33, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x10, R10 - IMULQ R9, R10 - SHRQ $0x33, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_calcBlockSize - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_calcBlockSize - -repeat_extend_back_loop_calcBlockSize: - CMPL DI, SI - JBE repeat_extend_back_end_calcBlockSize - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_calcBlockSize - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_calcBlockSize - -repeat_extend_back_end_calcBlockSize: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 5(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_calcBlockSize - MOVQ $0x00000000, ret+32(FP) - RET - -repeat_dst_size_check_calcBlockSize: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_calcBlockSize - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_calcBlockSize - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_calcBlockSize - CMPL SI, $0x00010000 - JB three_bytes_repeat_emit_calcBlockSize - CMPL SI, $0x01000000 - JB four_bytes_repeat_emit_calcBlockSize - ADDQ $0x05, CX - JMP memmove_long_repeat_emit_calcBlockSize - -four_bytes_repeat_emit_calcBlockSize: - ADDQ $0x04, CX - JMP memmove_long_repeat_emit_calcBlockSize - -three_bytes_repeat_emit_calcBlockSize: - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_calcBlockSize - -two_bytes_repeat_emit_calcBlockSize: - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_calcBlockSize - JMP memmove_long_repeat_emit_calcBlockSize - -one_byte_repeat_emit_calcBlockSize: - ADDQ $0x01, CX - -memmove_repeat_emit_calcBlockSize: - LEAQ (CX)(R8*1), CX - JMP emit_literal_done_repeat_emit_calcBlockSize - -memmove_long_repeat_emit_calcBlockSize: - LEAQ (CX)(R8*1), CX - -emit_literal_done_repeat_emit_calcBlockSize: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+8(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_calcBlockSize: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_calcBlockSize - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_calcBlockSize - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_calcBlockSize - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_calcBlockSize - -matchlen_bsf_16repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSize - -matchlen_match8_repeat_extend_calcBlockSize: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_calcBlockSize - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_calcBlockSize - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_calcBlockSize - -matchlen_bsf_8_repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSize - -matchlen_match4_repeat_extend_calcBlockSize: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_calcBlockSize - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_calcBlockSize - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_calcBlockSize: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_calcBlockSize - JB repeat_extend_forward_end_calcBlockSize - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_calcBlockSize - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_calcBlockSize - -matchlen_match1_repeat_extend_calcBlockSize: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_calcBlockSize - LEAL 1(R11), R11 - -repeat_extend_forward_end_calcBlockSize: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy - CMPL DI, $0x00010000 - JB two_byte_offset_repeat_as_copy_calcBlockSize - -four_bytes_loop_back_repeat_as_copy_calcBlockSize: - CMPL SI, $0x40 - JBE four_bytes_remain_repeat_as_copy_calcBlockSize - LEAL -64(SI), SI - ADDQ $0x05, CX - CMPL SI, $0x04 - JB four_bytes_remain_repeat_as_copy_calcBlockSize - JMP four_bytes_loop_back_repeat_as_copy_calcBlockSize - -four_bytes_remain_repeat_as_copy_calcBlockSize: - TESTL SI, SI - JZ repeat_end_emit_calcBlockSize - XORL SI, SI - ADDQ $0x05, CX - JMP repeat_end_emit_calcBlockSize - -two_byte_offset_repeat_as_copy_calcBlockSize: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_calcBlockSize - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_calcBlockSize - -two_byte_offset_short_repeat_as_copy_calcBlockSize: - MOVL SI, R8 - SHLL $0x02, R8 - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_calcBlockSize - CMPL DI, $0x00000800 - JAE emit_copy_three_repeat_as_copy_calcBlockSize - ADDQ $0x02, CX - JMP repeat_end_emit_calcBlockSize - -emit_copy_three_repeat_as_copy_calcBlockSize: - ADDQ $0x03, CX - -repeat_end_emit_calcBlockSize: - MOVL DX, 12(SP) - JMP search_loop_calcBlockSize - -no_repeat_found_calcBlockSize: - CMPL (BX)(SI*1), DI - JEQ candidate_match_calcBlockSize - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_calcBlockSize - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_calcBlockSize - MOVL 20(SP), DX - JMP search_loop_calcBlockSize - -candidate3_match_calcBlockSize: - ADDL $0x02, DX - JMP candidate_match_calcBlockSize - -candidate2_match_calcBlockSize: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_calcBlockSize: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_calcBlockSize - -match_extend_back_loop_calcBlockSize: - CMPL DX, DI - JBE match_extend_back_end_calcBlockSize - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_calcBlockSize - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_calcBlockSize - JMP match_extend_back_loop_calcBlockSize - -match_extend_back_end_calcBlockSize: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 5(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_calcBlockSize - MOVQ $0x00000000, ret+32(FP) - RET - -match_dst_size_check_calcBlockSize: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_calcBlockSize - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), DI - CMPL DI, $0x3c - JB one_byte_match_emit_calcBlockSize - CMPL DI, $0x00000100 - JB two_bytes_match_emit_calcBlockSize - CMPL DI, $0x00010000 - JB three_bytes_match_emit_calcBlockSize - CMPL DI, $0x01000000 - JB four_bytes_match_emit_calcBlockSize - ADDQ $0x05, CX - JMP memmove_long_match_emit_calcBlockSize - -four_bytes_match_emit_calcBlockSize: - ADDQ $0x04, CX - JMP memmove_long_match_emit_calcBlockSize - -three_bytes_match_emit_calcBlockSize: - ADDQ $0x03, CX - JMP memmove_long_match_emit_calcBlockSize - -two_bytes_match_emit_calcBlockSize: - ADDQ $0x02, CX - CMPL DI, $0x40 - JB memmove_match_emit_calcBlockSize - JMP memmove_long_match_emit_calcBlockSize - -one_byte_match_emit_calcBlockSize: - ADDQ $0x01, CX - -memmove_match_emit_calcBlockSize: - LEAQ (CX)(R9*1), CX - JMP emit_literal_done_match_emit_calcBlockSize - -memmove_long_match_emit_calcBlockSize: - LEAQ (CX)(R9*1), CX - -emit_literal_done_match_emit_calcBlockSize: -match_nolit_loop_calcBlockSize: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+8(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_calcBlockSize: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_calcBlockSize - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_calcBlockSize - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_calcBlockSize - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_calcBlockSize - -matchlen_bsf_16match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSize - -matchlen_match8_match_nolit_calcBlockSize: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_calcBlockSize - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_calcBlockSize - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_calcBlockSize - -matchlen_bsf_8_match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSize - -matchlen_match4_match_nolit_calcBlockSize: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_calcBlockSize - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_calcBlockSize - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_calcBlockSize: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_calcBlockSize - JB match_nolit_end_calcBlockSize - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_calcBlockSize - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_calcBlockSize - -matchlen_match1_match_nolit_calcBlockSize: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_calcBlockSize - LEAL 1(R10), R10 - -match_nolit_end_calcBlockSize: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy - CMPL SI, $0x00010000 - JB two_byte_offset_match_nolit_calcBlockSize - -four_bytes_loop_back_match_nolit_calcBlockSize: - CMPL R10, $0x40 - JBE four_bytes_remain_match_nolit_calcBlockSize - LEAL -64(R10), R10 - ADDQ $0x05, CX - CMPL R10, $0x04 - JB four_bytes_remain_match_nolit_calcBlockSize - JMP four_bytes_loop_back_match_nolit_calcBlockSize - -four_bytes_remain_match_nolit_calcBlockSize: - TESTL R10, R10 - JZ match_nolit_emitcopy_end_calcBlockSize - XORL SI, SI - ADDQ $0x05, CX - JMP match_nolit_emitcopy_end_calcBlockSize - -two_byte_offset_match_nolit_calcBlockSize: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_calcBlockSize - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_calcBlockSize - -two_byte_offset_short_match_nolit_calcBlockSize: - MOVL R10, DI - SHLL $0x02, DI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_calcBlockSize - CMPL SI, $0x00000800 - JAE emit_copy_three_match_nolit_calcBlockSize - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_calcBlockSize - -emit_copy_three_match_nolit_calcBlockSize: - ADDQ $0x03, CX - -match_nolit_emitcopy_end_calcBlockSize: - CMPL DX, 8(SP) - JAE emit_remainder_calcBlockSize - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_calcBlockSize - MOVQ $0x00000000, ret+32(FP) - RET - -match_nolit_dst_ok_calcBlockSize: - MOVQ $0x0000cf1bbcdcbf9b, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x10, R8 - IMULQ R9, R8 - SHRQ $0x33, R8 - SHLQ $0x10, SI - IMULQ R9, SI - SHRQ $0x33, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_calcBlockSize - INCL DX - JMP search_loop_calcBlockSize - -emit_remainder_calcBlockSize: - MOVQ src_len+8(FP), AX - SUBL 12(SP), AX - LEAQ 5(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_calcBlockSize - MOVQ $0x00000000, ret+32(FP) - RET - -emit_remainder_ok_calcBlockSize: - MOVQ src_len+8(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_calcBlockSize - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), AX - CMPL AX, $0x3c - JB one_byte_emit_remainder_calcBlockSize - CMPL AX, $0x00000100 - JB two_bytes_emit_remainder_calcBlockSize - CMPL AX, $0x00010000 - JB three_bytes_emit_remainder_calcBlockSize - CMPL AX, $0x01000000 - JB four_bytes_emit_remainder_calcBlockSize - ADDQ $0x05, CX - JMP memmove_long_emit_remainder_calcBlockSize - -four_bytes_emit_remainder_calcBlockSize: - ADDQ $0x04, CX - JMP memmove_long_emit_remainder_calcBlockSize - -three_bytes_emit_remainder_calcBlockSize: - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_calcBlockSize - -two_bytes_emit_remainder_calcBlockSize: - ADDQ $0x02, CX - CMPL AX, $0x40 - JB memmove_emit_remainder_calcBlockSize - JMP memmove_long_emit_remainder_calcBlockSize - -one_byte_emit_remainder_calcBlockSize: - ADDQ $0x01, CX - -memmove_emit_remainder_calcBlockSize: - LEAQ (CX)(SI*1), AX - MOVQ AX, CX - JMP emit_literal_done_emit_remainder_calcBlockSize - -memmove_long_emit_remainder_calcBlockSize: - LEAQ (CX)(SI*1), AX - MOVQ AX, CX - -emit_literal_done_emit_remainder_calcBlockSize: - MOVQ CX, ret+32(FP) - RET - -// func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int -// Requires: BMI, SSE2 -TEXT ·calcBlockSizeSmall(SB), $24-40 - MOVQ tmp+24(FP), AX - XORQ CX, CX - MOVQ $0x00000010, DX - MOVQ AX, BX - PXOR X0, X0 - -zero_loop_calcBlockSizeSmall: - MOVOU X0, (BX) - MOVOU X0, 16(BX) - MOVOU X0, 32(BX) - MOVOU X0, 48(BX) - MOVOU X0, 64(BX) - MOVOU X0, 80(BX) - MOVOU X0, 96(BX) - MOVOU X0, 112(BX) - ADDQ $0x80, BX - DECQ DX - JNZ zero_loop_calcBlockSizeSmall - MOVL $0x00000000, 12(SP) - MOVQ src_len+8(FP), DX - LEAQ -9(DX), BX - LEAQ -8(DX), SI - MOVL SI, 8(SP) - SHRQ $0x05, DX - SUBL DX, BX - LEAQ (CX)(BX*1), BX - MOVQ BX, (SP) - MOVL $0x00000001, DX - MOVL DX, 16(SP) - MOVQ src_base+0(FP), BX - -search_loop_calcBlockSizeSmall: - MOVL DX, SI - SUBL 12(SP), SI - SHRL $0x04, SI - LEAL 4(DX)(SI*1), SI - CMPL SI, 8(SP) - JAE emit_remainder_calcBlockSizeSmall - MOVQ (BX)(DX*1), DI - MOVL SI, 20(SP) - MOVQ $0x9e3779b1, R9 - MOVQ DI, R10 - MOVQ DI, R11 - SHRQ $0x08, R11 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x37, R10 - SHLQ $0x20, R11 - IMULQ R9, R11 - SHRQ $0x37, R11 - MOVL (AX)(R10*4), SI - MOVL (AX)(R11*4), R8 - MOVL DX, (AX)(R10*4) - LEAL 1(DX), R10 - MOVL R10, (AX)(R11*4) - MOVQ DI, R10 - SHRQ $0x10, R10 - SHLQ $0x20, R10 - IMULQ R9, R10 - SHRQ $0x37, R10 - MOVL DX, R9 - SUBL 16(SP), R9 - MOVL 1(BX)(R9*1), R11 - MOVQ DI, R9 - SHRQ $0x08, R9 - CMPL R9, R11 - JNE no_repeat_found_calcBlockSizeSmall - LEAL 1(DX), DI - MOVL 12(SP), SI - MOVL DI, R8 - SUBL 16(SP), R8 - JZ repeat_extend_back_end_calcBlockSizeSmall - -repeat_extend_back_loop_calcBlockSizeSmall: - CMPL DI, SI - JBE repeat_extend_back_end_calcBlockSizeSmall - MOVB -1(BX)(R8*1), R9 - MOVB -1(BX)(DI*1), R10 - CMPB R9, R10 - JNE repeat_extend_back_end_calcBlockSizeSmall - LEAL -1(DI), DI - DECL R8 - JNZ repeat_extend_back_loop_calcBlockSizeSmall - -repeat_extend_back_end_calcBlockSizeSmall: - MOVL DI, SI - SUBL 12(SP), SI - LEAQ 3(CX)(SI*1), SI - CMPQ SI, (SP) - JB repeat_dst_size_check_calcBlockSizeSmall - MOVQ $0x00000000, ret+32(FP) - RET - -repeat_dst_size_check_calcBlockSizeSmall: - MOVL 12(SP), SI - CMPL SI, DI - JEQ emit_literal_done_repeat_emit_calcBlockSizeSmall - MOVL DI, R8 - MOVL DI, 12(SP) - LEAQ (BX)(SI*1), R9 - SUBL SI, R8 - LEAL -1(R8), SI - CMPL SI, $0x3c - JB one_byte_repeat_emit_calcBlockSizeSmall - CMPL SI, $0x00000100 - JB two_bytes_repeat_emit_calcBlockSizeSmall - JB three_bytes_repeat_emit_calcBlockSizeSmall - -three_bytes_repeat_emit_calcBlockSizeSmall: - ADDQ $0x03, CX - JMP memmove_long_repeat_emit_calcBlockSizeSmall - -two_bytes_repeat_emit_calcBlockSizeSmall: - ADDQ $0x02, CX - CMPL SI, $0x40 - JB memmove_repeat_emit_calcBlockSizeSmall - JMP memmove_long_repeat_emit_calcBlockSizeSmall - -one_byte_repeat_emit_calcBlockSizeSmall: - ADDQ $0x01, CX - -memmove_repeat_emit_calcBlockSizeSmall: - LEAQ (CX)(R8*1), CX - JMP emit_literal_done_repeat_emit_calcBlockSizeSmall - -memmove_long_repeat_emit_calcBlockSizeSmall: - LEAQ (CX)(R8*1), CX - -emit_literal_done_repeat_emit_calcBlockSizeSmall: - ADDL $0x05, DX - MOVL DX, SI - SUBL 16(SP), SI - MOVQ src_len+8(FP), R8 - SUBL DX, R8 - LEAQ (BX)(DX*1), R9 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R11, R11 - -matchlen_loopback_16_repeat_extend_calcBlockSizeSmall: - CMPL R8, $0x10 - JB matchlen_match8_repeat_extend_calcBlockSizeSmall - MOVQ (R9)(R11*1), R10 - MOVQ 8(R9)(R11*1), R12 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_calcBlockSizeSmall - XORQ 8(SI)(R11*1), R12 - JNZ matchlen_bsf_16repeat_extend_calcBlockSizeSmall - LEAL -16(R8), R8 - LEAL 16(R11), R11 - JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall - -matchlen_bsf_16repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 - TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall - -matchlen_match8_repeat_extend_calcBlockSizeSmall: - CMPL R8, $0x08 - JB matchlen_match4_repeat_extend_calcBlockSizeSmall - MOVQ (R9)(R11*1), R10 - XORQ (SI)(R11*1), R10 - JNZ matchlen_bsf_8_repeat_extend_calcBlockSizeSmall - LEAL -8(R8), R8 - LEAL 8(R11), R11 - JMP matchlen_match4_repeat_extend_calcBlockSizeSmall - -matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 - TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall - -matchlen_match4_repeat_extend_calcBlockSizeSmall: - CMPL R8, $0x04 - JB matchlen_match2_repeat_extend_calcBlockSizeSmall - MOVL (R9)(R11*1), R10 - CMPL (SI)(R11*1), R10 - JNE matchlen_match2_repeat_extend_calcBlockSizeSmall - LEAL -4(R8), R8 - LEAL 4(R11), R11 - -matchlen_match2_repeat_extend_calcBlockSizeSmall: - CMPL R8, $0x01 - JE matchlen_match1_repeat_extend_calcBlockSizeSmall - JB repeat_extend_forward_end_calcBlockSizeSmall - MOVW (R9)(R11*1), R10 - CMPW (SI)(R11*1), R10 - JNE matchlen_match1_repeat_extend_calcBlockSizeSmall - LEAL 2(R11), R11 - SUBL $0x02, R8 - JZ repeat_extend_forward_end_calcBlockSizeSmall - -matchlen_match1_repeat_extend_calcBlockSizeSmall: - MOVB (R9)(R11*1), R10 - CMPB (SI)(R11*1), R10 - JNE repeat_extend_forward_end_calcBlockSizeSmall - LEAL 1(R11), R11 - -repeat_extend_forward_end_calcBlockSizeSmall: - ADDL R11, DX - MOVL DX, SI - SUBL DI, SI - MOVL 16(SP), DI - - // emitCopy -two_byte_offset_repeat_as_copy_calcBlockSizeSmall: - CMPL SI, $0x40 - JBE two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall - LEAL -60(SI), SI - ADDQ $0x03, CX - JMP two_byte_offset_repeat_as_copy_calcBlockSizeSmall - -two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall: - MOVL SI, DI - SHLL $0x02, DI - CMPL SI, $0x0c - JAE emit_copy_three_repeat_as_copy_calcBlockSizeSmall - ADDQ $0x02, CX - JMP repeat_end_emit_calcBlockSizeSmall - -emit_copy_three_repeat_as_copy_calcBlockSizeSmall: - ADDQ $0x03, CX - -repeat_end_emit_calcBlockSizeSmall: - MOVL DX, 12(SP) - JMP search_loop_calcBlockSizeSmall - -no_repeat_found_calcBlockSizeSmall: - CMPL (BX)(SI*1), DI - JEQ candidate_match_calcBlockSizeSmall - SHRQ $0x08, DI - MOVL (AX)(R10*4), SI - LEAL 2(DX), R9 - CMPL (BX)(R8*1), DI - JEQ candidate2_match_calcBlockSizeSmall - MOVL R9, (AX)(R10*4) - SHRQ $0x08, DI - CMPL (BX)(SI*1), DI - JEQ candidate3_match_calcBlockSizeSmall - MOVL 20(SP), DX - JMP search_loop_calcBlockSizeSmall - -candidate3_match_calcBlockSizeSmall: - ADDL $0x02, DX - JMP candidate_match_calcBlockSizeSmall - -candidate2_match_calcBlockSizeSmall: - MOVL R9, (AX)(R10*4) - INCL DX - MOVL R8, SI - -candidate_match_calcBlockSizeSmall: - MOVL 12(SP), DI - TESTL SI, SI - JZ match_extend_back_end_calcBlockSizeSmall - -match_extend_back_loop_calcBlockSizeSmall: - CMPL DX, DI - JBE match_extend_back_end_calcBlockSizeSmall - MOVB -1(BX)(SI*1), R8 - MOVB -1(BX)(DX*1), R9 - CMPB R8, R9 - JNE match_extend_back_end_calcBlockSizeSmall - LEAL -1(DX), DX - DECL SI - JZ match_extend_back_end_calcBlockSizeSmall - JMP match_extend_back_loop_calcBlockSizeSmall - -match_extend_back_end_calcBlockSizeSmall: - MOVL DX, DI - SUBL 12(SP), DI - LEAQ 3(CX)(DI*1), DI - CMPQ DI, (SP) - JB match_dst_size_check_calcBlockSizeSmall - MOVQ $0x00000000, ret+32(FP) - RET - -match_dst_size_check_calcBlockSizeSmall: - MOVL DX, DI - MOVL 12(SP), R8 - CMPL R8, DI - JEQ emit_literal_done_match_emit_calcBlockSizeSmall - MOVL DI, R9 - MOVL DI, 12(SP) - LEAQ (BX)(R8*1), DI - SUBL R8, R9 - LEAL -1(R9), DI - CMPL DI, $0x3c - JB one_byte_match_emit_calcBlockSizeSmall - CMPL DI, $0x00000100 - JB two_bytes_match_emit_calcBlockSizeSmall - JB three_bytes_match_emit_calcBlockSizeSmall - -three_bytes_match_emit_calcBlockSizeSmall: - ADDQ $0x03, CX - JMP memmove_long_match_emit_calcBlockSizeSmall - -two_bytes_match_emit_calcBlockSizeSmall: - ADDQ $0x02, CX - CMPL DI, $0x40 - JB memmove_match_emit_calcBlockSizeSmall - JMP memmove_long_match_emit_calcBlockSizeSmall - -one_byte_match_emit_calcBlockSizeSmall: - ADDQ $0x01, CX - -memmove_match_emit_calcBlockSizeSmall: - LEAQ (CX)(R9*1), CX - JMP emit_literal_done_match_emit_calcBlockSizeSmall - -memmove_long_match_emit_calcBlockSizeSmall: - LEAQ (CX)(R9*1), CX - -emit_literal_done_match_emit_calcBlockSizeSmall: -match_nolit_loop_calcBlockSizeSmall: - MOVL DX, DI - SUBL SI, DI - MOVL DI, 16(SP) - ADDL $0x04, DX - ADDL $0x04, SI - MOVQ src_len+8(FP), DI - SUBL DX, DI - LEAQ (BX)(DX*1), R8 - LEAQ (BX)(SI*1), SI - - // matchLen - XORL R10, R10 - -matchlen_loopback_16_match_nolit_calcBlockSizeSmall: - CMPL DI, $0x10 - JB matchlen_match8_match_nolit_calcBlockSizeSmall - MOVQ (R8)(R10*1), R9 - MOVQ 8(R8)(R10*1), R11 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_calcBlockSizeSmall - XORQ 8(SI)(R10*1), R11 - JNZ matchlen_bsf_16match_nolit_calcBlockSizeSmall - LEAL -16(DI), DI - LEAL 16(R10), R10 - JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall - -matchlen_bsf_16match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 - TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSizeSmall - -matchlen_match8_match_nolit_calcBlockSizeSmall: - CMPL DI, $0x08 - JB matchlen_match4_match_nolit_calcBlockSizeSmall - MOVQ (R8)(R10*1), R9 - XORQ (SI)(R10*1), R9 - JNZ matchlen_bsf_8_match_nolit_calcBlockSizeSmall - LEAL -8(DI), DI - LEAL 8(R10), R10 - JMP matchlen_match4_match_nolit_calcBlockSizeSmall - -matchlen_bsf_8_match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 - TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSizeSmall - -matchlen_match4_match_nolit_calcBlockSizeSmall: - CMPL DI, $0x04 - JB matchlen_match2_match_nolit_calcBlockSizeSmall - MOVL (R8)(R10*1), R9 - CMPL (SI)(R10*1), R9 - JNE matchlen_match2_match_nolit_calcBlockSizeSmall - LEAL -4(DI), DI - LEAL 4(R10), R10 - -matchlen_match2_match_nolit_calcBlockSizeSmall: - CMPL DI, $0x01 - JE matchlen_match1_match_nolit_calcBlockSizeSmall - JB match_nolit_end_calcBlockSizeSmall - MOVW (R8)(R10*1), R9 - CMPW (SI)(R10*1), R9 - JNE matchlen_match1_match_nolit_calcBlockSizeSmall - LEAL 2(R10), R10 - SUBL $0x02, DI - JZ match_nolit_end_calcBlockSizeSmall - -matchlen_match1_match_nolit_calcBlockSizeSmall: - MOVB (R8)(R10*1), R9 - CMPB (SI)(R10*1), R9 - JNE match_nolit_end_calcBlockSizeSmall - LEAL 1(R10), R10 - -match_nolit_end_calcBlockSizeSmall: - ADDL R10, DX - MOVL 16(SP), SI - ADDL $0x04, R10 - MOVL DX, 12(SP) - - // emitCopy -two_byte_offset_match_nolit_calcBlockSizeSmall: - CMPL R10, $0x40 - JBE two_byte_offset_short_match_nolit_calcBlockSizeSmall - LEAL -60(R10), R10 - ADDQ $0x03, CX - JMP two_byte_offset_match_nolit_calcBlockSizeSmall - -two_byte_offset_short_match_nolit_calcBlockSizeSmall: - MOVL R10, SI - SHLL $0x02, SI - CMPL R10, $0x0c - JAE emit_copy_three_match_nolit_calcBlockSizeSmall - ADDQ $0x02, CX - JMP match_nolit_emitcopy_end_calcBlockSizeSmall - -emit_copy_three_match_nolit_calcBlockSizeSmall: - ADDQ $0x03, CX - -match_nolit_emitcopy_end_calcBlockSizeSmall: - CMPL DX, 8(SP) - JAE emit_remainder_calcBlockSizeSmall - MOVQ -2(BX)(DX*1), DI - CMPQ CX, (SP) - JB match_nolit_dst_ok_calcBlockSizeSmall - MOVQ $0x00000000, ret+32(FP) - RET - -match_nolit_dst_ok_calcBlockSizeSmall: - MOVQ $0x9e3779b1, R9 - MOVQ DI, R8 - SHRQ $0x10, DI - MOVQ DI, SI - SHLQ $0x20, R8 - IMULQ R9, R8 - SHRQ $0x37, R8 - SHLQ $0x20, SI - IMULQ R9, SI - SHRQ $0x37, SI - LEAL -2(DX), R9 - LEAQ (AX)(SI*4), R10 - MOVL (R10), SI - MOVL R9, (AX)(R8*4) - MOVL DX, (R10) - CMPL (BX)(SI*1), DI - JEQ match_nolit_loop_calcBlockSizeSmall - INCL DX - JMP search_loop_calcBlockSizeSmall - -emit_remainder_calcBlockSizeSmall: - MOVQ src_len+8(FP), AX - SUBL 12(SP), AX - LEAQ 3(CX)(AX*1), AX - CMPQ AX, (SP) - JB emit_remainder_ok_calcBlockSizeSmall - MOVQ $0x00000000, ret+32(FP) - RET - -emit_remainder_ok_calcBlockSizeSmall: - MOVQ src_len+8(FP), AX - MOVL 12(SP), DX - CMPL DX, AX - JEQ emit_literal_done_emit_remainder_calcBlockSizeSmall - MOVL AX, SI - MOVL AX, 12(SP) - LEAQ (BX)(DX*1), AX - SUBL DX, SI - LEAL -1(SI), AX - CMPL AX, $0x3c - JB one_byte_emit_remainder_calcBlockSizeSmall - CMPL AX, $0x00000100 - JB two_bytes_emit_remainder_calcBlockSizeSmall - JB three_bytes_emit_remainder_calcBlockSizeSmall - -three_bytes_emit_remainder_calcBlockSizeSmall: - ADDQ $0x03, CX - JMP memmove_long_emit_remainder_calcBlockSizeSmall - -two_bytes_emit_remainder_calcBlockSizeSmall: - ADDQ $0x02, CX - CMPL AX, $0x40 - JB memmove_emit_remainder_calcBlockSizeSmall - JMP memmove_long_emit_remainder_calcBlockSizeSmall - -one_byte_emit_remainder_calcBlockSizeSmall: - ADDQ $0x01, CX - -memmove_emit_remainder_calcBlockSizeSmall: - LEAQ (CX)(SI*1), AX - MOVQ AX, CX - JMP emit_literal_done_emit_remainder_calcBlockSizeSmall - -memmove_long_emit_remainder_calcBlockSizeSmall: - LEAQ (CX)(SI*1), AX - MOVQ AX, CX - -emit_literal_done_emit_remainder_calcBlockSizeSmall: - MOVQ CX, ret+32(FP) - RET - -// func emitLiteral(dst []byte, lit []byte) int -// Requires: SSE2 -TEXT ·emitLiteral(SB), NOSPLIT, $0-56 - MOVQ lit_len+32(FP), DX - MOVQ dst_base+0(FP), AX - MOVQ lit_base+24(FP), CX - TESTQ DX, DX - JZ emit_literal_end_standalone_skip - MOVL DX, BX - LEAL -1(DX), SI - CMPL SI, $0x3c - JB one_byte_standalone - CMPL SI, $0x00000100 - JB two_bytes_standalone - CMPL SI, $0x00010000 - JB three_bytes_standalone - CMPL SI, $0x01000000 - JB four_bytes_standalone - MOVB $0xfc, (AX) - MOVL SI, 1(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP memmove_long_standalone - -four_bytes_standalone: - MOVL SI, DI - SHRL $0x10, DI - MOVB $0xf8, (AX) - MOVW SI, 1(AX) - MOVB DI, 3(AX) - ADDQ $0x04, BX - ADDQ $0x04, AX - JMP memmove_long_standalone - -three_bytes_standalone: - MOVB $0xf4, (AX) - MOVW SI, 1(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - JMP memmove_long_standalone - -two_bytes_standalone: - MOVB $0xf0, (AX) - MOVB SI, 1(AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - CMPL SI, $0x40 - JB memmove_standalone - JMP memmove_long_standalone - -one_byte_standalone: - SHLB $0x02, SI - MOVB SI, (AX) - ADDQ $0x01, BX - ADDQ $0x01, AX - -memmove_standalone: - // genMemMoveShort - CMPQ DX, $0x03 - JB emit_lit_memmove_standalone_memmove_move_1or2 - JE emit_lit_memmove_standalone_memmove_move_3 - CMPQ DX, $0x08 - JB emit_lit_memmove_standalone_memmove_move_4through7 - CMPQ DX, $0x10 - JBE emit_lit_memmove_standalone_memmove_move_8through16 - CMPQ DX, $0x20 - JBE emit_lit_memmove_standalone_memmove_move_17through32 - JMP emit_lit_memmove_standalone_memmove_move_33through64 - -emit_lit_memmove_standalone_memmove_move_1or2: - MOVB (CX), SI - MOVB -1(CX)(DX*1), CL - MOVB SI, (AX) - MOVB CL, -1(AX)(DX*1) - JMP emit_literal_end_standalone - -emit_lit_memmove_standalone_memmove_move_3: - MOVW (CX), SI - MOVB 2(CX), CL - MOVW SI, (AX) - MOVB CL, 2(AX) - JMP emit_literal_end_standalone - -emit_lit_memmove_standalone_memmove_move_4through7: - MOVL (CX), SI - MOVL -4(CX)(DX*1), CX - MOVL SI, (AX) - MOVL CX, -4(AX)(DX*1) - JMP emit_literal_end_standalone - -emit_lit_memmove_standalone_memmove_move_8through16: - MOVQ (CX), SI - MOVQ -8(CX)(DX*1), CX - MOVQ SI, (AX) - MOVQ CX, -8(AX)(DX*1) - JMP emit_literal_end_standalone - -emit_lit_memmove_standalone_memmove_move_17through32: - MOVOU (CX), X0 - MOVOU -16(CX)(DX*1), X1 - MOVOU X0, (AX) - MOVOU X1, -16(AX)(DX*1) - JMP emit_literal_end_standalone - -emit_lit_memmove_standalone_memmove_move_33through64: - MOVOU (CX), X0 - MOVOU 16(CX), X1 - MOVOU -32(CX)(DX*1), X2 - MOVOU -16(CX)(DX*1), X3 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(DX*1) - MOVOU X3, -16(AX)(DX*1) - JMP emit_literal_end_standalone - JMP emit_literal_end_standalone - -memmove_long_standalone: - // genMemMoveLong - MOVOU (CX), X0 - MOVOU 16(CX), X1 - MOVOU -32(CX)(DX*1), X2 - MOVOU -16(CX)(DX*1), X3 - MOVQ DX, DI - SHRQ $0x05, DI - MOVQ AX, SI - ANDL $0x0000001f, SI - MOVQ $0x00000040, R8 - SUBQ SI, R8 - DECQ DI - JA emit_lit_memmove_long_standalonelarge_forward_sse_loop_32 - LEAQ -32(CX)(R8*1), SI - LEAQ -32(AX)(R8*1), R9 - -emit_lit_memmove_long_standalonelarge_big_loop_back: - MOVOU (SI), X4 - MOVOU 16(SI), X5 - MOVOA X4, (R9) - MOVOA X5, 16(R9) - ADDQ $0x20, R9 - ADDQ $0x20, SI - ADDQ $0x20, R8 - DECQ DI - JNA emit_lit_memmove_long_standalonelarge_big_loop_back - -emit_lit_memmove_long_standalonelarge_forward_sse_loop_32: - MOVOU -32(CX)(R8*1), X4 - MOVOU -16(CX)(R8*1), X5 - MOVOA X4, -32(AX)(R8*1) - MOVOA X5, -16(AX)(R8*1) - ADDQ $0x20, R8 - CMPQ DX, R8 - JAE emit_lit_memmove_long_standalonelarge_forward_sse_loop_32 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(DX*1) - MOVOU X3, -16(AX)(DX*1) - JMP emit_literal_end_standalone - JMP emit_literal_end_standalone - -emit_literal_end_standalone_skip: - XORQ BX, BX - -emit_literal_end_standalone: - MOVQ BX, ret+48(FP) - RET - -// func emitRepeat(dst []byte, offset int, length int) int -TEXT ·emitRepeat(SB), NOSPLIT, $0-48 - XORQ BX, BX - MOVQ dst_base+0(FP), AX - MOVQ offset+24(FP), CX - MOVQ length+32(FP), DX - - // emitRepeat -emit_repeat_again_standalone: - MOVL DX, SI - LEAL -4(DX), DX - CMPL SI, $0x08 - JBE repeat_two_standalone - CMPL SI, $0x0c - JAE cant_repeat_two_offset_standalone - CMPL CX, $0x00000800 - JB repeat_two_offset_standalone - -cant_repeat_two_offset_standalone: - CMPL DX, $0x00000104 - JB repeat_three_standalone - CMPL DX, $0x00010100 - JB repeat_four_standalone - CMPL DX, $0x0100ffff - JB repeat_five_standalone - LEAL -16842747(DX), DX - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - ADDQ $0x05, BX - JMP emit_repeat_again_standalone - -repeat_five_standalone: - LEAL -65536(DX), DX - MOVL DX, CX - MOVW $0x001d, (AX) - MOVW DX, 2(AX) - SARL $0x10, CX - MOVB CL, 4(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_repeat_end - -repeat_four_standalone: - LEAL -256(DX), DX - MOVW $0x0019, (AX) - MOVW DX, 2(AX) - ADDQ $0x04, BX - ADDQ $0x04, AX - JMP gen_emit_repeat_end - -repeat_three_standalone: - LEAL -4(DX), DX - MOVW $0x0015, (AX) - MOVB DL, 2(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - JMP gen_emit_repeat_end - -repeat_two_standalone: - SHLL $0x02, DX - ORL $0x01, DX - MOVW DX, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_repeat_end - -repeat_two_offset_standalone: - XORQ SI, SI - LEAL 1(SI)(DX*4), DX - MOVB CL, 1(AX) - SARL $0x08, CX - SHLL $0x05, CX - ORL CX, DX - MOVB DL, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - -gen_emit_repeat_end: - MOVQ BX, ret+40(FP) - RET - -// func emitCopy(dst []byte, offset int, length int) int -TEXT ·emitCopy(SB), NOSPLIT, $0-48 - XORQ BX, BX - MOVQ dst_base+0(FP), AX - MOVQ offset+24(FP), CX - MOVQ length+32(FP), DX - - // emitCopy - CMPL CX, $0x00010000 - JB two_byte_offset_standalone - CMPL DX, $0x40 - JBE four_bytes_remain_standalone - MOVB $0xff, (AX) - MOVL CX, 1(AX) - LEAL -64(DX), DX - ADDQ $0x05, BX - ADDQ $0x05, AX - CMPL DX, $0x04 - JB four_bytes_remain_standalone - - // emitRepeat -emit_repeat_again_standalone_emit_copy: - MOVL DX, SI - LEAL -4(DX), DX - CMPL SI, $0x08 - JBE repeat_two_standalone_emit_copy - CMPL SI, $0x0c - JAE cant_repeat_two_offset_standalone_emit_copy - CMPL CX, $0x00000800 - JB repeat_two_offset_standalone_emit_copy - -cant_repeat_two_offset_standalone_emit_copy: - CMPL DX, $0x00000104 - JB repeat_three_standalone_emit_copy - CMPL DX, $0x00010100 - JB repeat_four_standalone_emit_copy - CMPL DX, $0x0100ffff - JB repeat_five_standalone_emit_copy - LEAL -16842747(DX), DX - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - ADDQ $0x05, BX - JMP emit_repeat_again_standalone_emit_copy - -repeat_five_standalone_emit_copy: - LEAL -65536(DX), DX - MOVL DX, CX - MOVW $0x001d, (AX) - MOVW DX, 2(AX) - SARL $0x10, CX - MOVB CL, 4(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_copy_end - -repeat_four_standalone_emit_copy: - LEAL -256(DX), DX - MOVW $0x0019, (AX) - MOVW DX, 2(AX) - ADDQ $0x04, BX - ADDQ $0x04, AX - JMP gen_emit_copy_end - -repeat_three_standalone_emit_copy: - LEAL -4(DX), DX - MOVW $0x0015, (AX) - MOVB DL, 2(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - JMP gen_emit_copy_end - -repeat_two_standalone_emit_copy: - SHLL $0x02, DX - ORL $0x01, DX - MOVW DX, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -repeat_two_offset_standalone_emit_copy: - XORQ SI, SI - LEAL 1(SI)(DX*4), DX - MOVB CL, 1(AX) - SARL $0x08, CX - SHLL $0x05, CX - ORL CX, DX - MOVB DL, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -four_bytes_remain_standalone: - TESTL DX, DX - JZ gen_emit_copy_end - XORL SI, SI - LEAL -1(SI)(DX*4), DX - MOVB DL, (AX) - MOVL CX, 1(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_copy_end - -two_byte_offset_standalone: - CMPL DX, $0x40 - JBE two_byte_offset_short_standalone - CMPL CX, $0x00000800 - JAE long_offset_short_standalone - MOVL $0x00000001, SI - LEAL 16(SI), SI - MOVB CL, 1(AX) - MOVL CX, DI - SHRL $0x08, DI - SHLL $0x05, DI - ORL DI, SI - MOVB SI, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - SUBL $0x08, DX - - // emitRepeat - LEAL -4(DX), DX - JMP cant_repeat_two_offset_standalone_emit_copy_short_2b - -emit_repeat_again_standalone_emit_copy_short_2b: - MOVL DX, SI - LEAL -4(DX), DX - CMPL SI, $0x08 - JBE repeat_two_standalone_emit_copy_short_2b - CMPL SI, $0x0c - JAE cant_repeat_two_offset_standalone_emit_copy_short_2b - CMPL CX, $0x00000800 - JB repeat_two_offset_standalone_emit_copy_short_2b - -cant_repeat_two_offset_standalone_emit_copy_short_2b: - CMPL DX, $0x00000104 - JB repeat_three_standalone_emit_copy_short_2b - CMPL DX, $0x00010100 - JB repeat_four_standalone_emit_copy_short_2b - CMPL DX, $0x0100ffff - JB repeat_five_standalone_emit_copy_short_2b - LEAL -16842747(DX), DX - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - ADDQ $0x05, BX - JMP emit_repeat_again_standalone_emit_copy_short_2b - -repeat_five_standalone_emit_copy_short_2b: - LEAL -65536(DX), DX - MOVL DX, CX - MOVW $0x001d, (AX) - MOVW DX, 2(AX) - SARL $0x10, CX - MOVB CL, 4(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_copy_end - -repeat_four_standalone_emit_copy_short_2b: - LEAL -256(DX), DX - MOVW $0x0019, (AX) - MOVW DX, 2(AX) - ADDQ $0x04, BX - ADDQ $0x04, AX - JMP gen_emit_copy_end - -repeat_three_standalone_emit_copy_short_2b: - LEAL -4(DX), DX - MOVW $0x0015, (AX) - MOVB DL, 2(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - JMP gen_emit_copy_end - -repeat_two_standalone_emit_copy_short_2b: - SHLL $0x02, DX - ORL $0x01, DX - MOVW DX, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -repeat_two_offset_standalone_emit_copy_short_2b: - XORQ SI, SI - LEAL 1(SI)(DX*4), DX - MOVB CL, 1(AX) - SARL $0x08, CX - SHLL $0x05, CX - ORL CX, DX - MOVB DL, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -long_offset_short_standalone: - MOVB $0xee, (AX) - MOVW CX, 1(AX) - LEAL -60(DX), DX - ADDQ $0x03, AX - ADDQ $0x03, BX - - // emitRepeat -emit_repeat_again_standalone_emit_copy_short: - MOVL DX, SI - LEAL -4(DX), DX - CMPL SI, $0x08 - JBE repeat_two_standalone_emit_copy_short - CMPL SI, $0x0c - JAE cant_repeat_two_offset_standalone_emit_copy_short - CMPL CX, $0x00000800 - JB repeat_two_offset_standalone_emit_copy_short - -cant_repeat_two_offset_standalone_emit_copy_short: - CMPL DX, $0x00000104 - JB repeat_three_standalone_emit_copy_short - CMPL DX, $0x00010100 - JB repeat_four_standalone_emit_copy_short - CMPL DX, $0x0100ffff - JB repeat_five_standalone_emit_copy_short - LEAL -16842747(DX), DX - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - ADDQ $0x05, BX - JMP emit_repeat_again_standalone_emit_copy_short - -repeat_five_standalone_emit_copy_short: - LEAL -65536(DX), DX - MOVL DX, CX - MOVW $0x001d, (AX) - MOVW DX, 2(AX) - SARL $0x10, CX - MOVB CL, 4(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_copy_end - -repeat_four_standalone_emit_copy_short: - LEAL -256(DX), DX - MOVW $0x0019, (AX) - MOVW DX, 2(AX) - ADDQ $0x04, BX - ADDQ $0x04, AX - JMP gen_emit_copy_end - -repeat_three_standalone_emit_copy_short: - LEAL -4(DX), DX - MOVW $0x0015, (AX) - MOVB DL, 2(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - JMP gen_emit_copy_end - -repeat_two_standalone_emit_copy_short: - SHLL $0x02, DX - ORL $0x01, DX - MOVW DX, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -repeat_two_offset_standalone_emit_copy_short: - XORQ SI, SI - LEAL 1(SI)(DX*4), DX - MOVB CL, 1(AX) - SARL $0x08, CX - SHLL $0x05, CX - ORL CX, DX - MOVB DL, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -two_byte_offset_short_standalone: - MOVL DX, SI - SHLL $0x02, SI - CMPL DX, $0x0c - JAE emit_copy_three_standalone - CMPL CX, $0x00000800 - JAE emit_copy_three_standalone - LEAL -15(SI), SI - MOVB CL, 1(AX) - SHRL $0x08, CX - SHLL $0x05, CX - ORL CX, SI - MOVB SI, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end - -emit_copy_three_standalone: - LEAL -2(SI), SI - MOVB SI, (AX) - MOVW CX, 1(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - -gen_emit_copy_end: - MOVQ BX, ret+40(FP) - RET - -// func emitCopyNoRepeat(dst []byte, offset int, length int) int -TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48 - XORQ BX, BX - MOVQ dst_base+0(FP), AX - MOVQ offset+24(FP), CX - MOVQ length+32(FP), DX - - // emitCopy - CMPL CX, $0x00010000 - JB two_byte_offset_standalone_snappy - -four_bytes_loop_back_standalone_snappy: - CMPL DX, $0x40 - JBE four_bytes_remain_standalone_snappy - MOVB $0xff, (AX) - MOVL CX, 1(AX) - LEAL -64(DX), DX - ADDQ $0x05, BX - ADDQ $0x05, AX - CMPL DX, $0x04 - JB four_bytes_remain_standalone_snappy - JMP four_bytes_loop_back_standalone_snappy - -four_bytes_remain_standalone_snappy: - TESTL DX, DX - JZ gen_emit_copy_end_snappy - XORL SI, SI - LEAL -1(SI)(DX*4), DX - MOVB DL, (AX) - MOVL CX, 1(AX) - ADDQ $0x05, BX - ADDQ $0x05, AX - JMP gen_emit_copy_end_snappy - -two_byte_offset_standalone_snappy: - CMPL DX, $0x40 - JBE two_byte_offset_short_standalone_snappy - MOVB $0xee, (AX) - MOVW CX, 1(AX) - LEAL -60(DX), DX - ADDQ $0x03, AX - ADDQ $0x03, BX - JMP two_byte_offset_standalone_snappy - -two_byte_offset_short_standalone_snappy: - MOVL DX, SI - SHLL $0x02, SI - CMPL DX, $0x0c - JAE emit_copy_three_standalone_snappy - CMPL CX, $0x00000800 - JAE emit_copy_three_standalone_snappy - LEAL -15(SI), SI - MOVB CL, 1(AX) - SHRL $0x08, CX - SHLL $0x05, CX - ORL CX, SI - MOVB SI, (AX) - ADDQ $0x02, BX - ADDQ $0x02, AX - JMP gen_emit_copy_end_snappy - -emit_copy_three_standalone_snappy: - LEAL -2(SI), SI - MOVB SI, (AX) - MOVW CX, 1(AX) - ADDQ $0x03, BX - ADDQ $0x03, AX - -gen_emit_copy_end_snappy: - MOVQ BX, ret+40(FP) - RET - -// func matchLen(a []byte, b []byte) int -// Requires: BMI -TEXT ·matchLen(SB), NOSPLIT, $0-56 - MOVQ a_base+0(FP), AX - MOVQ b_base+24(FP), CX - MOVQ a_len+8(FP), DX - - // matchLen - XORL SI, SI - -matchlen_loopback_16_standalone: - CMPL DX, $0x10 - JB matchlen_match8_standalone - MOVQ (AX)(SI*1), BX - MOVQ 8(AX)(SI*1), DI - XORQ (CX)(SI*1), BX - JNZ matchlen_bsf_8_standalone - XORQ 8(CX)(SI*1), DI - JNZ matchlen_bsf_16standalone - LEAL -16(DX), DX - LEAL 16(SI), SI - JMP matchlen_loopback_16_standalone - -matchlen_bsf_16standalone: -#ifdef GOAMD64_v3 - TZCNTQ DI, DI - -#else - BSFQ DI, DI - -#endif - SARQ $0x03, DI - LEAL 8(SI)(DI*1), SI - JMP gen_match_len_end - -matchlen_match8_standalone: - CMPL DX, $0x08 - JB matchlen_match4_standalone - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - JNZ matchlen_bsf_8_standalone - LEAL -8(DX), DX - LEAL 8(SI), SI - JMP matchlen_match4_standalone - -matchlen_bsf_8_standalone: -#ifdef GOAMD64_v3 - TZCNTQ BX, BX - -#else - BSFQ BX, BX - -#endif - SARQ $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end - -matchlen_match4_standalone: - CMPL DX, $0x04 - JB matchlen_match2_standalone - MOVL (AX)(SI*1), BX - CMPL (CX)(SI*1), BX - JNE matchlen_match2_standalone - LEAL -4(DX), DX - LEAL 4(SI), SI - -matchlen_match2_standalone: - CMPL DX, $0x01 - JE matchlen_match1_standalone - JB gen_match_len_end - MOVW (AX)(SI*1), BX - CMPW (CX)(SI*1), BX - JNE matchlen_match1_standalone - LEAL 2(SI), SI - SUBL $0x02, DX - JZ gen_match_len_end - -matchlen_match1_standalone: - MOVB (AX)(SI*1), BL - CMPB (CX)(SI*1), BL - JNE gen_match_len_end - LEAL 1(SI), SI - -gen_match_len_end: - MOVQ SI, ret+48(FP) - RET - -// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) -// Requires: SSE2 -TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 - XORQ SI, SI - MOVQ dst_base+0(FP), AX - MOVQ dst_len+8(FP), CX - MOVQ src_base+24(FP), DX - MOVQ src_len+32(FP), BX - LEAQ (DX)(BX*1), BX - LEAQ -8(AX)(CX*1), CX - XORQ DI, DI - -lz4_s2_loop: - CMPQ DX, BX - JAE lz4_s2_corrupt - CMPQ AX, CX - JAE lz4_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 - JB lz4_s2_ll_end - -lz4_s2_ll_loop: - INCQ DX - CMPQ DX, BX - JAE lz4_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff - JEQ lz4_s2_ll_loop - -lz4_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x04, R10 - CMPQ R8, BX - JAE lz4_s2_corrupt - INCQ DX - INCQ R8 - TESTQ R9, R9 - JZ lz4_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX - JAE lz4_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c - JB one_byte_lz4_s2 - CMPL R11, $0x00000100 - JB two_bytes_lz4_s2 - CMPL R11, $0x00010000 - JB three_bytes_lz4_s2 - CMPL R11, $0x01000000 - JB four_bytes_lz4_s2 - MOVB $0xfc, (AX) - MOVL R11, 1(AX) - ADDQ $0x05, AX - JMP memmove_long_lz4_s2 - -four_bytes_lz4_s2: - MOVL R11, R12 - SHRL $0x10, R12 - MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) - ADDQ $0x04, AX - JMP memmove_long_lz4_s2 - -three_bytes_lz4_s2: - MOVB $0xf4, (AX) - MOVW R11, 1(AX) - ADDQ $0x03, AX - JMP memmove_long_lz4_s2 - -two_bytes_lz4_s2: - MOVB $0xf0, (AX) - MOVB R11, 1(AX) - ADDQ $0x02, AX - CMPL R11, $0x40 - JB memmove_lz4_s2 - JMP memmove_long_lz4_s2 - -one_byte_lz4_s2: - SHLB $0x02, R11 - MOVB R11, (AX) - ADDQ $0x01, AX - -memmove_lz4_s2: - LEAQ (AX)(R9*1), R11 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_lz4_s2_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_lz4_s2_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_lz4_s2_memmove_move_17through32 - JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 - -emit_lit_memmove_lz4_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) - JMP memmove_end_copy_lz4_s2 - -emit_lit_memmove_lz4_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) - JMP memmove_end_copy_lz4_s2 - -emit_lit_memmove_lz4_s2_memmove_move_17through32: - MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 - MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) - JMP memmove_end_copy_lz4_s2 - -emit_lit_memmove_lz4_s2_memmove_move_33through64: - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - -memmove_end_copy_lz4_s2: - MOVQ R11, AX - JMP lz4_s2_lits_emit_done - -memmove_long_lz4_s2: - LEAQ (AX)(R9*1), R11 - - // genMemMoveLong - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 - JA emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 - -emit_lit_memmove_long_lz4_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_lz4_s2large_big_loop_back - -emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX - -lz4_s2_lits_emit_done: - MOVQ R8, DX - -lz4_s2_lits_done: - CMPQ DX, BX - JNE lz4_s2_match - CMPQ R10, $0x04 - JEQ lz4_s2_done - JMP lz4_s2_corrupt - -lz4_s2_match: - LEAQ 2(DX), R8 - CMPQ R8, BX - JAE lz4_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 - JZ lz4_s2_corrupt - CMPQ R9, SI - JA lz4_s2_corrupt - CMPQ R10, $0x13 - JNE lz4_s2_ml_done - -lz4_s2_ml_loop: - MOVBQZX (DX), R8 - INCQ DX - ADDQ R8, R10 - CMPQ DX, BX - JAE lz4_s2_corrupt - CMPQ R8, $0xff - JEQ lz4_s2_ml_loop - -lz4_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI - JNE lz4_s2_docopy - - // emitRepeat -emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2 - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2 - -cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2 - -repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4_s2_loop - -repeat_four_lz4_s2: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4_s2_loop - -repeat_three_lz4_s2: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4_s2_loop - -repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -lz4_s2_docopy: - MOVQ R9, DI - - // emitCopy - CMPL R10, $0x40 - JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 - JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) - ADDQ $0x02, AX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - -emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2_emit_copy_short_2b - -cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2_emit_copy_short_2b - -repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4_s2_loop - -repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4_s2_loop - -repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4_s2_loop - -repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -long_offset_short_lz4_s2: - MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 - ADDQ $0x03, AX - - // emitRepeat -emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2_emit_copy_short - -cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2_emit_copy_short - -repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4_s2_loop - -repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4_s2_loop - -repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4_s2_loop - -repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c - JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 - JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) - ADDQ $0x02, AX - JMP lz4_s2_loop - -emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) - ADDQ $0x03, AX - JMP lz4_s2_loop - -lz4_s2_done: - MOVQ dst_base+0(FP), CX - SUBQ CX, AX - MOVQ SI, uncompressed+48(FP) - MOVQ AX, dstUsed+56(FP) - RET - -lz4_s2_corrupt: - XORQ AX, AX - LEAQ -1(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -lz4_s2_dstfull: - XORQ AX, AX - LEAQ -2(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -// func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) -// Requires: SSE2 -TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 - XORQ SI, SI - MOVQ dst_base+0(FP), AX - MOVQ dst_len+8(FP), CX - MOVQ src_base+24(FP), DX - MOVQ src_len+32(FP), BX - LEAQ (DX)(BX*1), BX - LEAQ -8(AX)(CX*1), CX - XORQ DI, DI - -lz4s_s2_loop: - CMPQ DX, BX - JAE lz4s_s2_corrupt - CMPQ AX, CX - JAE lz4s_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 - JB lz4s_s2_ll_end - -lz4s_s2_ll_loop: - INCQ DX - CMPQ DX, BX - JAE lz4s_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff - JEQ lz4s_s2_ll_loop - -lz4s_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x03, R10 - CMPQ R8, BX - JAE lz4s_s2_corrupt - INCQ DX - INCQ R8 - TESTQ R9, R9 - JZ lz4s_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX - JAE lz4s_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c - JB one_byte_lz4s_s2 - CMPL R11, $0x00000100 - JB two_bytes_lz4s_s2 - CMPL R11, $0x00010000 - JB three_bytes_lz4s_s2 - CMPL R11, $0x01000000 - JB four_bytes_lz4s_s2 - MOVB $0xfc, (AX) - MOVL R11, 1(AX) - ADDQ $0x05, AX - JMP memmove_long_lz4s_s2 - -four_bytes_lz4s_s2: - MOVL R11, R12 - SHRL $0x10, R12 - MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) - ADDQ $0x04, AX - JMP memmove_long_lz4s_s2 - -three_bytes_lz4s_s2: - MOVB $0xf4, (AX) - MOVW R11, 1(AX) - ADDQ $0x03, AX - JMP memmove_long_lz4s_s2 - -two_bytes_lz4s_s2: - MOVB $0xf0, (AX) - MOVB R11, 1(AX) - ADDQ $0x02, AX - CMPL R11, $0x40 - JB memmove_lz4s_s2 - JMP memmove_long_lz4s_s2 - -one_byte_lz4s_s2: - SHLB $0x02, R11 - MOVB R11, (AX) - ADDQ $0x01, AX - -memmove_lz4s_s2: - LEAQ (AX)(R9*1), R11 - - // genMemMoveShort - CMPQ R9, $0x08 - JBE emit_lit_memmove_lz4s_s2_memmove_move_8 - CMPQ R9, $0x10 - JBE emit_lit_memmove_lz4s_s2_memmove_move_8through16 - CMPQ R9, $0x20 - JBE emit_lit_memmove_lz4s_s2_memmove_move_17through32 - JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 - -emit_lit_memmove_lz4s_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) - JMP memmove_end_copy_lz4s_s2 - -emit_lit_memmove_lz4s_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) - JMP memmove_end_copy_lz4s_s2 - -emit_lit_memmove_lz4s_s2_memmove_move_17through32: - MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 - MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) - JMP memmove_end_copy_lz4s_s2 - -emit_lit_memmove_lz4s_s2_memmove_move_33through64: - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - -memmove_end_copy_lz4s_s2: - MOVQ R11, AX - JMP lz4s_s2_lits_emit_done - -memmove_long_lz4s_s2: - LEAQ (AX)(R9*1), R11 - - // genMemMoveLong - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 - JA emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 - -emit_lit_memmove_long_lz4s_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) - ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 - JNA emit_lit_memmove_long_lz4s_s2large_big_loop_back - -emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 - JAE emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX - -lz4s_s2_lits_emit_done: - MOVQ R8, DX - -lz4s_s2_lits_done: - CMPQ DX, BX - JNE lz4s_s2_match - CMPQ R10, $0x03 - JEQ lz4s_s2_done - JMP lz4s_s2_corrupt - -lz4s_s2_match: - CMPQ R10, $0x03 - JEQ lz4s_s2_loop - LEAQ 2(DX), R8 - CMPQ R8, BX - JAE lz4s_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 - JZ lz4s_s2_corrupt - CMPQ R9, SI - JA lz4s_s2_corrupt - CMPQ R10, $0x12 - JNE lz4s_s2_ml_done - -lz4s_s2_ml_loop: - MOVBQZX (DX), R8 - INCQ DX - ADDQ R8, R10 - CMPQ DX, BX - JAE lz4s_s2_corrupt - CMPQ R8, $0xff - JEQ lz4s_s2_ml_loop - -lz4s_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI - JNE lz4s_s2_docopy - - // emitRepeat -emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2 - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2 - -cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2 - -repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4s_s2_loop - -repeat_four_lz4_s2: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4s_s2_loop - -repeat_three_lz4_s2: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4s_s2_loop - -repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -lz4s_s2_docopy: - MOVQ R9, DI - - // emitCopy - CMPL R10, $0x40 - JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 - JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) - ADDQ $0x02, AX - SUBL $0x08, R10 - - // emitRepeat - LEAL -4(R10), R10 - JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - -emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2_emit_copy_short_2b - -cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2_emit_copy_short_2b - -repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4s_s2_loop - -repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4s_s2_loop - -repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4s_s2_loop - -repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -long_offset_short_lz4_s2: - MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 - ADDQ $0x03, AX - - // emitRepeat -emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 - JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c - JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 - JB repeat_two_offset_lz4_s2_emit_copy_short - -cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 - JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 - JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff - JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 - MOVL $0xfffb001d, (AX) - MOVB $0xff, 4(AX) - ADDQ $0x05, AX - JMP emit_repeat_again_lz4_s2_emit_copy_short - -repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 - MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) - ADDQ $0x05, AX - JMP lz4s_s2_loop - -repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 - MOVW $0x0019, (AX) - MOVW R10, 2(AX) - ADDQ $0x04, AX - JMP lz4s_s2_loop - -repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 - MOVW $0x0015, (AX) - MOVB R10, 2(AX) - ADDQ $0x03, AX - JMP lz4s_s2_loop - -repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c - JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 - JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) - ADDQ $0x02, AX - JMP lz4s_s2_loop - -emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) - ADDQ $0x03, AX - JMP lz4s_s2_loop - -lz4s_s2_done: - MOVQ dst_base+0(FP), CX - SUBQ CX, AX - MOVQ SI, uncompressed+48(FP) - MOVQ AX, dstUsed+56(FP) - RET - -lz4s_s2_corrupt: - XORQ AX, AX - LEAQ -1(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -lz4s_s2_dstfull: - XORQ AX, AX - LEAQ -2(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -// func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) -// Requires: SSE2 -TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64 - XORQ SI, SI - MOVQ dst_base+0(FP), AX - MOVQ dst_len+8(FP), CX - MOVQ src_base+24(FP), DX - MOVQ src_len+32(FP), BX - LEAQ (DX)(BX*1), BX - LEAQ -8(AX)(CX*1), CX - -lz4_snappy_loop: - CMPQ DX, BX - JAE lz4_snappy_corrupt - CMPQ AX, CX - JAE lz4_snappy_dstfull - MOVBQZX (DX), DI - MOVQ DI, R8 - MOVQ DI, R9 - SHRQ $0x04, R8 - ANDQ $0x0f, R9 - CMPQ DI, $0xf0 - JB lz4_snappy_ll_end - -lz4_snappy_ll_loop: - INCQ DX - CMPQ DX, BX - JAE lz4_snappy_corrupt - MOVBQZX (DX), DI - ADDQ DI, R8 - CMPQ DI, $0xff - JEQ lz4_snappy_ll_loop - -lz4_snappy_ll_end: - LEAQ (DX)(R8*1), DI - ADDQ $0x04, R9 - CMPQ DI, BX - JAE lz4_snappy_corrupt - INCQ DX - INCQ DI - TESTQ R8, R8 - JZ lz4_snappy_lits_done - LEAQ (AX)(R8*1), R10 - CMPQ R10, CX - JAE lz4_snappy_dstfull - ADDQ R8, SI - LEAL -1(R8), R10 - CMPL R10, $0x3c - JB one_byte_lz4_snappy - CMPL R10, $0x00000100 - JB two_bytes_lz4_snappy - CMPL R10, $0x00010000 - JB three_bytes_lz4_snappy - CMPL R10, $0x01000000 - JB four_bytes_lz4_snappy - MOVB $0xfc, (AX) - MOVL R10, 1(AX) - ADDQ $0x05, AX - JMP memmove_long_lz4_snappy - -four_bytes_lz4_snappy: - MOVL R10, R11 - SHRL $0x10, R11 - MOVB $0xf8, (AX) - MOVW R10, 1(AX) - MOVB R11, 3(AX) - ADDQ $0x04, AX - JMP memmove_long_lz4_snappy - -three_bytes_lz4_snappy: - MOVB $0xf4, (AX) - MOVW R10, 1(AX) - ADDQ $0x03, AX - JMP memmove_long_lz4_snappy - -two_bytes_lz4_snappy: - MOVB $0xf0, (AX) - MOVB R10, 1(AX) - ADDQ $0x02, AX - CMPL R10, $0x40 - JB memmove_lz4_snappy - JMP memmove_long_lz4_snappy - -one_byte_lz4_snappy: - SHLB $0x02, R10 - MOVB R10, (AX) - ADDQ $0x01, AX - -memmove_lz4_snappy: - LEAQ (AX)(R8*1), R10 - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_lz4_snappy_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_lz4_snappy_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_lz4_snappy_memmove_move_17through32 - JMP emit_lit_memmove_lz4_snappy_memmove_move_33through64 - -emit_lit_memmove_lz4_snappy_memmove_move_8: - MOVQ (DX), R11 - MOVQ R11, (AX) - JMP memmove_end_copy_lz4_snappy - -emit_lit_memmove_lz4_snappy_memmove_move_8through16: - MOVQ (DX), R11 - MOVQ -8(DX)(R8*1), DX - MOVQ R11, (AX) - MOVQ DX, -8(AX)(R8*1) - JMP memmove_end_copy_lz4_snappy - -emit_lit_memmove_lz4_snappy_memmove_move_17through32: - MOVOU (DX), X0 - MOVOU -16(DX)(R8*1), X1 - MOVOU X0, (AX) - MOVOU X1, -16(AX)(R8*1) - JMP memmove_end_copy_lz4_snappy - -emit_lit_memmove_lz4_snappy_memmove_move_33through64: - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R8*1), X2 - MOVOU -16(DX)(R8*1), X3 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R8*1) - MOVOU X3, -16(AX)(R8*1) - -memmove_end_copy_lz4_snappy: - MOVQ R10, AX - JMP lz4_snappy_lits_emit_done - -memmove_long_lz4_snappy: - LEAQ (AX)(R8*1), R10 - - // genMemMoveLong - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R8*1), X2 - MOVOU -16(DX)(R8*1), X3 - MOVQ R8, R12 - SHRQ $0x05, R12 - MOVQ AX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32 - LEAQ -32(DX)(R13*1), R11 - LEAQ -32(AX)(R13*1), R14 - -emit_lit_memmove_long_lz4_snappylarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_lz4_snappylarge_big_loop_back - -emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32: - MOVOU -32(DX)(R13*1), X4 - MOVOU -16(DX)(R13*1), X5 - MOVOA X4, -32(AX)(R13*1) - MOVOA X5, -16(AX)(R13*1) - ADDQ $0x20, R13 - CMPQ R8, R13 - JAE emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R8*1) - MOVOU X3, -16(AX)(R8*1) - MOVQ R10, AX - -lz4_snappy_lits_emit_done: - MOVQ DI, DX - -lz4_snappy_lits_done: - CMPQ DX, BX - JNE lz4_snappy_match - CMPQ R9, $0x04 - JEQ lz4_snappy_done - JMP lz4_snappy_corrupt - -lz4_snappy_match: - LEAQ 2(DX), DI - CMPQ DI, BX - JAE lz4_snappy_corrupt - MOVWQZX (DX), R8 - MOVQ DI, DX - TESTQ R8, R8 - JZ lz4_snappy_corrupt - CMPQ R8, SI - JA lz4_snappy_corrupt - CMPQ R9, $0x13 - JNE lz4_snappy_ml_done - -lz4_snappy_ml_loop: - MOVBQZX (DX), DI - INCQ DX - ADDQ DI, R9 - CMPQ DX, BX - JAE lz4_snappy_corrupt - CMPQ DI, $0xff - JEQ lz4_snappy_ml_loop - -lz4_snappy_ml_done: - ADDQ R9, SI - - // emitCopy -two_byte_offset_lz4_s2: - CMPL R9, $0x40 - JBE two_byte_offset_short_lz4_s2 - MOVB $0xee, (AX) - MOVW R8, 1(AX) - LEAL -60(R9), R9 - ADDQ $0x03, AX - CMPQ AX, CX - JAE lz4_snappy_loop - JMP two_byte_offset_lz4_s2 - -two_byte_offset_short_lz4_s2: - MOVL R9, DI - SHLL $0x02, DI - CMPL R9, $0x0c - JAE emit_copy_three_lz4_s2 - CMPL R8, $0x00000800 - JAE emit_copy_three_lz4_s2 - LEAL -15(DI), DI - MOVB R8, 1(AX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, DI - MOVB DI, (AX) - ADDQ $0x02, AX - JMP lz4_snappy_loop - -emit_copy_three_lz4_s2: - LEAL -2(DI), DI - MOVB DI, (AX) - MOVW R8, 1(AX) - ADDQ $0x03, AX - JMP lz4_snappy_loop - -lz4_snappy_done: - MOVQ dst_base+0(FP), CX - SUBQ CX, AX - MOVQ SI, uncompressed+48(FP) - MOVQ AX, dstUsed+56(FP) - RET - -lz4_snappy_corrupt: - XORQ AX, AX - LEAQ -1(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -lz4_snappy_dstfull: - XORQ AX, AX - LEAQ -2(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -// func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) -// Requires: SSE2 -TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64 - XORQ SI, SI - MOVQ dst_base+0(FP), AX - MOVQ dst_len+8(FP), CX - MOVQ src_base+24(FP), DX - MOVQ src_len+32(FP), BX - LEAQ (DX)(BX*1), BX - LEAQ -8(AX)(CX*1), CX - -lz4s_snappy_loop: - CMPQ DX, BX - JAE lz4s_snappy_corrupt - CMPQ AX, CX - JAE lz4s_snappy_dstfull - MOVBQZX (DX), DI - MOVQ DI, R8 - MOVQ DI, R9 - SHRQ $0x04, R8 - ANDQ $0x0f, R9 - CMPQ DI, $0xf0 - JB lz4s_snappy_ll_end - -lz4s_snappy_ll_loop: - INCQ DX - CMPQ DX, BX - JAE lz4s_snappy_corrupt - MOVBQZX (DX), DI - ADDQ DI, R8 - CMPQ DI, $0xff - JEQ lz4s_snappy_ll_loop - -lz4s_snappy_ll_end: - LEAQ (DX)(R8*1), DI - ADDQ $0x03, R9 - CMPQ DI, BX - JAE lz4s_snappy_corrupt - INCQ DX - INCQ DI - TESTQ R8, R8 - JZ lz4s_snappy_lits_done - LEAQ (AX)(R8*1), R10 - CMPQ R10, CX - JAE lz4s_snappy_dstfull - ADDQ R8, SI - LEAL -1(R8), R10 - CMPL R10, $0x3c - JB one_byte_lz4s_snappy - CMPL R10, $0x00000100 - JB two_bytes_lz4s_snappy - CMPL R10, $0x00010000 - JB three_bytes_lz4s_snappy - CMPL R10, $0x01000000 - JB four_bytes_lz4s_snappy - MOVB $0xfc, (AX) - MOVL R10, 1(AX) - ADDQ $0x05, AX - JMP memmove_long_lz4s_snappy - -four_bytes_lz4s_snappy: - MOVL R10, R11 - SHRL $0x10, R11 - MOVB $0xf8, (AX) - MOVW R10, 1(AX) - MOVB R11, 3(AX) - ADDQ $0x04, AX - JMP memmove_long_lz4s_snappy - -three_bytes_lz4s_snappy: - MOVB $0xf4, (AX) - MOVW R10, 1(AX) - ADDQ $0x03, AX - JMP memmove_long_lz4s_snappy - -two_bytes_lz4s_snappy: - MOVB $0xf0, (AX) - MOVB R10, 1(AX) - ADDQ $0x02, AX - CMPL R10, $0x40 - JB memmove_lz4s_snappy - JMP memmove_long_lz4s_snappy - -one_byte_lz4s_snappy: - SHLB $0x02, R10 - MOVB R10, (AX) - ADDQ $0x01, AX - -memmove_lz4s_snappy: - LEAQ (AX)(R8*1), R10 - - // genMemMoveShort - CMPQ R8, $0x08 - JBE emit_lit_memmove_lz4s_snappy_memmove_move_8 - CMPQ R8, $0x10 - JBE emit_lit_memmove_lz4s_snappy_memmove_move_8through16 - CMPQ R8, $0x20 - JBE emit_lit_memmove_lz4s_snappy_memmove_move_17through32 - JMP emit_lit_memmove_lz4s_snappy_memmove_move_33through64 - -emit_lit_memmove_lz4s_snappy_memmove_move_8: - MOVQ (DX), R11 - MOVQ R11, (AX) - JMP memmove_end_copy_lz4s_snappy - -emit_lit_memmove_lz4s_snappy_memmove_move_8through16: - MOVQ (DX), R11 - MOVQ -8(DX)(R8*1), DX - MOVQ R11, (AX) - MOVQ DX, -8(AX)(R8*1) - JMP memmove_end_copy_lz4s_snappy - -emit_lit_memmove_lz4s_snappy_memmove_move_17through32: - MOVOU (DX), X0 - MOVOU -16(DX)(R8*1), X1 - MOVOU X0, (AX) - MOVOU X1, -16(AX)(R8*1) - JMP memmove_end_copy_lz4s_snappy - -emit_lit_memmove_lz4s_snappy_memmove_move_33through64: - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R8*1), X2 - MOVOU -16(DX)(R8*1), X3 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R8*1) - MOVOU X3, -16(AX)(R8*1) - -memmove_end_copy_lz4s_snappy: - MOVQ R10, AX - JMP lz4s_snappy_lits_emit_done - -memmove_long_lz4s_snappy: - LEAQ (AX)(R8*1), R10 - - // genMemMoveLong - MOVOU (DX), X0 - MOVOU 16(DX), X1 - MOVOU -32(DX)(R8*1), X2 - MOVOU -16(DX)(R8*1), X3 - MOVQ R8, R12 - SHRQ $0x05, R12 - MOVQ AX, R11 - ANDL $0x0000001f, R11 - MOVQ $0x00000040, R13 - SUBQ R11, R13 - DECQ R12 - JA emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32 - LEAQ -32(DX)(R13*1), R11 - LEAQ -32(AX)(R13*1), R14 - -emit_lit_memmove_long_lz4s_snappylarge_big_loop_back: - MOVOU (R11), X4 - MOVOU 16(R11), X5 - MOVOA X4, (R14) - MOVOA X5, 16(R14) - ADDQ $0x20, R14 - ADDQ $0x20, R11 - ADDQ $0x20, R13 - DECQ R12 - JNA emit_lit_memmove_long_lz4s_snappylarge_big_loop_back - -emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32: - MOVOU -32(DX)(R13*1), X4 - MOVOU -16(DX)(R13*1), X5 - MOVOA X4, -32(AX)(R13*1) - MOVOA X5, -16(AX)(R13*1) - ADDQ $0x20, R13 - CMPQ R8, R13 - JAE emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32 - MOVOU X0, (AX) - MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R8*1) - MOVOU X3, -16(AX)(R8*1) - MOVQ R10, AX - -lz4s_snappy_lits_emit_done: - MOVQ DI, DX - -lz4s_snappy_lits_done: - CMPQ DX, BX - JNE lz4s_snappy_match - CMPQ R9, $0x03 - JEQ lz4s_snappy_done - JMP lz4s_snappy_corrupt - -lz4s_snappy_match: - CMPQ R9, $0x03 - JEQ lz4s_snappy_loop - LEAQ 2(DX), DI - CMPQ DI, BX - JAE lz4s_snappy_corrupt - MOVWQZX (DX), R8 - MOVQ DI, DX - TESTQ R8, R8 - JZ lz4s_snappy_corrupt - CMPQ R8, SI - JA lz4s_snappy_corrupt - CMPQ R9, $0x12 - JNE lz4s_snappy_ml_done - -lz4s_snappy_ml_loop: - MOVBQZX (DX), DI - INCQ DX - ADDQ DI, R9 - CMPQ DX, BX - JAE lz4s_snappy_corrupt - CMPQ DI, $0xff - JEQ lz4s_snappy_ml_loop - -lz4s_snappy_ml_done: - ADDQ R9, SI - - // emitCopy -two_byte_offset_lz4_s2: - CMPL R9, $0x40 - JBE two_byte_offset_short_lz4_s2 - MOVB $0xee, (AX) - MOVW R8, 1(AX) - LEAL -60(R9), R9 - ADDQ $0x03, AX - CMPQ AX, CX - JAE lz4s_snappy_loop - JMP two_byte_offset_lz4_s2 - -two_byte_offset_short_lz4_s2: - MOVL R9, DI - SHLL $0x02, DI - CMPL R9, $0x0c - JAE emit_copy_three_lz4_s2 - CMPL R8, $0x00000800 - JAE emit_copy_three_lz4_s2 - LEAL -15(DI), DI - MOVB R8, 1(AX) - SHRL $0x08, R8 - SHLL $0x05, R8 - ORL R8, DI - MOVB DI, (AX) - ADDQ $0x02, AX - JMP lz4s_snappy_loop - -emit_copy_three_lz4_s2: - LEAL -2(DI), DI - MOVB DI, (AX) - MOVW R8, 1(AX) - ADDQ $0x03, AX - JMP lz4s_snappy_loop - -lz4s_snappy_done: - MOVQ dst_base+0(FP), CX - SUBQ CX, AX - MOVQ SI, uncompressed+48(FP) - MOVQ AX, dstUsed+56(FP) - RET - -lz4s_snappy_corrupt: - XORQ AX, AX - LEAQ -1(AX), SI - MOVQ SI, uncompressed+48(FP) - RET - -lz4s_snappy_dstfull: - XORQ AX, AX - LEAQ -2(AX), SI - MOVQ SI, uncompressed+48(FP) - RET diff --git a/vendor/github.com/klauspost/compress/s2/index.go b/vendor/github.com/klauspost/compress/s2/index.go deleted file mode 100644 index 4229957b9..000000000 --- a/vendor/github.com/klauspost/compress/s2/index.go +++ /dev/null @@ -1,602 +0,0 @@ -// Copyright (c) 2022+ Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "io" - "sort" -) - -const ( - S2IndexHeader = "s2idx\x00" - S2IndexTrailer = "\x00xdi2s" - maxIndexEntries = 1 << 16 - // If distance is less than this, we do not add the entry. - minIndexDist = 1 << 20 -) - -// Index represents an S2/Snappy index. -type Index struct { - TotalUncompressed int64 // Total Uncompressed size if known. Will be -1 if unknown. - TotalCompressed int64 // Total Compressed size if known. Will be -1 if unknown. - info []struct { - compressedOffset int64 - uncompressedOffset int64 - } - estBlockUncomp int64 -} - -func (i *Index) reset(maxBlock int) { - i.estBlockUncomp = int64(maxBlock) - i.TotalCompressed = -1 - i.TotalUncompressed = -1 - if len(i.info) > 0 { - i.info = i.info[:0] - } -} - -// allocInfos will allocate an empty slice of infos. -func (i *Index) allocInfos(n int) { - if n > maxIndexEntries { - panic("n > maxIndexEntries") - } - i.info = make([]struct { - compressedOffset int64 - uncompressedOffset int64 - }, 0, n) -} - -// add an uncompressed and compressed pair. -// Entries must be sent in order. -func (i *Index) add(compressedOffset, uncompressedOffset int64) error { - if i == nil { - return nil - } - lastIdx := len(i.info) - 1 - if lastIdx >= 0 { - latest := i.info[lastIdx] - if latest.uncompressedOffset == uncompressedOffset { - // Uncompressed didn't change, don't add entry, - // but update start index. - latest.compressedOffset = compressedOffset - i.info[lastIdx] = latest - return nil - } - if latest.uncompressedOffset > uncompressedOffset { - return fmt.Errorf("internal error: Earlier uncompressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset) - } - if latest.compressedOffset > compressedOffset { - return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset) - } - if latest.uncompressedOffset+minIndexDist > uncompressedOffset { - // Only add entry if distance is large enough. - return nil - } - } - i.info = append(i.info, struct { - compressedOffset int64 - uncompressedOffset int64 - }{compressedOffset: compressedOffset, uncompressedOffset: uncompressedOffset}) - return nil -} - -// Find the offset at or before the wanted (uncompressed) offset. -// If offset is 0 or positive it is the offset from the beginning of the file. -// If the uncompressed size is known, the offset must be within the file. -// If an offset outside the file is requested io.ErrUnexpectedEOF is returned. -// If the offset is negative, it is interpreted as the distance from the end of the file, -// where -1 represents the last byte. -// If offset from the end of the file is requested, but size is unknown, -// ErrUnsupported will be returned. -func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err error) { - if i.TotalUncompressed < 0 { - return 0, 0, ErrCorrupt - } - if offset < 0 { - offset = i.TotalUncompressed + offset - if offset < 0 { - return 0, 0, io.ErrUnexpectedEOF - } - } - if offset > i.TotalUncompressed { - return 0, 0, io.ErrUnexpectedEOF - } - if len(i.info) > 200 { - n := sort.Search(len(i.info), func(n int) bool { - return i.info[n].uncompressedOffset > offset - }) - if n == 0 { - n = 1 - } - return i.info[n-1].compressedOffset, i.info[n-1].uncompressedOffset, nil - } - for _, info := range i.info { - if info.uncompressedOffset > offset { - break - } - compressedOff = info.compressedOffset - uncompressedOff = info.uncompressedOffset - } - return compressedOff, uncompressedOff, nil -} - -// reduce to stay below maxIndexEntries -func (i *Index) reduce() { - if len(i.info) < maxIndexEntries && i.estBlockUncomp >= minIndexDist { - return - } - - // Algorithm, keep 1, remove removeN entries... - removeN := (len(i.info) + 1) / maxIndexEntries - src := i.info - j := 0 - - // Each block should be at least 1MB, but don't reduce below 1000 entries. - for i.estBlockUncomp*(int64(removeN)+1) < minIndexDist && len(i.info)/(removeN+1) > 1000 { - removeN++ - } - for idx := 0; idx < len(src); idx++ { - i.info[j] = src[idx] - j++ - idx += removeN - } - i.info = i.info[:j] - // Update maxblock estimate. - i.estBlockUncomp += i.estBlockUncomp * int64(removeN) -} - -func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte { - i.reduce() - var tmp [binary.MaxVarintLen64]byte - - initSize := len(b) - // We make the start a skippable header+size. - b = append(b, ChunkTypeIndex, 0, 0, 0) - b = append(b, []byte(S2IndexHeader)...) - // Total Uncompressed size - n := binary.PutVarint(tmp[:], uncompTotal) - b = append(b, tmp[:n]...) - // Total Compressed size - n = binary.PutVarint(tmp[:], compTotal) - b = append(b, tmp[:n]...) - // Put EstBlockUncomp size - n = binary.PutVarint(tmp[:], i.estBlockUncomp) - b = append(b, tmp[:n]...) - // Put length - n = binary.PutVarint(tmp[:], int64(len(i.info))) - b = append(b, tmp[:n]...) - - // Check if we should add uncompressed offsets - var hasUncompressed byte - for idx, info := range i.info { - if idx == 0 { - if info.uncompressedOffset != 0 { - hasUncompressed = 1 - break - } - continue - } - if info.uncompressedOffset != i.info[idx-1].uncompressedOffset+i.estBlockUncomp { - hasUncompressed = 1 - break - } - } - b = append(b, hasUncompressed) - - // Add each entry - if hasUncompressed == 1 { - for idx, info := range i.info { - uOff := info.uncompressedOffset - if idx > 0 { - prev := i.info[idx-1] - uOff -= prev.uncompressedOffset + (i.estBlockUncomp) - } - n = binary.PutVarint(tmp[:], uOff) - b = append(b, tmp[:n]...) - } - } - - // Initial compressed size estimate. - cPredict := i.estBlockUncomp / 2 - - for idx, info := range i.info { - cOff := info.compressedOffset - if idx > 0 { - prev := i.info[idx-1] - cOff -= prev.compressedOffset + cPredict - // Update compressed size prediction, with half the error. - cPredict += cOff / 2 - } - n = binary.PutVarint(tmp[:], cOff) - b = append(b, tmp[:n]...) - } - - // Add Total Size. - // Stored as fixed size for easier reading. - binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)-initSize+4+len(S2IndexTrailer))) - b = append(b, tmp[:4]...) - // Trailer - b = append(b, []byte(S2IndexTrailer)...) - - // Update size - chunkLen := len(b) - initSize - skippableFrameHeader - b[initSize+1] = uint8(chunkLen >> 0) - b[initSize+2] = uint8(chunkLen >> 8) - b[initSize+3] = uint8(chunkLen >> 16) - //fmt.Printf("chunklen: 0x%x Uncomp:%d, Comp:%d\n", chunkLen, uncompTotal, compTotal) - return b -} - -// Load a binary index. -// A zero value Index can be used or a previous one can be reused. -func (i *Index) Load(b []byte) ([]byte, error) { - if len(b) <= 4+len(S2IndexHeader)+len(S2IndexTrailer) { - return b, io.ErrUnexpectedEOF - } - if b[0] != ChunkTypeIndex { - return b, ErrCorrupt - } - chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16 - b = b[4:] - - // Validate we have enough... - if len(b) < chunkLen { - return b, io.ErrUnexpectedEOF - } - if !bytes.Equal(b[:len(S2IndexHeader)], []byte(S2IndexHeader)) { - return b, ErrUnsupported - } - b = b[len(S2IndexHeader):] - - // Total Uncompressed - if v, n := binary.Varint(b); n <= 0 || v < 0 { - return b, ErrCorrupt - } else { - i.TotalUncompressed = v - b = b[n:] - } - - // Total Compressed - if v, n := binary.Varint(b); n <= 0 { - return b, ErrCorrupt - } else { - i.TotalCompressed = v - b = b[n:] - } - - // Read EstBlockUncomp - if v, n := binary.Varint(b); n <= 0 { - return b, ErrCorrupt - } else { - if v < 0 { - return b, ErrCorrupt - } - i.estBlockUncomp = v - b = b[n:] - } - - var entries int - if v, n := binary.Varint(b); n <= 0 { - return b, ErrCorrupt - } else { - if v < 0 || v > maxIndexEntries { - return b, ErrCorrupt - } - entries = int(v) - b = b[n:] - } - if cap(i.info) < entries { - i.allocInfos(entries) - } - i.info = i.info[:entries] - - if len(b) < 1 { - return b, io.ErrUnexpectedEOF - } - hasUncompressed := b[0] - b = b[1:] - if hasUncompressed&1 != hasUncompressed { - return b, ErrCorrupt - } - - // Add each uncompressed entry - for idx := range i.info { - var uOff int64 - if hasUncompressed != 0 { - // Load delta - if v, n := binary.Varint(b); n <= 0 { - return b, ErrCorrupt - } else { - uOff = v - b = b[n:] - } - } - - if idx > 0 { - prev := i.info[idx-1].uncompressedOffset - uOff += prev + (i.estBlockUncomp) - if uOff <= prev { - return b, ErrCorrupt - } - } - if uOff < 0 { - return b, ErrCorrupt - } - i.info[idx].uncompressedOffset = uOff - } - - // Initial compressed size estimate. - cPredict := i.estBlockUncomp / 2 - - // Add each compressed entry - for idx := range i.info { - var cOff int64 - if v, n := binary.Varint(b); n <= 0 { - return b, ErrCorrupt - } else { - cOff = v - b = b[n:] - } - - if idx > 0 { - // Update compressed size prediction, with half the error. - cPredictNew := cPredict + cOff/2 - - prev := i.info[idx-1].compressedOffset - cOff += prev + cPredict - if cOff <= prev { - return b, ErrCorrupt - } - cPredict = cPredictNew - } - if cOff < 0 { - return b, ErrCorrupt - } - i.info[idx].compressedOffset = cOff - } - if len(b) < 4+len(S2IndexTrailer) { - return b, io.ErrUnexpectedEOF - } - // Skip size... - b = b[4:] - - // Check trailer... - if !bytes.Equal(b[:len(S2IndexTrailer)], []byte(S2IndexTrailer)) { - return b, ErrCorrupt - } - return b[len(S2IndexTrailer):], nil -} - -// LoadStream will load an index from the end of the supplied stream. -// ErrUnsupported will be returned if the signature cannot be found. -// ErrCorrupt will be returned if unexpected values are found. -// io.ErrUnexpectedEOF is returned if there are too few bytes. -// IO errors are returned as-is. -func (i *Index) LoadStream(rs io.ReadSeeker) error { - // Go to end. - _, err := rs.Seek(-10, io.SeekEnd) - if err != nil { - return err - } - var tmp [10]byte - _, err = io.ReadFull(rs, tmp[:]) - if err != nil { - return err - } - // Check trailer... - if !bytes.Equal(tmp[4:4+len(S2IndexTrailer)], []byte(S2IndexTrailer)) { - return ErrUnsupported - } - sz := binary.LittleEndian.Uint32(tmp[:4]) - if sz > maxChunkSize+skippableFrameHeader { - return ErrCorrupt - } - _, err = rs.Seek(-int64(sz), io.SeekEnd) - if err != nil { - return err - } - - // Read index. - buf := make([]byte, sz) - _, err = io.ReadFull(rs, buf) - if err != nil { - return err - } - _, err = i.Load(buf) - return err -} - -// IndexStream will return an index for a stream. -// The stream structure will be checked, but -// data within blocks is not verified. -// The returned index can either be appended to the end of the stream -// or stored separately. -func IndexStream(r io.Reader) ([]byte, error) { - var i Index - var buf [maxChunkSize]byte - var readHeader bool - for { - _, err := io.ReadFull(r, buf[:4]) - if err != nil { - if err == io.EOF { - return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil - } - return nil, err - } - // Start of this chunk. - startChunk := i.TotalCompressed - i.TotalCompressed += 4 - - chunkType := buf[0] - if !readHeader { - if chunkType != chunkTypeStreamIdentifier { - return nil, ErrCorrupt - } - readHeader = true - } - chunkLen := int(buf[1]) | int(buf[2])<<8 | int(buf[3])<<16 - if chunkLen < checksumSize { - return nil, ErrCorrupt - } - - i.TotalCompressed += int64(chunkLen) - _, err = io.ReadFull(r, buf[:chunkLen]) - if err != nil { - return nil, io.ErrUnexpectedEOF - } - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - // Section 4.2. Compressed data (chunk type 0x00). - // Skip checksum. - dLen, err := DecodedLen(buf[checksumSize:]) - if err != nil { - return nil, err - } - if dLen > maxBlockSize { - return nil, ErrCorrupt - } - if i.estBlockUncomp == 0 { - // Use first block for estimate... - i.estBlockUncomp = int64(dLen) - } - err = i.add(startChunk, i.TotalUncompressed) - if err != nil { - return nil, err - } - i.TotalUncompressed += int64(dLen) - continue - case chunkTypeUncompressedData: - n2 := chunkLen - checksumSize - if n2 > maxBlockSize { - return nil, ErrCorrupt - } - if i.estBlockUncomp == 0 { - // Use first block for estimate... - i.estBlockUncomp = int64(n2) - } - err = i.add(startChunk, i.TotalUncompressed) - if err != nil { - return nil, err - } - i.TotalUncompressed += int64(n2) - continue - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - return nil, ErrCorrupt - } - - if string(buf[:len(magicBody)]) != magicBody { - if string(buf[:len(magicBody)]) != magicBodySnappy { - return nil, ErrCorrupt - } - } - - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - return nil, ErrUnsupported - } - if chunkLen > maxChunkSize { - return nil, ErrUnsupported - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - } -} - -// JSON returns the index as JSON text. -func (i *Index) JSON() []byte { - type offset struct { - CompressedOffset int64 `json:"compressed"` - UncompressedOffset int64 `json:"uncompressed"` - } - x := struct { - TotalUncompressed int64 `json:"total_uncompressed"` // Total Uncompressed size if known. Will be -1 if unknown. - TotalCompressed int64 `json:"total_compressed"` // Total Compressed size if known. Will be -1 if unknown. - Offsets []offset `json:"offsets"` - EstBlockUncomp int64 `json:"est_block_uncompressed"` - }{ - TotalUncompressed: i.TotalUncompressed, - TotalCompressed: i.TotalCompressed, - EstBlockUncomp: i.estBlockUncomp, - } - for _, v := range i.info { - x.Offsets = append(x.Offsets, offset{CompressedOffset: v.compressedOffset, UncompressedOffset: v.uncompressedOffset}) - } - b, _ := json.MarshalIndent(x, "", " ") - return b -} - -// RemoveIndexHeaders will trim all headers and trailers from a given index. -// This is expected to save 20 bytes. -// These can be restored using RestoreIndexHeaders. -// This removes a layer of security, but is the most compact representation. -// Returns nil if headers contains errors. -// The returned slice references the provided slice. -func RemoveIndexHeaders(b []byte) []byte { - const save = 4 + len(S2IndexHeader) + len(S2IndexTrailer) + 4 - if len(b) <= save { - return nil - } - if b[0] != ChunkTypeIndex { - return nil - } - chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16 - b = b[4:] - - // Validate we have enough... - if len(b) < chunkLen { - return nil - } - b = b[:chunkLen] - - if !bytes.Equal(b[:len(S2IndexHeader)], []byte(S2IndexHeader)) { - return nil - } - b = b[len(S2IndexHeader):] - if !bytes.HasSuffix(b, []byte(S2IndexTrailer)) { - return nil - } - b = bytes.TrimSuffix(b, []byte(S2IndexTrailer)) - - if len(b) < 4 { - return nil - } - return b[:len(b)-4] -} - -// RestoreIndexHeaders will index restore headers removed by RemoveIndexHeaders. -// No error checking is performed on the input. -// If a 0 length slice is sent, it is returned without modification. -func RestoreIndexHeaders(in []byte) []byte { - if len(in) == 0 { - return in - } - b := make([]byte, 0, 4+len(S2IndexHeader)+len(in)+len(S2IndexTrailer)+4) - b = append(b, ChunkTypeIndex, 0, 0, 0) - b = append(b, []byte(S2IndexHeader)...) - b = append(b, in...) - - var tmp [4]byte - binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(S2IndexTrailer))) - b = append(b, tmp[:4]...) - // Trailer - b = append(b, []byte(S2IndexTrailer)...) - - chunkLen := len(b) - skippableFrameHeader - b[1] = uint8(chunkLen >> 0) - b[2] = uint8(chunkLen >> 8) - b[3] = uint8(chunkLen >> 16) - return b -} diff --git a/vendor/github.com/klauspost/compress/s2/lz4convert.go b/vendor/github.com/klauspost/compress/s2/lz4convert.go deleted file mode 100644 index 46ed908e3..000000000 --- a/vendor/github.com/klauspost/compress/s2/lz4convert.go +++ /dev/null @@ -1,585 +0,0 @@ -// Copyright (c) 2022 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "encoding/binary" - "errors" - "fmt" -) - -// LZ4Converter provides conversion from LZ4 blocks as defined here: -// https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md -type LZ4Converter struct { -} - -// ErrDstTooSmall is returned when provided destination is too small. -var ErrDstTooSmall = errors.New("s2: destination too small") - -// ConvertBlock will convert an LZ4 block and append it as an S2 -// block without block length to dst. -// The uncompressed size is returned as well. -// dst must have capacity to contain the entire compressed block. -func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) { - if len(src) == 0 { - return dst, 0, nil - } - const debug = false - const inline = true - const lz4MinMatch = 4 - - s, d := 0, len(dst) - dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { - res, sz := cvtLZ4BlockAsm(dst[d:], src) - if res < 0 { - const ( - errCorrupt = -1 - errDstTooSmall = -2 - ) - switch res { - case errCorrupt: - return nil, 0, ErrCorrupt - case errDstTooSmall: - return nil, 0, ErrDstTooSmall - default: - return nil, 0, fmt.Errorf("unexpected result: %d", res) - } - } - if d+sz > len(dst) { - return nil, 0, ErrDstTooSmall - } - return dst[:d+sz], res, nil - } - - dLimit := len(dst) - 10 - var lastOffset uint16 - var uncompressed int - if debug { - fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) - } - - for { - if s >= len(src) { - return dst[:d], 0, ErrCorrupt - } - // Read literal info - token := src[s] - ll := int(token >> 4) - ml := int(lz4MinMatch + (token & 0xf)) - - // If upper nibble is 15, literal length is extended - if token >= 0xf0 { - for { - s++ - if s >= len(src) { - if debug { - fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return dst[:d], 0, ErrCorrupt - } - val := src[s] - ll += int(val) - if val != 255 { - break - } - } - } - // Skip past token - if s+ll >= len(src) { - if debug { - fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) - } - return nil, 0, ErrCorrupt - } - s++ - if ll > 0 { - if d+ll > dLimit { - return nil, 0, ErrDstTooSmall - } - if debug { - fmt.Printf("emit %d literals\n", ll) - } - d += emitLiteralGo(dst[d:], src[s:s+ll]) - s += ll - uncompressed += ll - } - - // Check if we are done... - if s == len(src) && ml == lz4MinMatch { - break - } - // 2 byte offset - if s >= len(src)-2 { - if debug { - fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) - } - return nil, 0, ErrCorrupt - } - offset := binary.LittleEndian.Uint16(src[s:]) - s += 2 - if offset == 0 { - if debug { - fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) - } - return nil, 0, ErrCorrupt - } - if int(offset) > uncompressed { - if debug { - fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) - } - return nil, 0, ErrCorrupt - } - - if ml == lz4MinMatch+15 { - for { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - s++ - ml += int(val) - if val != 255 { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - break - } - } - } - if offset == lastOffset { - if debug { - fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset) - } - if !inline { - d += emitRepeat16(dst[d:], offset, ml) - } else { - length := ml - dst := dst[d:] - for len(dst) > 5 { - // Repeat offset, make length cheaper - length -= 4 - if length <= 4 { - dst[0] = uint8(length)<<2 | tagCopy1 - dst[1] = 0 - d += 2 - break - } - if length < 8 && offset < 2048 { - // Encode WITH offset - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 - d += 2 - break - } - if length < (1<<8)+4 { - length -= 4 - dst[2] = uint8(length) - dst[1] = 0 - dst[0] = 5<<2 | tagCopy1 - d += 3 - break - } - if length < (1<<16)+(1<<8) { - length -= 1 << 8 - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 6<<2 | tagCopy1 - d += 4 - break - } - const maxRepeat = (1 << 24) - 1 - length -= 1 << 16 - left := 0 - if length > maxRepeat { - left = length - maxRepeat + 4 - length = maxRepeat - 4 - } - dst[4] = uint8(length >> 16) - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 7<<2 | tagCopy1 - if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break - } - d += 5 - break - } - } - } else { - if debug { - fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) - } - if !inline { - d += emitCopy16(dst[d:], offset, ml) - } else { - length := ml - dst := dst[d:] - for len(dst) > 5 { - // Offset no more than 2 bytes. - if length > 64 { - off := 3 - if offset < 2048 { - // emit 8 bytes as tagCopy1, rest as repeats. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 - length -= 8 - off = 2 - } else { - // Emit a length 60 copy, encoded as 3 bytes. - // Emit remaining as repeat value (minimum 4 bytes). - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 59<<2 | tagCopy2 - length -= 60 - } - // Emit remaining as repeats, at least 4 bytes remain. - d += off + emitRepeat16(dst[off:], offset, length) - break - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = uint8(length-1)<<2 | tagCopy2 - d += 3 - break - } - // Emit the remaining copy, encoded as 2 bytes. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - d += 2 - break - } - } - lastOffset = offset - } - uncompressed += ml - if d > dLimit { - return nil, 0, ErrDstTooSmall - } - } - - return dst[:d], uncompressed, nil -} - -// ConvertBlockSnappy will convert an LZ4 block and append it -// as a Snappy block without block length to dst. -// The uncompressed size is returned as well. -// dst must have capacity to contain the entire compressed block. -func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) { - if len(src) == 0 { - return dst, 0, nil - } - const debug = false - const lz4MinMatch = 4 - - s, d := 0, len(dst) - dst = dst[:cap(dst)] - // Use assembly when possible - if !debug && hasAmd64Asm { - res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src) - if res < 0 { - const ( - errCorrupt = -1 - errDstTooSmall = -2 - ) - switch res { - case errCorrupt: - return nil, 0, ErrCorrupt - case errDstTooSmall: - return nil, 0, ErrDstTooSmall - default: - return nil, 0, fmt.Errorf("unexpected result: %d", res) - } - } - if d+sz > len(dst) { - return nil, 0, ErrDstTooSmall - } - return dst[:d+sz], res, nil - } - - dLimit := len(dst) - 10 - var uncompressed int - if debug { - fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) - } - - for { - if s >= len(src) { - return nil, 0, ErrCorrupt - } - // Read literal info - token := src[s] - ll := int(token >> 4) - ml := int(lz4MinMatch + (token & 0xf)) - - // If upper nibble is 15, literal length is extended - if token >= 0xf0 { - for { - s++ - if s >= len(src) { - if debug { - fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - ll += int(val) - if val != 255 { - break - } - } - } - // Skip past token - if s+ll >= len(src) { - if debug { - fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) - } - return nil, 0, ErrCorrupt - } - s++ - if ll > 0 { - if d+ll > dLimit { - return nil, 0, ErrDstTooSmall - } - if debug { - fmt.Printf("emit %d literals\n", ll) - } - d += emitLiteralGo(dst[d:], src[s:s+ll]) - s += ll - uncompressed += ll - } - - // Check if we are done... - if s == len(src) && ml == lz4MinMatch { - break - } - // 2 byte offset - if s >= len(src)-2 { - if debug { - fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) - } - return nil, 0, ErrCorrupt - } - offset := binary.LittleEndian.Uint16(src[s:]) - s += 2 - if offset == 0 { - if debug { - fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) - } - return nil, 0, ErrCorrupt - } - if int(offset) > uncompressed { - if debug { - fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) - } - return nil, 0, ErrCorrupt - } - - if ml == lz4MinMatch+15 { - for { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - s++ - ml += int(val) - if val != 255 { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - break - } - } - } - if debug { - fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) - } - length := ml - // d += emitCopyNoRepeat(dst[d:], int(offset), ml) - for length > 0 { - if d >= dLimit { - return nil, 0, ErrDstTooSmall - } - - // Offset no more than 2 bytes. - if length > 64 { - // Emit a length 64 copy, encoded as 3 bytes. - dst[d+2] = uint8(offset >> 8) - dst[d+1] = uint8(offset) - dst[d+0] = 63<<2 | tagCopy2 - length -= 64 - d += 3 - continue - } - if length >= 12 || offset >= 2048 || length < 4 { - // Emit the remaining copy, encoded as 3 bytes. - dst[d+2] = uint8(offset >> 8) - dst[d+1] = uint8(offset) - dst[d+0] = uint8(length-1)<<2 | tagCopy2 - d += 3 - break - } - // Emit the remaining copy, encoded as 2 bytes. - dst[d+1] = uint8(offset) - dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - d += 2 - break - } - uncompressed += ml - if d > dLimit { - return nil, 0, ErrDstTooSmall - } - } - - return dst[:d], uncompressed, nil -} - -// emitRepeat writes a repeat chunk and returns the number of bytes written. -// Length must be at least 4 and < 1<<24 -func emitRepeat16(dst []byte, offset uint16, length int) int { - // Repeat offset, make length cheaper - length -= 4 - if length <= 4 { - dst[0] = uint8(length)<<2 | tagCopy1 - dst[1] = 0 - return 2 - } - if length < 8 && offset < 2048 { - // Encode WITH offset - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 - return 2 - } - if length < (1<<8)+4 { - length -= 4 - dst[2] = uint8(length) - dst[1] = 0 - dst[0] = 5<<2 | tagCopy1 - return 3 - } - if length < (1<<16)+(1<<8) { - length -= 1 << 8 - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 6<<2 | tagCopy1 - return 4 - } - const maxRepeat = (1 << 24) - 1 - length -= 1 << 16 - left := 0 - if length > maxRepeat { - left = length - maxRepeat + 4 - length = maxRepeat - 4 - } - dst[4] = uint8(length >> 16) - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 7<<2 | tagCopy1 - if left > 0 { - return 5 + emitRepeat16(dst[5:], offset, left) - } - return 5 -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= math.MaxUint16 -// 4 <= length && length <= math.MaxUint32 -func emitCopy16(dst []byte, offset uint16, length int) int { - // Offset no more than 2 bytes. - if length > 64 { - off := 3 - if offset < 2048 { - // emit 8 bytes as tagCopy1, rest as repeats. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 - length -= 8 - off = 2 - } else { - // Emit a length 60 copy, encoded as 3 bytes. - // Emit remaining as repeat value (minimum 4 bytes). - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 59<<2 | tagCopy2 - length -= 60 - } - // Emit remaining as repeats, at least 4 bytes remain. - return off + emitRepeat16(dst[off:], offset, length) - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = uint8(length-1)<<2 | tagCopy2 - return 3 - } - // Emit the remaining copy, encoded as 2 bytes. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - return 2 -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// -// dst is long enough to hold the encoded bytes -// 0 <= len(lit) && len(lit) <= math.MaxUint32 -func emitLiteralGo(dst, lit []byte) int { - if len(lit) == 0 { - return 0 - } - i, n := 0, uint(len(lit)-1) - switch { - case n < 60: - dst[0] = uint8(n)<<2 | tagLiteral - i = 1 - case n < 1<<8: - dst[1] = uint8(n) - dst[0] = 60<<2 | tagLiteral - i = 2 - case n < 1<<16: - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 61<<2 | tagLiteral - i = 3 - case n < 1<<24: - dst[3] = uint8(n >> 16) - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 62<<2 | tagLiteral - i = 4 - default: - dst[4] = uint8(n >> 24) - dst[3] = uint8(n >> 16) - dst[2] = uint8(n >> 8) - dst[1] = uint8(n) - dst[0] = 63<<2 | tagLiteral - i = 5 - } - return i + copy(dst[i:], lit) -} diff --git a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go b/vendor/github.com/klauspost/compress/s2/lz4sconvert.go deleted file mode 100644 index 000f39719..000000000 --- a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go +++ /dev/null @@ -1,467 +0,0 @@ -// Copyright (c) 2022 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "encoding/binary" - "fmt" -) - -// LZ4sConverter provides conversion from LZ4s. -// (Intel modified LZ4 Blocks) -// https://cdrdv2-public.intel.com/743912/743912-qat-programmers-guide-v2.0.pdf -// LZ4s is a variant of LZ4 block format. LZ4s should be considered as an intermediate compressed block format. -// The LZ4s format is selected when the application sets the compType to CPA_DC_LZ4S in CpaDcSessionSetupData. -// The LZ4s block returned by the Intel® QAT hardware can be used by an external -// software post-processing to generate other compressed data formats. -// The following table lists the differences between LZ4 and LZ4s block format. LZ4s block format uses -// the same high-level formatting as LZ4 block format with the following encoding changes: -// For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte. -// ONLY "Min match of 4 bytes" is supported. -type LZ4sConverter struct { -} - -// ConvertBlock will convert an LZ4s block and append it as an S2 -// block without block length to dst. -// The uncompressed size is returned as well. -// dst must have capacity to contain the entire compressed block. -func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) { - if len(src) == 0 { - return dst, 0, nil - } - const debug = false - const inline = true - const lz4MinMatch = 3 - - s, d := 0, len(dst) - dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { - res, sz := cvtLZ4sBlockAsm(dst[d:], src) - if res < 0 { - const ( - errCorrupt = -1 - errDstTooSmall = -2 - ) - switch res { - case errCorrupt: - return nil, 0, ErrCorrupt - case errDstTooSmall: - return nil, 0, ErrDstTooSmall - default: - return nil, 0, fmt.Errorf("unexpected result: %d", res) - } - } - if d+sz > len(dst) { - return nil, 0, ErrDstTooSmall - } - return dst[:d+sz], res, nil - } - - dLimit := len(dst) - 10 - var lastOffset uint16 - var uncompressed int - if debug { - fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) - } - - for { - if s >= len(src) { - return dst[:d], 0, ErrCorrupt - } - // Read literal info - token := src[s] - ll := int(token >> 4) - ml := int(lz4MinMatch + (token & 0xf)) - - // If upper nibble is 15, literal length is extended - if token >= 0xf0 { - for { - s++ - if s >= len(src) { - if debug { - fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return dst[:d], 0, ErrCorrupt - } - val := src[s] - ll += int(val) - if val != 255 { - break - } - } - } - // Skip past token - if s+ll >= len(src) { - if debug { - fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) - } - return nil, 0, ErrCorrupt - } - s++ - if ll > 0 { - if d+ll > dLimit { - return nil, 0, ErrDstTooSmall - } - if debug { - fmt.Printf("emit %d literals\n", ll) - } - d += emitLiteralGo(dst[d:], src[s:s+ll]) - s += ll - uncompressed += ll - } - - // Check if we are done... - if ml == lz4MinMatch { - if s == len(src) { - break - } - // 0 bytes. - continue - } - // 2 byte offset - if s >= len(src)-2 { - if debug { - fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) - } - return nil, 0, ErrCorrupt - } - offset := binary.LittleEndian.Uint16(src[s:]) - s += 2 - if offset == 0 { - if debug { - fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) - } - return nil, 0, ErrCorrupt - } - if int(offset) > uncompressed { - if debug { - fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) - } - return nil, 0, ErrCorrupt - } - - if ml == lz4MinMatch+15 { - for { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - s++ - ml += int(val) - if val != 255 { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - break - } - } - } - if offset == lastOffset { - if debug { - fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset) - } - if !inline { - d += emitRepeat16(dst[d:], offset, ml) - } else { - length := ml - dst := dst[d:] - for len(dst) > 5 { - // Repeat offset, make length cheaper - length -= 4 - if length <= 4 { - dst[0] = uint8(length)<<2 | tagCopy1 - dst[1] = 0 - d += 2 - break - } - if length < 8 && offset < 2048 { - // Encode WITH offset - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1 - d += 2 - break - } - if length < (1<<8)+4 { - length -= 4 - dst[2] = uint8(length) - dst[1] = 0 - dst[0] = 5<<2 | tagCopy1 - d += 3 - break - } - if length < (1<<16)+(1<<8) { - length -= 1 << 8 - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 6<<2 | tagCopy1 - d += 4 - break - } - const maxRepeat = (1 << 24) - 1 - length -= 1 << 16 - left := 0 - if length > maxRepeat { - left = length - maxRepeat + 4 - length = maxRepeat - 4 - } - dst[4] = uint8(length >> 16) - dst[3] = uint8(length >> 8) - dst[2] = uint8(length >> 0) - dst[1] = 0 - dst[0] = 7<<2 | tagCopy1 - if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break - } - d += 5 - break - } - } - } else { - if debug { - fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) - } - if !inline { - d += emitCopy16(dst[d:], offset, ml) - } else { - length := ml - dst := dst[d:] - for len(dst) > 5 { - // Offset no more than 2 bytes. - if length > 64 { - off := 3 - if offset < 2048 { - // emit 8 bytes as tagCopy1, rest as repeats. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1 - length -= 8 - off = 2 - } else { - // Emit a length 60 copy, encoded as 3 bytes. - // Emit remaining as repeat value (minimum 4 bytes). - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = 59<<2 | tagCopy2 - length -= 60 - } - // Emit remaining as repeats, at least 4 bytes remain. - d += off + emitRepeat16(dst[off:], offset, length) - break - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[2] = uint8(offset >> 8) - dst[1] = uint8(offset) - dst[0] = uint8(length-1)<<2 | tagCopy2 - d += 3 - break - } - // Emit the remaining copy, encoded as 2 bytes. - dst[1] = uint8(offset) - dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - d += 2 - break - } - } - lastOffset = offset - } - uncompressed += ml - if d > dLimit { - return nil, 0, ErrDstTooSmall - } - } - - return dst[:d], uncompressed, nil -} - -// ConvertBlockSnappy will convert an LZ4s block and append it -// as a Snappy block without block length to dst. -// The uncompressed size is returned as well. -// dst must have capacity to contain the entire compressed block. -func (l *LZ4sConverter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) { - if len(src) == 0 { - return dst, 0, nil - } - const debug = false - const lz4MinMatch = 3 - - s, d := 0, len(dst) - dst = dst[:cap(dst)] - // Use assembly when possible - if !debug && hasAmd64Asm { - res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src) - if res < 0 { - const ( - errCorrupt = -1 - errDstTooSmall = -2 - ) - switch res { - case errCorrupt: - return nil, 0, ErrCorrupt - case errDstTooSmall: - return nil, 0, ErrDstTooSmall - default: - return nil, 0, fmt.Errorf("unexpected result: %d", res) - } - } - if d+sz > len(dst) { - return nil, 0, ErrDstTooSmall - } - return dst[:d+sz], res, nil - } - - dLimit := len(dst) - 10 - var uncompressed int - if debug { - fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst)) - } - - for { - if s >= len(src) { - return nil, 0, ErrCorrupt - } - // Read literal info - token := src[s] - ll := int(token >> 4) - ml := int(lz4MinMatch + (token & 0xf)) - - // If upper nibble is 15, literal length is extended - if token >= 0xf0 { - for { - s++ - if s >= len(src) { - if debug { - fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - ll += int(val) - if val != 255 { - break - } - } - } - // Skip past token - if s+ll >= len(src) { - if debug { - fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src)) - } - return nil, 0, ErrCorrupt - } - s++ - if ll > 0 { - if d+ll > dLimit { - return nil, 0, ErrDstTooSmall - } - if debug { - fmt.Printf("emit %d literals\n", ll) - } - d += emitLiteralGo(dst[d:], src[s:s+ll]) - s += ll - uncompressed += ll - } - - // Check if we are done... - if ml == lz4MinMatch { - if s == len(src) { - break - } - // 0 bytes. - continue - } - // 2 byte offset - if s >= len(src)-2 { - if debug { - fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2) - } - return nil, 0, ErrCorrupt - } - offset := binary.LittleEndian.Uint16(src[s:]) - s += 2 - if offset == 0 { - if debug { - fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s) - } - return nil, 0, ErrCorrupt - } - if int(offset) > uncompressed { - if debug { - fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed) - } - return nil, 0, ErrCorrupt - } - - if ml == lz4MinMatch+15 { - for { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - val := src[s] - s++ - ml += int(val) - if val != 255 { - if s >= len(src) { - if debug { - fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src)) - } - return nil, 0, ErrCorrupt - } - break - } - } - } - if debug { - fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset) - } - length := ml - // d += emitCopyNoRepeat(dst[d:], int(offset), ml) - for length > 0 { - if d >= dLimit { - return nil, 0, ErrDstTooSmall - } - - // Offset no more than 2 bytes. - if length > 64 { - // Emit a length 64 copy, encoded as 3 bytes. - dst[d+2] = uint8(offset >> 8) - dst[d+1] = uint8(offset) - dst[d+0] = 63<<2 | tagCopy2 - length -= 64 - d += 3 - continue - } - if length >= 12 || offset >= 2048 || length < 4 { - // Emit the remaining copy, encoded as 3 bytes. - dst[d+2] = uint8(offset >> 8) - dst[d+1] = uint8(offset) - dst[d+0] = uint8(length-1)<<2 | tagCopy2 - d += 3 - break - } - // Emit the remaining copy, encoded as 2 bytes. - dst[d+1] = uint8(offset) - dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - d += 2 - break - } - uncompressed += ml - if d > dLimit { - return nil, 0, ErrDstTooSmall - } - } - - return dst[:d], uncompressed, nil -} diff --git a/vendor/github.com/klauspost/compress/s2/reader.go b/vendor/github.com/klauspost/compress/s2/reader.go deleted file mode 100644 index 8372d752f..000000000 --- a/vendor/github.com/klauspost/compress/s2/reader.go +++ /dev/null @@ -1,1075 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019+ Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "math" - "runtime" - "sync" -) - -// ErrCantSeek is returned if the stream cannot be seeked. -type ErrCantSeek struct { - Reason string -} - -// Error returns the error as string. -func (e ErrCantSeek) Error() string { - return fmt.Sprintf("s2: Can't seek because %s", e.Reason) -} - -// NewReader returns a new Reader that decompresses from r, using the framing -// format described at -// https://github.com/google/snappy/blob/master/framing_format.txt with S2 changes. -func NewReader(r io.Reader, opts ...ReaderOption) *Reader { - nr := Reader{ - r: r, - maxBlock: maxBlockSize, - } - for _, opt := range opts { - if err := opt(&nr); err != nil { - nr.err = err - return &nr - } - } - nr.maxBufSize = MaxEncodedLen(nr.maxBlock) + checksumSize - if nr.lazyBuf > 0 { - nr.buf = make([]byte, MaxEncodedLen(nr.lazyBuf)+checksumSize) - } else { - nr.buf = make([]byte, MaxEncodedLen(defaultBlockSize)+checksumSize) - } - nr.readHeader = nr.ignoreStreamID - nr.paramsOK = true - return &nr -} - -// ReaderOption is an option for creating a decoder. -type ReaderOption func(*Reader) error - -// ReaderMaxBlockSize allows to control allocations if the stream -// has been compressed with a smaller WriterBlockSize, or with the default 1MB. -// Blocks must be this size or smaller to decompress, -// otherwise the decoder will return ErrUnsupported. -// -// For streams compressed with Snappy this can safely be set to 64KB (64 << 10). -// -// Default is the maximum limit of 4MB. -func ReaderMaxBlockSize(blockSize int) ReaderOption { - return func(r *Reader) error { - if blockSize > maxBlockSize || blockSize <= 0 { - return errors.New("s2: block size too large. Must be <= 4MB and > 0") - } - if r.lazyBuf == 0 && blockSize < defaultBlockSize { - r.lazyBuf = blockSize - } - r.maxBlock = blockSize - return nil - } -} - -// ReaderAllocBlock allows to control upfront stream allocations -// and not allocate for frames bigger than this initially. -// If frames bigger than this is seen a bigger buffer will be allocated. -// -// Default is 1MB, which is default output size. -func ReaderAllocBlock(blockSize int) ReaderOption { - return func(r *Reader) error { - if blockSize > maxBlockSize || blockSize < 1024 { - return errors.New("s2: invalid ReaderAllocBlock. Must be <= 4MB and >= 1024") - } - r.lazyBuf = blockSize - return nil - } -} - -// ReaderIgnoreStreamIdentifier will make the reader skip the expected -// stream identifier at the beginning of the stream. -// This can be used when serving a stream that has been forwarded to a specific point. -func ReaderIgnoreStreamIdentifier() ReaderOption { - return func(r *Reader) error { - r.ignoreStreamID = true - return nil - } -} - -// ReaderSkippableCB will register a callback for chuncks with the specified ID. -// ID must be a Reserved skippable chunks ID, 0x80-0xfd (inclusive). -// For each chunk with the ID, the callback is called with the content. -// Any returned non-nil error will abort decompression. -// Only one callback per ID is supported, latest sent will be used. -// You can peek the stream, triggering the callback, by doing a Read with a 0 -// byte buffer. -func ReaderSkippableCB(id uint8, fn func(r io.Reader) error) ReaderOption { - return func(r *Reader) error { - if id < 0x80 || id > 0xfd { - return fmt.Errorf("ReaderSkippableCB: Invalid id provided, must be 0x80-0xfd (inclusive)") - } - r.skippableCB[id-0x80] = fn - return nil - } -} - -// ReaderIgnoreCRC will make the reader skip CRC calculation and checks. -func ReaderIgnoreCRC() ReaderOption { - return func(r *Reader) error { - r.ignoreCRC = true - return nil - } -} - -// Reader is an io.Reader that can read Snappy-compressed bytes. -type Reader struct { - r io.Reader - err error - decoded []byte - buf []byte - skippableCB [0xff - 0x80]func(r io.Reader) error - blockStart int64 // Uncompressed offset at start of current. - index *Index - - // decoded[i:j] contains decoded bytes that have not yet been passed on. - i, j int - // maximum block size allowed. - maxBlock int - // maximum expected buffer size. - maxBufSize int - // alloc a buffer this size if > 0. - lazyBuf int - readHeader bool - paramsOK bool - snappyFrame bool - ignoreStreamID bool - ignoreCRC bool -} - -// GetBufferCapacity returns the capacity of the internal buffer. -// This might be useful to know when reusing the same reader in combination -// with the lazy buffer option. -func (r *Reader) GetBufferCapacity() int { - return cap(r.buf) -} - -// ensureBufferSize will ensure that the buffer can take at least n bytes. -// If false is returned the buffer exceeds maximum allowed size. -func (r *Reader) ensureBufferSize(n int) bool { - if n > r.maxBufSize { - r.err = ErrCorrupt - return false - } - if cap(r.buf) >= n { - return true - } - // Realloc buffer. - r.buf = make([]byte, n) - return true -} - -// Reset discards any buffered data, resets all state, and switches the Snappy -// reader to read from r. This permits reusing a Reader rather than allocating -// a new one. -func (r *Reader) Reset(reader io.Reader) { - if !r.paramsOK { - return - } - r.index = nil - r.r = reader - r.err = nil - r.i = 0 - r.j = 0 - r.blockStart = 0 - r.readHeader = r.ignoreStreamID -} - -func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { - if _, r.err = io.ReadFull(r.r, p); r.err != nil { - if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - } - return false - } - return true -} - -// skippable will skip n bytes. -// If the supplied reader supports seeking that is used. -// tmp is used as a temporary buffer for reading. -// The supplied slice does not need to be the size of the read. -func (r *Reader) skippable(tmp []byte, n int, allowEOF bool, id uint8) (ok bool) { - if id < 0x80 { - r.err = fmt.Errorf("internal error: skippable id < 0x80") - return false - } - if fn := r.skippableCB[id-0x80]; fn != nil { - rd := io.LimitReader(r.r, int64(n)) - r.err = fn(rd) - if r.err != nil { - return false - } - _, r.err = io.CopyBuffer(ioutil.Discard, rd, tmp) - return r.err == nil - } - if rs, ok := r.r.(io.ReadSeeker); ok { - _, err := rs.Seek(int64(n), io.SeekCurrent) - if err == nil { - return true - } - if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - return false - } - } - for n > 0 { - if n < len(tmp) { - tmp = tmp[:n] - } - if _, r.err = io.ReadFull(r.r, tmp); r.err != nil { - if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - } - return false - } - n -= len(tmp) - } - return true -} - -// Read satisfies the io.Reader interface. -func (r *Reader) Read(p []byte) (int, error) { - if r.err != nil { - return 0, r.err - } - for { - if r.i < r.j { - n := copy(p, r.decoded[r.i:r.j]) - r.i += n - return n, nil - } - if !r.readFull(r.buf[:4], true) { - return 0, r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return 0, r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - r.blockStart += int64(r.j) - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - if !r.ensureBufferSize(chunkLen) { - if r.err == nil { - r.err = ErrUnsupported - } - return 0, r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf, false) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - n, err := DecodedLen(buf) - if err != nil { - r.err = err - return 0, r.err - } - if r.snappyFrame && n > maxSnappyBlockSize { - r.err = ErrCorrupt - return 0, r.err - } - - if n > len(r.decoded) { - if n > r.maxBlock { - r.err = ErrCorrupt - return 0, r.err - } - r.decoded = make([]byte, n) - } - if _, err := Decode(r.decoded, buf); err != nil { - r.err = err - return 0, r.err - } - if !r.ignoreCRC && crc(r.decoded[:n]) != checksum { - r.err = ErrCRC - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeUncompressedData: - r.blockStart += int64(r.j) - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - if !r.ensureBufferSize(chunkLen) { - if r.err == nil { - r.err = ErrUnsupported - } - return 0, r.err - } - buf := r.buf[:checksumSize] - if !r.readFull(buf, false) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n := chunkLen - checksumSize - if r.snappyFrame && n > maxSnappyBlockSize { - r.err = ErrCorrupt - return 0, r.err - } - if n > len(r.decoded) { - if n > r.maxBlock { - r.err = ErrCorrupt - return 0, r.err - } - r.decoded = make([]byte, n) - } - if !r.readFull(r.decoded[:n], false) { - return 0, r.err - } - if !r.ignoreCRC && crc(r.decoded[:n]) != checksum { - r.err = ErrCRC - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return 0, r.err - } - if !r.readFull(r.buf[:len(magicBody)], false) { - return 0, r.err - } - if string(r.buf[:len(magicBody)]) != magicBody { - if string(r.buf[:len(magicBody)]) != magicBodySnappy { - r.err = ErrCorrupt - return 0, r.err - } else { - r.snappyFrame = true - } - } else { - r.snappyFrame = false - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - // fmt.Printf("ERR chunktype: 0x%x\n", chunkType) - r.err = ErrUnsupported - return 0, r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if chunkLen > maxChunkSize { - // fmt.Printf("ERR chunkLen: 0x%x\n", chunkLen) - r.err = ErrUnsupported - return 0, r.err - } - - // fmt.Printf("skippable: ID: 0x%x, len: 0x%x\n", chunkType, chunkLen) - if !r.skippable(r.buf, chunkLen, false, chunkType) { - return 0, r.err - } - } -} - -// DecodeConcurrent will decode the full stream to w. -// This function should not be combined with reading, seeking or other operations. -// Up to 'concurrent' goroutines will be used. -// If <= 0, runtime.NumCPU will be used. -// On success the number of bytes decompressed nil and is returned. -// This is mainly intended for bigger streams. -func (r *Reader) DecodeConcurrent(w io.Writer, concurrent int) (written int64, err error) { - if r.i > 0 || r.j > 0 || r.blockStart > 0 { - return 0, errors.New("DecodeConcurrent called after ") - } - if concurrent <= 0 { - concurrent = runtime.NumCPU() - } - - // Write to output - var errMu sync.Mutex - var aErr error - setErr := func(e error) (ok bool) { - errMu.Lock() - defer errMu.Unlock() - if e == nil { - return aErr == nil - } - if aErr == nil { - aErr = e - } - return false - } - hasErr := func() (ok bool) { - errMu.Lock() - v := aErr != nil - errMu.Unlock() - return v - } - - var aWritten int64 - toRead := make(chan []byte, concurrent) - writtenBlocks := make(chan []byte, concurrent) - queue := make(chan chan []byte, concurrent) - reUse := make(chan chan []byte, concurrent) - for i := 0; i < concurrent; i++ { - toRead <- make([]byte, 0, r.maxBufSize) - writtenBlocks <- make([]byte, 0, r.maxBufSize) - reUse <- make(chan []byte, 1) - } - // Writer - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for toWrite := range queue { - entry := <-toWrite - reUse <- toWrite - if hasErr() || entry == nil { - if entry != nil { - writtenBlocks <- entry - } - continue - } - if hasErr() { - writtenBlocks <- entry - continue - } - n, err := w.Write(entry) - want := len(entry) - writtenBlocks <- entry - if err != nil { - setErr(err) - continue - } - if n != want { - setErr(io.ErrShortWrite) - continue - } - aWritten += int64(n) - } - }() - - defer func() { - if r.err != nil { - setErr(r.err) - } else if err != nil { - setErr(err) - } - close(queue) - wg.Wait() - if err == nil { - err = aErr - } - written = aWritten - }() - - // Reader - for !hasErr() { - if !r.readFull(r.buf[:4], true) { - if r.err == io.EOF { - r.err = nil - } - return 0, r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return 0, r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - r.blockStart += int64(r.j) - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - if chunkLen > r.maxBufSize { - r.err = ErrCorrupt - return 0, r.err - } - orgBuf := <-toRead - buf := orgBuf[:chunkLen] - - if !r.readFull(buf, false) { - return 0, r.err - } - - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - n, err := DecodedLen(buf) - if err != nil { - r.err = err - return 0, r.err - } - if r.snappyFrame && n > maxSnappyBlockSize { - r.err = ErrCorrupt - return 0, r.err - } - - if n > r.maxBlock { - r.err = ErrCorrupt - return 0, r.err - } - wg.Add(1) - - decoded := <-writtenBlocks - entry := <-reUse - queue <- entry - go func() { - defer wg.Done() - decoded = decoded[:n] - _, err := Decode(decoded, buf) - toRead <- orgBuf - if err != nil { - writtenBlocks <- decoded - setErr(err) - entry <- nil - return - } - if !r.ignoreCRC && crc(decoded) != checksum { - writtenBlocks <- decoded - setErr(ErrCRC) - entry <- nil - return - } - entry <- decoded - }() - continue - - case chunkTypeUncompressedData: - - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - if chunkLen > r.maxBufSize { - r.err = ErrCorrupt - return 0, r.err - } - // Grab write buffer - orgBuf := <-writtenBlocks - buf := orgBuf[:checksumSize] - if !r.readFull(buf, false) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read content. - n := chunkLen - checksumSize - - if r.snappyFrame && n > maxSnappyBlockSize { - r.err = ErrCorrupt - return 0, r.err - } - if n > r.maxBlock { - r.err = ErrCorrupt - return 0, r.err - } - // Read uncompressed - buf = orgBuf[:n] - if !r.readFull(buf, false) { - return 0, r.err - } - - if !r.ignoreCRC && crc(buf) != checksum { - r.err = ErrCRC - return 0, r.err - } - entry := <-reUse - queue <- entry - entry <- buf - continue - - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return 0, r.err - } - if !r.readFull(r.buf[:len(magicBody)], false) { - return 0, r.err - } - if string(r.buf[:len(magicBody)]) != magicBody { - if string(r.buf[:len(magicBody)]) != magicBodySnappy { - r.err = ErrCorrupt - return 0, r.err - } else { - r.snappyFrame = true - } - } else { - r.snappyFrame = false - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - // fmt.Printf("ERR chunktype: 0x%x\n", chunkType) - r.err = ErrUnsupported - return 0, r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if chunkLen > maxChunkSize { - // fmt.Printf("ERR chunkLen: 0x%x\n", chunkLen) - r.err = ErrUnsupported - return 0, r.err - } - - // fmt.Printf("skippable: ID: 0x%x, len: 0x%x\n", chunkType, chunkLen) - if !r.skippable(r.buf, chunkLen, false, chunkType) { - return 0, r.err - } - } - return 0, r.err -} - -// Skip will skip n bytes forward in the decompressed output. -// For larger skips this consumes less CPU and is faster than reading output and discarding it. -// CRC is not checked on skipped blocks. -// io.ErrUnexpectedEOF is returned if the stream ends before all bytes have been skipped. -// If a decoding error is encountered subsequent calls to Read will also fail. -func (r *Reader) Skip(n int64) error { - if n < 0 { - return errors.New("attempted negative skip") - } - if r.err != nil { - return r.err - } - - for n > 0 { - if r.i < r.j { - // Skip in buffer. - // decoded[i:j] contains decoded bytes that have not yet been passed on. - left := int64(r.j - r.i) - if left >= n { - tmp := int64(r.i) + n - if tmp > math.MaxInt32 { - return errors.New("s2: internal overflow in skip") - } - r.i = int(tmp) - return nil - } - n -= int64(r.j - r.i) - r.i = r.j - } - - // Buffer empty; read blocks until we have content. - if !r.readFull(r.buf[:4], true) { - if r.err == io.EOF { - r.err = io.ErrUnexpectedEOF - } - return r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - r.blockStart += int64(r.j) - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return r.err - } - if !r.ensureBufferSize(chunkLen) { - if r.err == nil { - r.err = ErrUnsupported - } - return r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf, false) { - return r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - dLen, err := DecodedLen(buf) - if err != nil { - r.err = err - return r.err - } - if dLen > r.maxBlock { - r.err = ErrCorrupt - return r.err - } - // Check if destination is within this block - if int64(dLen) > n { - if len(r.decoded) < dLen { - r.decoded = make([]byte, dLen) - } - if _, err := Decode(r.decoded, buf); err != nil { - r.err = err - return r.err - } - if crc(r.decoded[:dLen]) != checksum { - r.err = ErrCorrupt - return r.err - } - } else { - // Skip block completely - n -= int64(dLen) - r.blockStart += int64(dLen) - dLen = 0 - } - r.i, r.j = 0, dLen - continue - case chunkTypeUncompressedData: - r.blockStart += int64(r.j) - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return r.err - } - if !r.ensureBufferSize(chunkLen) { - if r.err != nil { - r.err = ErrUnsupported - } - return r.err - } - buf := r.buf[:checksumSize] - if !r.readFull(buf, false) { - return r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n2 := chunkLen - checksumSize - if n2 > len(r.decoded) { - if n2 > r.maxBlock { - r.err = ErrCorrupt - return r.err - } - r.decoded = make([]byte, n2) - } - if !r.readFull(r.decoded[:n2], false) { - return r.err - } - if int64(n2) < n { - if crc(r.decoded[:n2]) != checksum { - r.err = ErrCorrupt - return r.err - } - } - r.i, r.j = 0, n2 - continue - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return r.err - } - if !r.readFull(r.buf[:len(magicBody)], false) { - return r.err - } - if string(r.buf[:len(magicBody)]) != magicBody { - if string(r.buf[:len(magicBody)]) != magicBodySnappy { - r.err = ErrCorrupt - return r.err - } - } - - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - r.err = ErrUnsupported - return r.err - } - if chunkLen > maxChunkSize { - r.err = ErrUnsupported - return r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if !r.skippable(r.buf, chunkLen, false, chunkType) { - return r.err - } - } - return nil -} - -// ReadSeeker provides random or forward seeking in compressed content. -// See Reader.ReadSeeker -type ReadSeeker struct { - *Reader - readAtMu sync.Mutex -} - -// ReadSeeker will return an io.ReadSeeker and io.ReaderAt -// compatible version of the reader. -// If 'random' is specified the returned io.Seeker can be used for -// random seeking, otherwise only forward seeking is supported. -// Enabling random seeking requires the original input to support -// the io.Seeker interface. -// A custom index can be specified which will be used if supplied. -// When using a custom index, it will not be read from the input stream. -// The ReadAt position will affect regular reads and the current position of Seek. -// So using Read after ReadAt will continue from where the ReadAt stopped. -// No functions should be used concurrently. -// The returned ReadSeeker contains a shallow reference to the existing Reader, -// meaning changes performed to one is reflected in the other. -func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) { - // Read index if provided. - if len(index) != 0 { - if r.index == nil { - r.index = &Index{} - } - if _, err := r.index.Load(index); err != nil { - return nil, ErrCantSeek{Reason: "loading index returned: " + err.Error()} - } - } - - // Check if input is seekable - rs, ok := r.r.(io.ReadSeeker) - if !ok { - if !random { - return &ReadSeeker{Reader: r}, nil - } - return nil, ErrCantSeek{Reason: "input stream isn't seekable"} - } - - if r.index != nil { - // Seekable and index, ok... - return &ReadSeeker{Reader: r}, nil - } - - // Load from stream. - r.index = &Index{} - - // Read current position. - pos, err := rs.Seek(0, io.SeekCurrent) - if err != nil { - return nil, ErrCantSeek{Reason: "seeking input returned: " + err.Error()} - } - err = r.index.LoadStream(rs) - if err != nil { - if err == ErrUnsupported { - // If we don't require random seeking, reset input and return. - if !random { - _, err = rs.Seek(pos, io.SeekStart) - if err != nil { - return nil, ErrCantSeek{Reason: "resetting stream returned: " + err.Error()} - } - r.index = nil - return &ReadSeeker{Reader: r}, nil - } - return nil, ErrCantSeek{Reason: "input stream does not contain an index"} - } - return nil, ErrCantSeek{Reason: "reading index returned: " + err.Error()} - } - - // reset position. - _, err = rs.Seek(pos, io.SeekStart) - if err != nil { - return nil, ErrCantSeek{Reason: "seeking input returned: " + err.Error()} - } - return &ReadSeeker{Reader: r}, nil -} - -// Seek allows seeking in compressed data. -func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) { - if r.err != nil { - if !errors.Is(r.err, io.EOF) { - return 0, r.err - } - // Reset on EOF - r.err = nil - } - - // Calculate absolute offset. - absOffset := offset - - switch whence { - case io.SeekStart: - case io.SeekCurrent: - absOffset = r.blockStart + int64(r.i) + offset - case io.SeekEnd: - if r.index == nil { - return 0, ErrUnsupported - } - absOffset = r.index.TotalUncompressed + offset - default: - r.err = ErrUnsupported - return 0, r.err - } - - if absOffset < 0 { - return 0, errors.New("seek before start of file") - } - - if !r.readHeader { - // Make sure we read the header. - _, r.err = r.Read([]byte{}) - if r.err != nil { - return 0, r.err - } - } - - // If we are inside current block no need to seek. - // This includes no offset changes. - if absOffset >= r.blockStart && absOffset < r.blockStart+int64(r.j) { - r.i = int(absOffset - r.blockStart) - return r.blockStart + int64(r.i), nil - } - - rs, ok := r.r.(io.ReadSeeker) - if r.index == nil || !ok { - currOffset := r.blockStart + int64(r.i) - if absOffset >= currOffset { - err := r.Skip(absOffset - currOffset) - return r.blockStart + int64(r.i), err - } - return 0, ErrUnsupported - } - - // We can seek and we have an index. - c, u, err := r.index.Find(absOffset) - if err != nil { - return r.blockStart + int64(r.i), err - } - - // Seek to next block - _, err = rs.Seek(c, io.SeekStart) - if err != nil { - return 0, err - } - - r.i = r.j // Remove rest of current block. - r.blockStart = u - int64(r.j) // Adjust current block start for accounting. - if u < absOffset { - // Forward inside block - return absOffset, r.Skip(absOffset - u) - } - if u > absOffset { - return 0, fmt.Errorf("s2 seek: (internal error) u (%d) > absOffset (%d)", u, absOffset) - } - return absOffset, nil -} - -// ReadAt reads len(p) bytes into p starting at offset off in the -// underlying input source. It returns the number of bytes -// read (0 <= n <= len(p)) and any error encountered. -// -// When ReadAt returns n < len(p), it returns a non-nil error -// explaining why more bytes were not returned. In this respect, -// ReadAt is stricter than Read. -// -// Even if ReadAt returns n < len(p), it may use all of p as scratch -// space during the call. If some data is available but not len(p) bytes, -// ReadAt blocks until either all the data is available or an error occurs. -// In this respect ReadAt is different from Read. -// -// If the n = len(p) bytes returned by ReadAt are at the end of the -// input source, ReadAt may return either err == EOF or err == nil. -// -// If ReadAt is reading from an input source with a seek offset, -// ReadAt should not affect nor be affected by the underlying -// seek offset. -// -// Clients of ReadAt can execute parallel ReadAt calls on the -// same input source. This is however not recommended. -func (r *ReadSeeker) ReadAt(p []byte, offset int64) (int, error) { - r.readAtMu.Lock() - defer r.readAtMu.Unlock() - _, err := r.Seek(offset, io.SeekStart) - if err != nil { - return 0, err - } - n := 0 - for n < len(p) { - n2, err := r.Read(p[n:]) - if err != nil { - // This will include io.EOF - return n + n2, err - } - n += n2 - } - return n, nil -} - -// ReadByte satisfies the io.ByteReader interface. -func (r *Reader) ReadByte() (byte, error) { - if r.err != nil { - return 0, r.err - } - if r.i < r.j { - c := r.decoded[r.i] - r.i++ - return c, nil - } - var tmp [1]byte - for i := 0; i < 10; i++ { - n, err := r.Read(tmp[:]) - if err != nil { - return 0, err - } - if n == 1 { - return tmp[0], nil - } - } - return 0, io.ErrNoProgress -} - -// SkippableCB will register a callback for chunks with the specified ID. -// ID must be a Reserved skippable chunks ID, 0x80-0xfd (inclusive). -// For each chunk with the ID, the callback is called with the content. -// Any returned non-nil error will abort decompression. -// Only one callback per ID is supported, latest sent will be used. -// Sending a nil function will disable previous callbacks. -// You can peek the stream, triggering the callback, by doing a Read with a 0 -// byte buffer. -func (r *Reader) SkippableCB(id uint8, fn func(r io.Reader) error) error { - if id < 0x80 || id >= chunkTypePadding { - return fmt.Errorf("ReaderSkippableCB: Invalid id provided, must be 0x80-0xfe (inclusive)") - } - r.skippableCB[id-0x80] = fn - return nil -} diff --git a/vendor/github.com/klauspost/compress/s2/s2.go b/vendor/github.com/klauspost/compress/s2/s2.go deleted file mode 100644 index cbd1ed64d..000000000 --- a/vendor/github.com/klauspost/compress/s2/s2.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package s2 implements the S2 compression format. -// -// S2 is an extension of Snappy. Similar to Snappy S2 is aimed for high throughput, -// which is why it features concurrent compression for bigger payloads. -// -// Decoding is compatible with Snappy compressed content, -// but content compressed with S2 cannot be decompressed by Snappy. -// -// For more information on Snappy/S2 differences see README in: https://github.com/klauspost/compress/tree/master/s2 -// -// There are actually two S2 formats: block and stream. They are related, -// but different: trying to decompress block-compressed data as a S2 stream -// will fail, and vice versa. The block format is the Decode and Encode -// functions and the stream format is the Reader and Writer types. -// -// A "better" compression option is available. This will trade some compression -// speed -// -// The block format, the more common case, is used when the complete size (the -// number of bytes) of the original data is known upfront, at the time -// compression starts. The stream format, also known as the framing format, is -// for when that isn't always true. -// -// Blocks to not offer much data protection, so it is up to you to -// add data validation of decompressed blocks. -// -// Streams perform CRC validation of the decompressed data. -// Stream compression will also be performed on multiple CPU cores concurrently -// significantly improving throughput. -package s2 - -import ( - "bytes" - "hash/crc32" - - "github.com/klauspost/compress/internal/race" -) - -/* -Each encoded block begins with the varint-encoded length of the decoded data, -followed by a sequence of chunks. Chunks begin and end on byte boundaries. The -first byte of each chunk is broken into its 2 least and 6 most significant bits -called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. -Zero means a literal tag. All other values mean a copy tag. - -For literal tags: - - If m < 60, the next 1 + m bytes are literal bytes. - - Otherwise, let n be the little-endian unsigned integer denoted by the next - m - 59 bytes. The next 1 + n bytes after that are literal bytes. - -For copy tags, length bytes are copied from offset bytes ago, in the style of -Lempel-Ziv compression algorithms. In particular: - - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). - The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 - of the offset. The next byte is bits 0-7 of the offset. - - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). - The length is 1 + m. The offset is the little-endian unsigned integer - denoted by the next 2 bytes. - - For l == 3, the offset ranges in [0, 1<<32) and the length in - [1, 65). The length is 1 + m. The offset is the little-endian unsigned - integer denoted by the next 4 bytes. -*/ -const ( - tagLiteral = 0x00 - tagCopy1 = 0x01 - tagCopy2 = 0x02 - tagCopy4 = 0x03 -) - -const ( - checksumSize = 4 - chunkHeaderSize = 4 - magicChunk = "\xff\x06\x00\x00" + magicBody - magicChunkSnappy = "\xff\x06\x00\x00" + magicBodySnappy - magicBodySnappy = "sNaPpY" - magicBody = "S2sTwO" - - // maxBlockSize is the maximum size of the input to encodeBlock. - // - // For the framing format (Writer type instead of Encode function), - // this is the maximum uncompressed size of a block. - maxBlockSize = 4 << 20 - - // minBlockSize is the minimum size of block setting when creating a writer. - minBlockSize = 4 << 10 - - skippableFrameHeader = 4 - maxChunkSize = 1<<24 - 1 // 16777215 - - // Default block size - defaultBlockSize = 1 << 20 - - // maxSnappyBlockSize is the maximum snappy block size. - maxSnappyBlockSize = 1 << 16 - - obufHeaderLen = checksumSize + chunkHeaderSize -) - -const ( - chunkTypeCompressedData = 0x00 - chunkTypeUncompressedData = 0x01 - ChunkTypeIndex = 0x99 - chunkTypePadding = 0xfe - chunkTypeStreamIdentifier = 0xff -) - -var ( - crcTable = crc32.MakeTable(crc32.Castagnoli) - magicChunkSnappyBytes = []byte(magicChunkSnappy) // Can be passed to functions where it escapes. - magicChunkBytes = []byte(magicChunk) // Can be passed to functions where it escapes. -) - -// crc implements the checksum specified in section 3 of -// https://github.com/google/snappy/blob/master/framing_format.txt -func crc(b []byte) uint32 { - race.ReadSlice(b) - - c := crc32.Update(0, crcTable, b) - return c>>15 | c<<17 + 0xa282ead8 -} - -// literalExtraSize returns the extra size of encoding n literals. -// n should be >= 0 and <= math.MaxUint32. -func literalExtraSize(n int64) int64 { - if n == 0 { - return 0 - } - switch { - case n < 60: - return 1 - case n < 1<<8: - return 2 - case n < 1<<16: - return 3 - case n < 1<<24: - return 4 - default: - return 5 - } -} - -type byter interface { - Bytes() []byte -} - -var _ byter = &bytes.Buffer{} diff --git a/vendor/github.com/klauspost/compress/s2/writer.go b/vendor/github.com/klauspost/compress/s2/writer.go deleted file mode 100644 index fd15078f7..000000000 --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ /dev/null @@ -1,1064 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Copyright (c) 2019+ Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package s2 - -import ( - "crypto/rand" - "encoding/binary" - "errors" - "fmt" - "io" - "runtime" - "sync" - - "github.com/klauspost/compress/internal/race" -) - -const ( - levelUncompressed = iota + 1 - levelFast - levelBetter - levelBest -) - -// NewWriter returns a new Writer that compresses to w, using the -// framing format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -// -// Users must call Close to guarantee all data has been forwarded to -// the underlying io.Writer and that resources are released. -// They may also call Flush zero or more times before calling Close. -func NewWriter(w io.Writer, opts ...WriterOption) *Writer { - w2 := Writer{ - blockSize: defaultBlockSize, - concurrency: runtime.GOMAXPROCS(0), - randSrc: rand.Reader, - level: levelFast, - } - for _, opt := range opts { - if err := opt(&w2); err != nil { - w2.errState = err - return &w2 - } - } - w2.obufLen = obufHeaderLen + MaxEncodedLen(w2.blockSize) - w2.paramsOK = true - w2.ibuf = make([]byte, 0, w2.blockSize) - w2.buffers.New = func() interface{} { - return make([]byte, w2.obufLen) - } - w2.Reset(w) - return &w2 -} - -// Writer is an io.Writer that can write Snappy-compressed bytes. -type Writer struct { - errMu sync.Mutex - errState error - - // ibuf is a buffer for the incoming (uncompressed) bytes. - ibuf []byte - - blockSize int - obufLen int - concurrency int - written int64 - uncompWritten int64 // Bytes sent to compression - output chan chan result - buffers sync.Pool - pad int - - writer io.Writer - randSrc io.Reader - writerWg sync.WaitGroup - index Index - customEnc func(dst, src []byte) int - - // wroteStreamHeader is whether we have written the stream header. - wroteStreamHeader bool - paramsOK bool - snappy bool - flushOnWrite bool - appendIndex bool - bufferCB func([]byte) - level uint8 -} - -type result struct { - b []byte - // return when writing - ret []byte - // Uncompressed start offset - startOffset int64 -} - -// err returns the previously set error. -// If no error has been set it is set to err if not nil. -func (w *Writer) err(err error) error { - w.errMu.Lock() - errSet := w.errState - if errSet == nil && err != nil { - w.errState = err - errSet = err - } - w.errMu.Unlock() - return errSet -} - -// Reset discards the writer's state and switches the Snappy writer to write to w. -// This permits reusing a Writer rather than allocating a new one. -func (w *Writer) Reset(writer io.Writer) { - if !w.paramsOK { - return - } - // Close previous writer, if any. - if w.output != nil { - close(w.output) - w.writerWg.Wait() - w.output = nil - } - w.errState = nil - w.ibuf = w.ibuf[:0] - w.wroteStreamHeader = false - w.written = 0 - w.writer = writer - w.uncompWritten = 0 - w.index.reset(w.blockSize) - - // If we didn't get a writer, stop here. - if writer == nil { - return - } - // If no concurrency requested, don't spin up writer goroutine. - if w.concurrency == 1 { - return - } - - toWrite := make(chan chan result, w.concurrency) - w.output = toWrite - w.writerWg.Add(1) - - // Start a writer goroutine that will write all output in order. - go func() { - defer w.writerWg.Done() - - // Get a queued write. - for write := range toWrite { - // Wait for the data to be available. - input := <-write - if input.ret != nil && w.bufferCB != nil { - w.bufferCB(input.ret) - input.ret = nil - } - in := input.b - if len(in) > 0 { - if w.err(nil) == nil { - // Don't expose data from previous buffers. - toWrite := in[:len(in):len(in)] - // Write to output. - n, err := writer.Write(toWrite) - if err == nil && n != len(toWrite) { - err = io.ErrShortBuffer - } - _ = w.err(err) - w.err(w.index.add(w.written, input.startOffset)) - w.written += int64(n) - } - } - if cap(in) >= w.obufLen { - w.buffers.Put(in) - } - // close the incoming write request. - // This can be used for synchronizing flushes. - close(write) - } - }() -} - -// Write satisfies the io.Writer interface. -func (w *Writer) Write(p []byte) (nRet int, errRet error) { - if err := w.err(nil); err != nil { - return 0, err - } - if w.flushOnWrite { - return w.write(p) - } - // If we exceed the input buffer size, start writing - for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err(nil) == nil { - var n int - if len(w.ibuf) == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, _ = w.write(p) - } else { - n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - w.write(w.ibuf) - w.ibuf = w.ibuf[:0] - } - nRet += n - p = p[n:] - } - if err := w.err(nil); err != nil { - return nRet, err - } - // p should always be able to fit into w.ibuf now. - n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - nRet += n - return nRet, nil -} - -// ReadFrom implements the io.ReaderFrom interface. -// Using this is typically more efficient since it avoids a memory copy. -// ReadFrom reads data from r until EOF or error. -// The return value n is the number of bytes read. -// Any error except io.EOF encountered during the read is also returned. -func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { - if err := w.err(nil); err != nil { - return 0, err - } - if len(w.ibuf) > 0 { - err := w.AsyncFlush() - if err != nil { - return 0, err - } - } - if br, ok := r.(byter); ok { - buf := br.Bytes() - if err := w.EncodeBuffer(buf); err != nil { - return 0, err - } - return int64(len(buf)), w.AsyncFlush() - } - for { - inbuf := w.buffers.Get().([]byte)[:w.blockSize+obufHeaderLen] - n2, err := io.ReadFull(r, inbuf[obufHeaderLen:]) - if err != nil { - if err == io.ErrUnexpectedEOF { - err = io.EOF - } - if err != io.EOF { - return n, w.err(err) - } - } - if n2 == 0 { - if cap(inbuf) >= w.obufLen { - w.buffers.Put(inbuf) - } - break - } - n += int64(n2) - err2 := w.writeFull(inbuf[:n2+obufHeaderLen]) - if w.err(err2) != nil { - break - } - - if err != nil { - // We got EOF and wrote everything - break - } - } - - return n, w.err(nil) -} - -// AddSkippableBlock will add a skippable block to the stream. -// The ID must be 0x80-0xfe (inclusive). -// Length of the skippable block must be <= 16777215 bytes. -func (w *Writer) AddSkippableBlock(id uint8, data []byte) (err error) { - if err := w.err(nil); err != nil { - return err - } - if len(data) == 0 { - return nil - } - if id < 0x80 || id > chunkTypePadding { - return fmt.Errorf("invalid skippable block id %x", id) - } - if len(data) > maxChunkSize { - return fmt.Errorf("skippable block excessed maximum size") - } - var header [4]byte - chunkLen := len(data) - header[0] = id - header[1] = uint8(chunkLen >> 0) - header[2] = uint8(chunkLen >> 8) - header[3] = uint8(chunkLen >> 16) - if w.concurrency == 1 { - write := func(b []byte) error { - n, err := w.writer.Write(b) - if err = w.err(err); err != nil { - return err - } - if n != len(b) { - return w.err(io.ErrShortWrite) - } - w.written += int64(n) - return w.err(nil) - } - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - if w.snappy { - if err := write([]byte(magicChunkSnappy)); err != nil { - return err - } - } else { - if err := write([]byte(magicChunk)); err != nil { - return err - } - } - } - if err := write(header[:]); err != nil { - return err - } - return write(data) - } - - // Create output... - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - hWriter := make(chan result) - w.output <- hWriter - if w.snappy { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes} - } else { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes} - } - } - - // Copy input. - inbuf := w.buffers.Get().([]byte)[:4] - copy(inbuf, header[:]) - inbuf = append(inbuf, data...) - - output := make(chan result, 1) - // Queue output. - w.output <- output - output <- result{startOffset: w.uncompWritten, b: inbuf} - - return nil -} - -// EncodeBuffer will add a buffer to the stream. -// This is the fastest way to encode a stream, -// but the input buffer cannot be written to by the caller -// until Flush or Close has been called when concurrency != 1. -// -// Use the WriterBufferDone to receive a callback when the buffer is done -// Processing. -// -// Note that input is not buffered. -// This means that each write will result in discrete blocks being created. -// For buffered writes, use the regular Write function. -func (w *Writer) EncodeBuffer(buf []byte) (err error) { - if err := w.err(nil); err != nil { - return err - } - - if w.flushOnWrite { - _, err := w.write(buf) - return err - } - // Flush queued data first. - if len(w.ibuf) > 0 { - err := w.AsyncFlush() - if err != nil { - return err - } - } - if w.concurrency == 1 { - _, err := w.writeSync(buf) - if w.bufferCB != nil { - w.bufferCB(buf) - } - return err - } - - // Spawn goroutine and write block to output channel. - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - hWriter := make(chan result) - w.output <- hWriter - if w.snappy { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes} - } else { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes} - } - } - orgBuf := buf - for len(buf) > 0 { - // Cut input. - uncompressed := buf - if len(uncompressed) > w.blockSize { - uncompressed = uncompressed[:w.blockSize] - } - buf = buf[len(uncompressed):] - // Get an output buffer. - obuf := w.buffers.Get().([]byte)[:len(uncompressed)+obufHeaderLen] - race.WriteSlice(obuf) - - output := make(chan result) - // Queue output now, so we keep order. - w.output <- output - res := result{ - startOffset: w.uncompWritten, - } - w.uncompWritten += int64(len(uncompressed)) - if len(buf) == 0 && w.bufferCB != nil { - res.ret = orgBuf - } - go func() { - race.ReadSlice(uncompressed) - - checksum := crc(uncompressed) - - // Set to uncompressed. - chunkType := uint8(chunkTypeUncompressedData) - chunkLen := 4 + len(uncompressed) - - // Attempt compressing. - n := binary.PutUvarint(obuf[obufHeaderLen:], uint64(len(uncompressed))) - n2 := w.encodeBlock(obuf[obufHeaderLen+n:], uncompressed) - - // Check if we should use this, or store as uncompressed instead. - if n2 > 0 { - chunkType = uint8(chunkTypeCompressedData) - chunkLen = 4 + n + n2 - obuf = obuf[:obufHeaderLen+n+n2] - } else { - // copy uncompressed - copy(obuf[obufHeaderLen:], uncompressed) - } - - // Fill in the per-chunk header that comes before the body. - obuf[0] = chunkType - obuf[1] = uint8(chunkLen >> 0) - obuf[2] = uint8(chunkLen >> 8) - obuf[3] = uint8(chunkLen >> 16) - obuf[4] = uint8(checksum >> 0) - obuf[5] = uint8(checksum >> 8) - obuf[6] = uint8(checksum >> 16) - obuf[7] = uint8(checksum >> 24) - - // Queue final output. - res.b = obuf - output <- res - }() - } - return nil -} - -func (w *Writer) encodeBlock(obuf, uncompressed []byte) int { - if w.customEnc != nil { - if ret := w.customEnc(obuf, uncompressed); ret >= 0 { - return ret - } - } - if w.snappy { - switch w.level { - case levelFast: - return encodeBlockSnappy(obuf, uncompressed) - case levelBetter: - return encodeBlockBetterSnappy(obuf, uncompressed) - case levelBest: - return encodeBlockBestSnappy(obuf, uncompressed) - } - return 0 - } - switch w.level { - case levelFast: - return encodeBlock(obuf, uncompressed) - case levelBetter: - return encodeBlockBetter(obuf, uncompressed) - case levelBest: - return encodeBlockBest(obuf, uncompressed, nil) - } - return 0 -} - -func (w *Writer) write(p []byte) (nRet int, errRet error) { - if err := w.err(nil); err != nil { - return 0, err - } - if w.concurrency == 1 { - return w.writeSync(p) - } - - // Spawn goroutine and write block to output channel. - for len(p) > 0 { - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - hWriter := make(chan result) - w.output <- hWriter - if w.snappy { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes} - } else { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes} - } - } - - var uncompressed []byte - if len(p) > w.blockSize { - uncompressed, p = p[:w.blockSize], p[w.blockSize:] - } else { - uncompressed, p = p, nil - } - - // Copy input. - // If the block is incompressible, this is used for the result. - inbuf := w.buffers.Get().([]byte)[:len(uncompressed)+obufHeaderLen] - obuf := w.buffers.Get().([]byte)[:w.obufLen] - copy(inbuf[obufHeaderLen:], uncompressed) - uncompressed = inbuf[obufHeaderLen:] - - output := make(chan result) - // Queue output now, so we keep order. - w.output <- output - res := result{ - startOffset: w.uncompWritten, - } - w.uncompWritten += int64(len(uncompressed)) - - go func() { - checksum := crc(uncompressed) - - // Set to uncompressed. - chunkType := uint8(chunkTypeUncompressedData) - chunkLen := 4 + len(uncompressed) - - // Attempt compressing. - n := binary.PutUvarint(obuf[obufHeaderLen:], uint64(len(uncompressed))) - n2 := w.encodeBlock(obuf[obufHeaderLen+n:], uncompressed) - - // Check if we should use this, or store as uncompressed instead. - if n2 > 0 { - chunkType = uint8(chunkTypeCompressedData) - chunkLen = 4 + n + n2 - obuf = obuf[:obufHeaderLen+n+n2] - } else { - // Use input as output. - obuf, inbuf = inbuf, obuf - } - - // Fill in the per-chunk header that comes before the body. - obuf[0] = chunkType - obuf[1] = uint8(chunkLen >> 0) - obuf[2] = uint8(chunkLen >> 8) - obuf[3] = uint8(chunkLen >> 16) - obuf[4] = uint8(checksum >> 0) - obuf[5] = uint8(checksum >> 8) - obuf[6] = uint8(checksum >> 16) - obuf[7] = uint8(checksum >> 24) - - // Queue final output. - res.b = obuf - output <- res - - // Put unused buffer back in pool. - w.buffers.Put(inbuf) - }() - nRet += len(uncompressed) - } - return nRet, nil -} - -// writeFull is a special version of write that will always write the full buffer. -// Data to be compressed should start at offset obufHeaderLen and fill the remainder of the buffer. -// The data will be written as a single block. -// The caller is not allowed to use inbuf after this function has been called. -func (w *Writer) writeFull(inbuf []byte) (errRet error) { - if err := w.err(nil); err != nil { - return err - } - - if w.concurrency == 1 { - _, err := w.writeSync(inbuf[obufHeaderLen:]) - if cap(inbuf) >= w.obufLen { - w.buffers.Put(inbuf) - } - return err - } - - // Spawn goroutine and write block to output channel. - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - hWriter := make(chan result) - w.output <- hWriter - if w.snappy { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes} - } else { - hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes} - } - } - - // Get an output buffer. - obuf := w.buffers.Get().([]byte)[:w.obufLen] - uncompressed := inbuf[obufHeaderLen:] - - output := make(chan result) - // Queue output now, so we keep order. - w.output <- output - res := result{ - startOffset: w.uncompWritten, - } - w.uncompWritten += int64(len(uncompressed)) - - go func() { - checksum := crc(uncompressed) - - // Set to uncompressed. - chunkType := uint8(chunkTypeUncompressedData) - chunkLen := 4 + len(uncompressed) - - // Attempt compressing. - n := binary.PutUvarint(obuf[obufHeaderLen:], uint64(len(uncompressed))) - n2 := w.encodeBlock(obuf[obufHeaderLen+n:], uncompressed) - - // Check if we should use this, or store as uncompressed instead. - if n2 > 0 { - chunkType = uint8(chunkTypeCompressedData) - chunkLen = 4 + n + n2 - obuf = obuf[:obufHeaderLen+n+n2] - } else { - // Use input as output. - obuf, inbuf = inbuf, obuf - } - - // Fill in the per-chunk header that comes before the body. - obuf[0] = chunkType - obuf[1] = uint8(chunkLen >> 0) - obuf[2] = uint8(chunkLen >> 8) - obuf[3] = uint8(chunkLen >> 16) - obuf[4] = uint8(checksum >> 0) - obuf[5] = uint8(checksum >> 8) - obuf[6] = uint8(checksum >> 16) - obuf[7] = uint8(checksum >> 24) - - // Queue final output. - res.b = obuf - output <- res - - // Put unused buffer back in pool. - w.buffers.Put(inbuf) - }() - return nil -} - -func (w *Writer) writeSync(p []byte) (nRet int, errRet error) { - if err := w.err(nil); err != nil { - return 0, err - } - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - var n int - var err error - if w.snappy { - n, err = w.writer.Write(magicChunkSnappyBytes) - } else { - n, err = w.writer.Write(magicChunkBytes) - } - if err != nil { - return 0, w.err(err) - } - if n != len(magicChunk) { - return 0, w.err(io.ErrShortWrite) - } - w.written += int64(n) - } - - for len(p) > 0 { - var uncompressed []byte - if len(p) > w.blockSize { - uncompressed, p = p[:w.blockSize], p[w.blockSize:] - } else { - uncompressed, p = p, nil - } - - obuf := w.buffers.Get().([]byte)[:w.obufLen] - checksum := crc(uncompressed) - - // Set to uncompressed. - chunkType := uint8(chunkTypeUncompressedData) - chunkLen := 4 + len(uncompressed) - - // Attempt compressing. - n := binary.PutUvarint(obuf[obufHeaderLen:], uint64(len(uncompressed))) - n2 := w.encodeBlock(obuf[obufHeaderLen+n:], uncompressed) - - if n2 > 0 { - chunkType = uint8(chunkTypeCompressedData) - chunkLen = 4 + n + n2 - obuf = obuf[:obufHeaderLen+n+n2] - } else { - obuf = obuf[:8] - } - - // Fill in the per-chunk header that comes before the body. - obuf[0] = chunkType - obuf[1] = uint8(chunkLen >> 0) - obuf[2] = uint8(chunkLen >> 8) - obuf[3] = uint8(chunkLen >> 16) - obuf[4] = uint8(checksum >> 0) - obuf[5] = uint8(checksum >> 8) - obuf[6] = uint8(checksum >> 16) - obuf[7] = uint8(checksum >> 24) - - n, err := w.writer.Write(obuf) - if err != nil { - return 0, w.err(err) - } - if n != len(obuf) { - return 0, w.err(io.ErrShortWrite) - } - w.err(w.index.add(w.written, w.uncompWritten)) - w.written += int64(n) - w.uncompWritten += int64(len(uncompressed)) - - if chunkType == chunkTypeUncompressedData { - // Write uncompressed data. - n, err := w.writer.Write(uncompressed) - if err != nil { - return 0, w.err(err) - } - if n != len(uncompressed) { - return 0, w.err(io.ErrShortWrite) - } - w.written += int64(n) - } - w.buffers.Put(obuf) - // Queue final output. - nRet += len(uncompressed) - } - return nRet, nil -} - -// AsyncFlush writes any buffered bytes to a block and starts compressing it. -// It does not wait for the output has been written as Flush() does. -func (w *Writer) AsyncFlush() error { - if err := w.err(nil); err != nil { - return err - } - - // Queue any data still in input buffer. - if len(w.ibuf) != 0 { - if !w.wroteStreamHeader { - _, err := w.writeSync(w.ibuf) - w.ibuf = w.ibuf[:0] - return w.err(err) - } else { - _, err := w.write(w.ibuf) - w.ibuf = w.ibuf[:0] - err = w.err(err) - if err != nil { - return err - } - } - } - return w.err(nil) -} - -// Flush flushes the Writer to its underlying io.Writer. -// This does not apply padding. -func (w *Writer) Flush() error { - if err := w.AsyncFlush(); err != nil { - return err - } - if w.output == nil { - return w.err(nil) - } - - // Send empty buffer - res := make(chan result) - w.output <- res - // Block until this has been picked up. - res <- result{b: nil, startOffset: w.uncompWritten} - // When it is closed, we have flushed. - <-res - return w.err(nil) -} - -// Close calls Flush and then closes the Writer. -// Calling Close multiple times is ok, -// but calling CloseIndex after this will make it not return the index. -func (w *Writer) Close() error { - _, err := w.closeIndex(w.appendIndex) - return err -} - -// CloseIndex calls Close and returns an index on first call. -// This is not required if you are only adding index to a stream. -func (w *Writer) CloseIndex() ([]byte, error) { - return w.closeIndex(true) -} - -func (w *Writer) closeIndex(idx bool) ([]byte, error) { - err := w.Flush() - if w.output != nil { - close(w.output) - w.writerWg.Wait() - w.output = nil - } - - var index []byte - if w.err(err) == nil && w.writer != nil { - // Create index. - if idx { - compSize := int64(-1) - if w.pad <= 1 { - compSize = w.written - } - index = w.index.appendTo(w.ibuf[:0], w.uncompWritten, compSize) - // Count as written for padding. - if w.appendIndex { - w.written += int64(len(index)) - } - } - - if w.pad > 1 { - tmp := w.ibuf[:0] - if len(index) > 0 { - // Allocate another buffer. - tmp = w.buffers.Get().([]byte)[:0] - defer w.buffers.Put(tmp) - } - add := calcSkippableFrame(w.written, int64(w.pad)) - frame, err := skippableFrame(tmp, add, w.randSrc) - if err = w.err(err); err != nil { - return nil, err - } - n, err2 := w.writer.Write(frame) - if err2 == nil && n != len(frame) { - err2 = io.ErrShortWrite - } - _ = w.err(err2) - } - if len(index) > 0 && w.appendIndex { - n, err2 := w.writer.Write(index) - if err2 == nil && n != len(index) { - err2 = io.ErrShortWrite - } - _ = w.err(err2) - } - } - err = w.err(errClosed) - if err == errClosed { - return index, nil - } - return nil, err -} - -// calcSkippableFrame will return a total size to be added for written -// to be divisible by multiple. -// The value will always be > skippableFrameHeader. -// The function will panic if written < 0 or wantMultiple <= 0. -func calcSkippableFrame(written, wantMultiple int64) int { - if wantMultiple <= 0 { - panic("wantMultiple <= 0") - } - if written < 0 { - panic("written < 0") - } - leftOver := written % wantMultiple - if leftOver == 0 { - return 0 - } - toAdd := wantMultiple - leftOver - for toAdd < skippableFrameHeader { - toAdd += wantMultiple - } - return int(toAdd) -} - -// skippableFrame will add a skippable frame with a total size of bytes. -// total should be >= skippableFrameHeader and < maxBlockSize + skippableFrameHeader -func skippableFrame(dst []byte, total int, r io.Reader) ([]byte, error) { - if total == 0 { - return dst, nil - } - if total < skippableFrameHeader { - return dst, fmt.Errorf("s2: requested skippable frame (%d) < 4", total) - } - if int64(total) >= maxBlockSize+skippableFrameHeader { - return dst, fmt.Errorf("s2: requested skippable frame (%d) >= max 1<<24", total) - } - // Chunk type 0xfe "Section 4.4 Padding (chunk type 0xfe)" - dst = append(dst, chunkTypePadding) - f := uint32(total - skippableFrameHeader) - // Add chunk length. - dst = append(dst, uint8(f), uint8(f>>8), uint8(f>>16)) - // Add data - start := len(dst) - dst = append(dst, make([]byte, f)...) - _, err := io.ReadFull(r, dst[start:]) - return dst, err -} - -var errClosed = errors.New("s2: Writer is closed") - -// WriterOption is an option for creating a encoder. -type WriterOption func(*Writer) error - -// WriterConcurrency will set the concurrency, -// meaning the maximum number of decoders to run concurrently. -// The value supplied must be at least 1. -// By default this will be set to GOMAXPROCS. -func WriterConcurrency(n int) WriterOption { - return func(w *Writer) error { - if n <= 0 { - return errors.New("concurrency must be at least 1") - } - w.concurrency = n - return nil - } -} - -// WriterAddIndex will append an index to the end of a stream -// when it is closed. -func WriterAddIndex() WriterOption { - return func(w *Writer) error { - w.appendIndex = true - return nil - } -} - -// WriterBetterCompression will enable better compression. -// EncodeBetter compresses better than Encode but typically with a -// 10-40% speed decrease on both compression and decompression. -func WriterBetterCompression() WriterOption { - return func(w *Writer) error { - w.level = levelBetter - return nil - } -} - -// WriterBestCompression will enable better compression. -// EncodeBest compresses better than Encode but typically with a -// big speed decrease on compression. -func WriterBestCompression() WriterOption { - return func(w *Writer) error { - w.level = levelBest - return nil - } -} - -// WriterUncompressed will bypass compression. -// The stream will be written as uncompressed blocks only. -// If concurrency is > 1 CRC and output will still be done async. -func WriterUncompressed() WriterOption { - return func(w *Writer) error { - w.level = levelUncompressed - return nil - } -} - -// WriterBufferDone will perform a callback when EncodeBuffer has finished -// writing a buffer to the output and the buffer can safely be reused. -// If the buffer was split into several blocks, it will be sent after the last block. -// Callbacks will not be done concurrently. -func WriterBufferDone(fn func(b []byte)) WriterOption { - return func(w *Writer) error { - w.bufferCB = fn - return nil - } -} - -// WriterBlockSize allows to override the default block size. -// Blocks will be this size or smaller. -// Minimum size is 4KB and maximum size is 4MB. -// -// Bigger blocks may give bigger throughput on systems with many cores, -// and will increase compression slightly, but it will limit the possible -// concurrency for smaller payloads for both encoding and decoding. -// Default block size is 1MB. -// -// When writing Snappy compatible output using WriterSnappyCompat, -// the maximum block size is 64KB. -func WriterBlockSize(n int) WriterOption { - return func(w *Writer) error { - if w.snappy && n > maxSnappyBlockSize || n < minBlockSize { - return errors.New("s2: block size too large. Must be <= 64K and >=4KB on for snappy compatible output") - } - if n > maxBlockSize || n < minBlockSize { - return errors.New("s2: block size too large. Must be <= 4MB and >=4KB") - } - w.blockSize = n - return nil - } -} - -// WriterPadding will add padding to all output so the size will be a multiple of n. -// This can be used to obfuscate the exact output size or make blocks of a certain size. -// The contents will be a skippable frame, so it will be invisible by the decoder. -// n must be > 0 and <= 4MB. -// The padded area will be filled with data from crypto/rand.Reader. -// The padding will be applied whenever Close is called on the writer. -func WriterPadding(n int) WriterOption { - return func(w *Writer) error { - if n <= 0 { - return fmt.Errorf("s2: padding must be at least 1") - } - // No need to waste our time. - if n == 1 { - w.pad = 0 - } - if n > maxBlockSize { - return fmt.Errorf("s2: padding must less than 4MB") - } - w.pad = n - return nil - } -} - -// WriterPaddingSrc will get random data for padding from the supplied source. -// By default crypto/rand is used. -func WriterPaddingSrc(reader io.Reader) WriterOption { - return func(w *Writer) error { - w.randSrc = reader - return nil - } -} - -// WriterSnappyCompat will write snappy compatible output. -// The output can be decompressed using either snappy or s2. -// If block size is more than 64KB it is set to that. -func WriterSnappyCompat() WriterOption { - return func(w *Writer) error { - w.snappy = true - if w.blockSize > 64<<10 { - // We choose 8 bytes less than 64K, since that will make literal emits slightly more effective. - // And allows us to skip some size checks. - w.blockSize = (64 << 10) - 8 - } - return nil - } -} - -// WriterFlushOnWrite will compress blocks on each call to the Write function. -// -// This is quite inefficient as blocks size will depend on the write size. -// -// Use WriterConcurrency(1) to also make sure that output is flushed. -// When Write calls return, otherwise they will be written when compression is done. -func WriterFlushOnWrite() WriterOption { - return func(w *Writer) error { - w.flushOnWrite = true - return nil - } -} - -// WriterCustomEncoder allows to override the encoder for blocks on the stream. -// The function must compress 'src' into 'dst' and return the bytes used in dst as an integer. -// Block size (initial varint) should not be added by the encoder. -// Returning value 0 indicates the block could not be compressed. -// Returning a negative value indicates that compression should be attempted. -// The function should expect to be called concurrently. -func WriterCustomEncoder(fn func(dst, src []byte) int) WriterOption { - return func(w *Writer) error { - w.customEnc = fn - return nil - } -} diff --git a/vendor/github.com/klauspost/compress/s2sx.mod b/vendor/github.com/klauspost/compress/s2sx.mod deleted file mode 100644 index 5a4412f90..000000000 --- a/vendor/github.com/klauspost/compress/s2sx.mod +++ /dev/null @@ -1,4 +0,0 @@ -module github.com/klauspost/compress - -go 1.19 - diff --git a/vendor/github.com/klauspost/compress/s2sx.sum b/vendor/github.com/klauspost/compress/s2sx.sum deleted file mode 100644 index e69de29bb..000000000 --- a/vendor/github.com/klauspost/compress/s2sx.sum +++ /dev/null diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md deleted file mode 100644 index 92e2347bb..000000000 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ /dev/null @@ -1,441 +0,0 @@ -# zstd - -[Zstandard](https://facebook.github.io/zstd/) is a real-time compression algorithm, providing high compression ratios. -It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. -A high performance compression algorithm is implemented. For now focused on speed. - -This package provides [compression](#Compressor) to and [decompression](#Decompressor) of Zstandard content. - -This package is pure Go and without use of "unsafe". - -The `zstd` package is provided as open source software using a Go standard license. - -Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors. - -For seekable zstd streams, see [this excellent package](https://github.com/SaveTheRbtz/zstd-seekable-format-go). - -## Installation - -Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`. - -[](https://pkg.go.dev/github.com/klauspost/compress/zstd) - -## Compressor - -### Status: - -STABLE - there may always be subtle bugs, a wide variety of content has been tested and the library is actively -used by several projects. This library is being [fuzz-tested](https://github.com/klauspost/compress-fuzz) for all updates. - -There may still be specific combinations of data types/size/settings that could lead to edge cases, -so as always, testing is recommended. - -For now, a high speed (fastest) and medium-fast (default) compressor has been implemented. - -* The "Fastest" compression ratio is roughly equivalent to zstd level 1. -* The "Default" compression ratio is roughly equivalent to zstd level 3 (default). -* The "Better" compression ratio is roughly equivalent to zstd level 7. -* The "Best" compression ratio is roughly equivalent to zstd level 11. - -In terms of speed, it is typically 2x as fast as the stdlib deflate/gzip in its fastest mode. -The compression ratio compared to stdlib is around level 3, but usually 3x as fast. - - -### Usage - -An Encoder can be used for either compressing a stream via the -`io.WriteCloser` interface supported by the Encoder or as multiple independent -tasks via the `EncodeAll` function. -Smaller encodes are encouraged to use the EncodeAll function. -Use `NewWriter` to create a new instance that can be used for both. - -To create a writer with default options, do like this: - -```Go -// Compress input to output. -func Compress(in io.Reader, out io.Writer) error { - enc, err := zstd.NewWriter(out) - if err != nil { - return err - } - _, err = io.Copy(enc, in) - if err != nil { - enc.Close() - return err - } - return enc.Close() -} -``` - -Now you can encode by writing data to `enc`. The output will be finished writing when `Close()` is called. -Even if your encode fails, you should still call `Close()` to release any resources that may be held up. - -The above is fine for big encodes. However, whenever possible try to *reuse* the writer. - -To reuse the encoder, you can use the `Reset(io.Writer)` function to change to another output. -This will allow the encoder to reuse all resources and avoid wasteful allocations. - -Currently stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part -of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change -in the future. So if you want to limit concurrency for future updates, specify the concurrency -you would like. - -If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)` -which will compress input as each block is completed, blocking on writes until each has completed. - -You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined -compression settings can be specified. - -#### Future Compatibility Guarantees - -This will be an evolving project. When using this package it is important to note that both the compression efficiency and speed may change. - -The goal will be to keep the default efficiency at the default zstd (level 3). -However the encoding should never be assumed to remain the same, -and you should not use hashes of compressed output for similarity checks. - -The Encoder can be assumed to produce the same output from the exact same code version. -However, the may be modes in the future that break this, -although they will not be enabled without an explicit option. - -This encoder is not designed to (and will probably never) output the exact same bitstream as the reference encoder. - -Also note, that the cgo decompressor currently does not [report all errors on invalid input](https://github.com/DataDog/zstd/issues/59), -[omits error checks](https://github.com/DataDog/zstd/issues/61), [ignores checksums](https://github.com/DataDog/zstd/issues/43) -and seems to ignore concatenated streams, even though [it is part of the spec](https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frames). - -#### Blocks - -For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`. - -`EncodeAll` will encode all input in src and append it to dst. -This function can be called concurrently. -Each call will only run on a same goroutine as the caller. - -Encoded blocks can be concatenated and the result will be the combined input stream. -Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`. - -Especially when encoding blocks you should take special care to reuse the encoder. -This will effectively make it run without allocations after a warmup period. -To make it run completely without allocations, supply a destination buffer with space for all content. - -```Go -import "github.com/klauspost/compress/zstd" - -// Create a writer that caches compressors. -// For this operation type we supply a nil Reader. -var encoder, _ = zstd.NewWriter(nil) - -// Compress a buffer. -// If you have a destination buffer, the allocation in the call can also be eliminated. -func Compress(src []byte) []byte { - return encoder.EncodeAll(src, make([]byte, 0, len(src))) -} -``` - -You can control the maximum number of concurrent encodes using the `WithEncoderConcurrency(n)` -option when creating the writer. - -Using the Encoder for both a stream and individual blocks concurrently is safe. - -### Performance - -I have collected some speed examples to compare speed and compression against other compressors. - -* `file` is the input file. -* `out` is the compressor used. `zskp` is this package. `zstd` is the Datadog cgo library. `gzstd/gzkp` is gzip standard and this library. -* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default"; 3 is "better", 4 is "best". -* `insize`/`outsize` is the input/output size. -* `millis` is the number of milliseconds used for compression. -* `mb/s` is megabytes (2^20 bytes) per second. - -``` -Silesia Corpus: -http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip - -This package: -file out level insize outsize millis mb/s -silesia.tar zskp 1 211947520 73821326 634 318.47 -silesia.tar zskp 2 211947520 67655404 1508 133.96 -silesia.tar zskp 3 211947520 64746933 3000 67.37 -silesia.tar zskp 4 211947520 60073508 16926 11.94 - -cgo zstd: -silesia.tar zstd 1 211947520 73605392 543 371.56 -silesia.tar zstd 3 211947520 66793289 864 233.68 -silesia.tar zstd 6 211947520 62916450 1913 105.66 -silesia.tar zstd 9 211947520 60212393 5063 39.92 - -gzip, stdlib/this package: -silesia.tar gzstd 1 211947520 80007735 1498 134.87 -silesia.tar gzkp 1 211947520 80088272 1009 200.31 - -GOB stream of binary data. Highly compressible. -https://files.klauspost.com/compress/gob-stream.7z - -file out level insize outsize millis mb/s -gob-stream zskp 1 1911399616 233948096 3230 564.34 -gob-stream zskp 2 1911399616 203997694 4997 364.73 -gob-stream zskp 3 1911399616 173526523 13435 135.68 -gob-stream zskp 4 1911399616 162195235 47559 38.33 - -gob-stream zstd 1 1911399616 249810424 2637 691.26 -gob-stream zstd 3 1911399616 208192146 3490 522.31 -gob-stream zstd 6 1911399616 193632038 6687 272.56 -gob-stream zstd 9 1911399616 177620386 16175 112.70 - -gob-stream gzstd 1 1911399616 357382013 9046 201.49 -gob-stream gzkp 1 1911399616 359136669 4885 373.08 - -The test data for the Large Text Compression Benchmark is the first -10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. -http://mattmahoney.net/dc/textdata.html - -file out level insize outsize millis mb/s -enwik9 zskp 1 1000000000 343833605 3687 258.64 -enwik9 zskp 2 1000000000 317001237 7672 124.29 -enwik9 zskp 3 1000000000 291915823 15923 59.89 -enwik9 zskp 4 1000000000 261710291 77697 12.27 - -enwik9 zstd 1 1000000000 358072021 3110 306.65 -enwik9 zstd 3 1000000000 313734672 4784 199.35 -enwik9 zstd 6 1000000000 295138875 10290 92.68 -enwik9 zstd 9 1000000000 278348700 28549 33.40 - -enwik9 gzstd 1 1000000000 382578136 8608 110.78 -enwik9 gzkp 1 1000000000 382781160 5628 169.45 - -Highly compressible JSON file. -https://files.klauspost.com/compress/github-june-2days-2019.json.zst - -file out level insize outsize millis mb/s -github-june-2days-2019.json zskp 1 6273951764 697439532 9789 611.17 -github-june-2days-2019.json zskp 2 6273951764 610876538 18553 322.49 -github-june-2days-2019.json zskp 3 6273951764 517662858 44186 135.41 -github-june-2days-2019.json zskp 4 6273951764 464617114 165373 36.18 - -github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 -github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 -github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 -github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 - -github-june-2days-2019.json gzstd 1 6273951764 1164397768 26793 223.32 -github-june-2days-2019.json gzkp 1 6273951764 1120631856 17693 338.16 - -VM Image, Linux mint with a few installed applications: -https://files.klauspost.com/compress/rawstudio-mint14.7z - -file out level insize outsize millis mb/s -rawstudio-mint14.tar zskp 1 8558382592 3718400221 18206 448.29 -rawstudio-mint14.tar zskp 2 8558382592 3326118337 37074 220.15 -rawstudio-mint14.tar zskp 3 8558382592 3163842361 87306 93.49 -rawstudio-mint14.tar zskp 4 8558382592 2970480650 783862 10.41 - -rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 -rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 -rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 -rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 - -rawstudio-mint14.tar gzstd 1 8558382592 3926234992 51345 158.96 -rawstudio-mint14.tar gzkp 1 8558382592 3960117298 36722 222.26 - -CSV data: -https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst - -file out level insize outsize millis mb/s -nyc-taxi-data-10M.csv zskp 1 3325605752 641319332 9462 335.17 -nyc-taxi-data-10M.csv zskp 2 3325605752 588976126 17570 180.50 -nyc-taxi-data-10M.csv zskp 3 3325605752 529329260 32432 97.79 -nyc-taxi-data-10M.csv zskp 4 3325605752 474949772 138025 22.98 - -nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 -nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 -nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 -nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 - -nyc-taxi-data-10M.csv gzstd 1 3325605752 928654908 21270 149.11 -nyc-taxi-data-10M.csv gzkp 1 3325605752 922273214 13929 227.68 -``` - -## Decompressor - -Status: STABLE - there may still be subtle bugs, but a wide variety of content has been tested. - -This library is being continuously [fuzz-tested](https://github.com/klauspost/compress-fuzz), -kindly supplied by [fuzzit.dev](https://fuzzit.dev/). -The main purpose of the fuzz testing is to ensure that it is not possible to crash the decoder, -or run it past its limits with ANY input provided. - -### Usage - -The package has been designed for two main usages, big streams of data and smaller in-memory buffers. -There are two main usages of the package for these. Both of them are accessed by creating a `Decoder`. - -For streaming use a simple setup could look like this: - -```Go -import "github.com/klauspost/compress/zstd" - -func Decompress(in io.Reader, out io.Writer) error { - d, err := zstd.NewReader(in) - if err != nil { - return err - } - defer d.Close() - - // Copy content... - _, err = io.Copy(out, d) - return err -} -``` - -It is important to use the "Close" function when you no longer need the Reader to stop running goroutines, -when running with default settings. -Goroutines will exit once an error has been returned, including `io.EOF` at the end of a stream. - -Streams are decoded concurrently in 4 asynchronous stages to give the best possible throughput. -However, if you prefer synchronous decompression, use `WithDecoderConcurrency(1)` which will decompress data -as it is being requested only. - -For decoding buffers, it could look something like this: - -```Go -import "github.com/klauspost/compress/zstd" - -// Create a reader that caches decompressors. -// For this operation type we supply a nil Reader. -var decoder, _ = zstd.NewReader(nil, zstd.WithDecoderConcurrency(0)) - -// Decompress a buffer. We don't supply a destination buffer, -// so it will be allocated by the decoder. -func Decompress(src []byte) ([]byte, error) { - return decoder.DecodeAll(src, nil) -} -``` - -Both of these cases should provide the functionality needed. -The decoder can be used for *concurrent* decompression of multiple buffers. -By default 4 decompressors will be created. - -It will only allow a certain number of concurrent operations to run. -To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. -It is possible to use `WithDecoderConcurrency(0)` to create GOMAXPROCS decoders. - -### Dictionaries - -Data compressed with [dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression) can be decompressed. - -Dictionaries are added individually to Decoders. -Dictionaries are generated by the `zstd --train` command and contains an initial state for the decoder. -To add a dictionary use the `WithDecoderDicts(dicts ...[]byte)` option with the dictionary data. -Several dictionaries can be added at once. - -The dictionary will be used automatically for the data that specifies them. -A re-used Decoder will still contain the dictionaries registered. - -When registering multiple dictionaries with the same ID, the last one will be used. - -It is possible to use dictionaries when compressing data. - -To enable a dictionary use `WithEncoderDict(dict []byte)`. Here only one dictionary will be used -and it will likely be used even if it doesn't improve compression. - -The used dictionary must be used to decompress the content. - -For any real gains, the dictionary should be built with similar data. -If an unsuitable dictionary is used the output may be slightly larger than using no dictionary. -Use the [zstd commandline tool](https://github.com/facebook/zstd/releases) to build a dictionary from sample data. -For information see [zstd dictionary information](https://github.com/facebook/zstd#the-case-for-small-data-compression). - -For now there is a fixed startup performance penalty for compressing content with dictionaries. -This will likely be improved over time. Just be aware to test performance when implementing. - -### Allocation-less operation - -The decoder has been designed to operate without allocations after a warmup. - -This means that you should *store* the decoder for best performance. -To re-use a stream decoder, use the `Reset(r io.Reader) error` to switch to another stream. -A decoder can safely be re-used even if the previous stream failed. - -To release the resources, you must call the `Close()` function on a decoder. -After this it can *no longer be reused*, but all running goroutines will be stopped. -So you *must* use this if you will no longer need the Reader. - -For decompressing smaller buffers a single decoder can be used. -When decoding buffers, you can supply a destination slice with length 0 and your expected capacity. -In this case no unneeded allocations should be made. - -### Concurrency - -The buffer decoder does everything on the same goroutine and does nothing concurrently. -It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that. - -The stream decoder will create goroutines that: - -1) Reads input and splits the input into blocks. -2) Decompression of literals. -3) Decompression of sequences. -4) Reconstruction of output stream. - -So effectively this also means the decoder will "read ahead" and prepare data to always be available for output. - -The concurrency level will, for streams, determine how many blocks ahead the compression will start. - -Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency. - -In practice this means that concurrency is often limited to utilizing about 3 cores effectively. - -### Benchmarks - -The first two are streaming decodes and the last are smaller inputs. - -Running on AMD Ryzen 9 3950X 16-Core Processor. AMD64 assembly used. - -``` -BenchmarkDecoderSilesia-32 5 206878840 ns/op 1024.50 MB/s 49808 B/op 43 allocs/op -BenchmarkDecoderEnwik9-32 1 1271809000 ns/op 786.28 MB/s 72048 B/op 52 allocs/op - -Concurrent blocks, performance: - -BenchmarkDecoder_DecodeAllParallel/kppkn.gtb.zst-32 67356 17857 ns/op 10321.96 MB/s 22.48 pct 102 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/geo.protodata.zst-32 266656 4421 ns/op 26823.21 MB/s 11.89 pct 19 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/plrabn12.txt.zst-32 20992 56842 ns/op 8477.17 MB/s 39.90 pct 754 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/lcet10.txt.zst-32 27456 43932 ns/op 9714.01 MB/s 33.27 pct 524 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/asyoulik.txt.zst-32 78432 15047 ns/op 8319.15 MB/s 40.34 pct 66 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/alice29.txt.zst-32 65800 18436 ns/op 8249.63 MB/s 37.75 pct 88 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/html_x_4.zst-32 102993 11523 ns/op 35546.09 MB/s 3.637 pct 143 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/paper-100k.pdf.zst-32 1000000 1070 ns/op 95720.98 MB/s 80.53 pct 3 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/fireworks.jpeg.zst-32 749802 1752 ns/op 70272.35 MB/s 100.0 pct 5 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/urls.10K.zst-32 22640 52934 ns/op 13263.37 MB/s 26.25 pct 1014 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/html.zst-32 226412 5232 ns/op 19572.27 MB/s 14.49 pct 20 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/comp-data.bin.zst-32 923041 1276 ns/op 3194.71 MB/s 31.26 pct 0 B/op 0 allocs/op -``` - -This reflects the performance around May 2022, but this may be out of date. - -## Zstd inside ZIP files - -It is possible to use zstandard to compress individual files inside zip archives. -While this isn't widely supported it can be useful for internal files. - -To support the compression and decompression of these files you must register a compressor and decompressor. - -It is highly recommended registering the (de)compressors on individual zip Reader/Writer and NOT -use the global registration functions. The main reason for this is that 2 registrations from -different packages will result in a panic. - -It is a good idea to only have a single compressor and decompressor, since they can be used for multiple zip -files concurrently, and using a single instance will allow reusing some resources. - -See [this example](https://pkg.go.dev/github.com/klauspost/compress/zstd#example-ZipCompressor) for -how to compress and decompress files inside zip archives. - -# Contributions - -Contributions are always welcome. -For new features/fixes, remember to add tests and for performance enhancements include benchmarks. - -For general feedback and experience reports, feel free to open an issue or write me on [Twitter](https://twitter.com/sh0dan). - -This package includes the excellent [`github.com/cespare/xxhash`](https://github.com/cespare/xxhash) package Copyright (c) 2016 Caleb Spare. diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go deleted file mode 100644 index 25ca98394..000000000 --- a/vendor/github.com/klauspost/compress/zstd/bitreader.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "encoding/binary" - "errors" - "fmt" - "io" - "math/bits" -) - -// bitReader reads a bitstream in reverse. -// The last set bit indicates the start of the stream and is used -// for aligning the input. -type bitReader struct { - in []byte - value uint64 // Maybe use [16]byte, but shifting is awkward. - bitsRead uint8 -} - -// init initializes and resets the bit reader. -func (b *bitReader) init(in []byte) error { - if len(in) < 1 { - return errors.New("corrupt stream: too short") - } - b.in = in - // The highest bit of the last byte indicates where to start - v := in[len(in)-1] - if v == 0 { - return errors.New("corrupt stream, did not find end of stream") - } - b.bitsRead = 64 - b.value = 0 - if len(in) >= 8 { - b.fillFastStart() - } else { - b.fill() - b.fill() - } - b.bitsRead += 8 - uint8(highBits(uint32(v))) - return nil -} - -// getBits will return n bits. n can be 0. -func (b *bitReader) getBits(n uint8) int { - if n == 0 /*|| b.bitsRead >= 64 */ { - return 0 - } - return int(b.get32BitsFast(n)) -} - -// get32BitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReader) get32BitsFast(n uint8) uint32 { - const regMask = 64 - 1 - v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) - b.bitsRead += n - return v -} - -// fillFast() will make sure at least 32 bits are available. -// There must be at least 4 bytes available. -func (b *bitReader) fillFast() { - if b.bitsRead < 32 { - return - } - v := b.in[len(b.in)-4:] - b.in = b.in[:len(b.in)-4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 -} - -// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. -func (b *bitReader) fillFastStart() { - v := b.in[len(b.in)-8:] - b.in = b.in[:len(b.in)-8] - b.value = binary.LittleEndian.Uint64(v) - b.bitsRead = 0 -} - -// fill() will make sure at least 32 bits are available. -func (b *bitReader) fill() { - if b.bitsRead < 32 { - return - } - if len(b.in) >= 4 { - v := b.in[len(b.in)-4:] - b.in = b.in[:len(b.in)-4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 - return - } - - b.bitsRead -= uint8(8 * len(b.in)) - for len(b.in) > 0 { - b.value = (b.value << 8) | uint64(b.in[len(b.in)-1]) - b.in = b.in[:len(b.in)-1] - } -} - -// finished returns true if all bits have been read from the bit stream. -func (b *bitReader) finished() bool { - return len(b.in) == 0 && b.bitsRead >= 64 -} - -// overread returns true if more bits have been requested than is on the stream. -func (b *bitReader) overread() bool { - return b.bitsRead > 64 -} - -// remain returns the number of bits remaining. -func (b *bitReader) remain() uint { - return 8*uint(len(b.in)) + 64 - uint(b.bitsRead) -} - -// close the bitstream and returns an error if out-of-buffer reads occurred. -func (b *bitReader) close() error { - // Release reference. - b.in = nil - if !b.finished() { - return fmt.Errorf("%d extra bits on block, should be 0", b.remain()) - } - if b.bitsRead > 64 { - return io.ErrUnexpectedEOF - } - return nil -} - -func highBits(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go deleted file mode 100644 index 1952f175b..000000000 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2018 Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. - -package zstd - -// bitWriter will write bits. -// First bit will be LSB of the first byte of output. -type bitWriter struct { - bitContainer uint64 - nBits uint8 - out []byte -} - -// bitMask16 is bitmasks. Has extra to avoid bounds check. -var bitMask16 = [32]uint16{ - 0, 1, 3, 7, 0xF, 0x1F, - 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, - 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF, - 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, - 0xFFFF, 0xFFFF} /* up to 16 bits */ - -var bitMask32 = [32]uint32{ - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, - 0x1ffff, 0x3ffff, 0x7FFFF, 0xfFFFF, 0x1fFFFF, 0x3fFFFF, 0x7fFFFF, 0xffFFFF, - 0x1ffFFFF, 0x3ffFFFF, 0x7ffFFFF, 0xfffFFFF, 0x1fffFFFF, 0x3fffFFFF, 0x7fffFFFF, -} // up to 32 bits - -// addBits16NC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16NC(value uint16, bits uint8) { - b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) - b.nBits += bits -} - -// addBits32NC will add up to 31 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits32NC(value uint32, bits uint8) { - b.bitContainer |= uint64(value&bitMask32[bits&31]) << (b.nBits & 63) - b.nBits += bits -} - -// addBits64NC will add up to 64 bits. -// There must be space for 32 bits. -func (b *bitWriter) addBits64NC(value uint64, bits uint8) { - if bits <= 31 { - b.addBits32Clean(uint32(value), bits) - return - } - b.addBits32Clean(uint32(value), 32) - b.flush32() - b.addBits32Clean(uint32(value>>32), bits-32) -} - -// addBits32Clean will add up to 32 bits. -// It will not check if there is space for them. -// The input must not contain more bits than specified. -func (b *bitWriter) addBits32Clean(value uint32, bits uint8) { - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// flush32 will flush out, so there are at least 32 bits available for writing. -func (b *bitWriter) flush32() { - if b.nBits < 32 { - return - } - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24)) - b.nBits -= 32 - b.bitContainer >>= 32 -} - -// flushAlign will flush remaining full bytes and align to next byte boundary. -func (b *bitWriter) flushAlign() { - nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { - b.out = append(b.out, byte(b.bitContainer>>(i*8))) - } - b.nBits = 0 - b.bitContainer = 0 -} - -// close will write the alignment bit and write the final byte(s) -// to the output. -func (b *bitWriter) close() { - // End mark - b.addBits16Clean(1, 1) - // flush until next byte. - b.flushAlign() -} - -// reset and continue writing by appending to out. -func (b *bitWriter) reset(out []byte) { - b.bitContainer = 0 - b.nBits = 0 - b.out = out -} diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go deleted file mode 100644 index 9c28840c3..000000000 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ /dev/null @@ -1,731 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "hash/crc32" - "io" - "os" - "path/filepath" - "sync" - - "github.com/klauspost/compress/huff0" - "github.com/klauspost/compress/zstd/internal/xxhash" -) - -type blockType uint8 - -//go:generate stringer -type=blockType,literalsBlockType,seqCompMode,tableIndex - -const ( - blockTypeRaw blockType = iota - blockTypeRLE - blockTypeCompressed - blockTypeReserved -) - -type literalsBlockType uint8 - -const ( - literalsBlockRaw literalsBlockType = iota - literalsBlockRLE - literalsBlockCompressed - literalsBlockTreeless -) - -const ( - // maxCompressedBlockSize is the biggest allowed compressed block size (128KB) - maxCompressedBlockSize = 128 << 10 - - compressedBlockOverAlloc = 16 - maxCompressedBlockSizeAlloc = 128<<10 + compressedBlockOverAlloc - - // Maximum possible block size (all Raw+Uncompressed). - maxBlockSize = (1 << 21) - 1 - - maxMatchLen = 131074 - maxSequences = 0x7f00 + 0xffff - - // We support slightly less than the reference decoder to be able to - // use ints on 32 bit archs. - maxOffsetBits = 30 -) - -var ( - huffDecoderPool = sync.Pool{New: func() interface{} { - return &huff0.Scratch{} - }} - - fseDecoderPool = sync.Pool{New: func() interface{} { - return &fseDecoder{} - }} -) - -type blockDec struct { - // Raw source data of the block. - data []byte - dataStorage []byte - - // Destination of the decoded data. - dst []byte - - // Buffer for literals data. - literalBuf []byte - - // Window size of the block. - WindowSize uint64 - - err error - - // Check against this crc, if hasCRC is true. - checkCRC uint32 - hasCRC bool - - // Frame to use for singlethreaded decoding. - // Should not be used by the decoder itself since parent may be another frame. - localFrame *frameDec - - sequence []seqVals - - async struct { - newHist *history - literals []byte - seqData []byte - seqSize int // Size of uncompressed sequences - fcs uint64 - } - - // Block is RLE, this is the size. - RLESize uint32 - - Type blockType - - // Is this the last block of a frame? - Last bool - - // Use less memory - lowMem bool -} - -func (b *blockDec) String() string { - if b == nil { - return "<nil>" - } - return fmt.Sprintf("Steam Size: %d, Type: %v, Last: %t, Window: %d", len(b.data), b.Type, b.Last, b.WindowSize) -} - -func newBlockDec(lowMem bool) *blockDec { - b := blockDec{ - lowMem: lowMem, - } - return &b -} - -// reset will reset the block. -// Input must be a start of a block and will be at the end of the block when returned. -func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { - b.WindowSize = windowSize - tmp, err := br.readSmall(3) - if err != nil { - println("Reading block header:", err) - return err - } - bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) - b.Last = bh&1 != 0 - b.Type = blockType((bh >> 1) & 3) - // find size. - cSize := int(bh >> 3) - maxSize := maxCompressedBlockSizeAlloc - switch b.Type { - case blockTypeReserved: - return ErrReservedBlockType - case blockTypeRLE: - if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) { - if debugDecoder { - printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b) - } - return ErrWindowSizeExceeded - } - b.RLESize = uint32(cSize) - if b.lowMem { - maxSize = cSize - } - cSize = 1 - case blockTypeCompressed: - if debugDecoder { - println("Data size on stream:", cSize) - } - b.RLESize = 0 - maxSize = maxCompressedBlockSizeAlloc - if windowSize < maxCompressedBlockSize && b.lowMem { - maxSize = int(windowSize) + compressedBlockOverAlloc - } - if cSize > maxCompressedBlockSize || uint64(cSize) > b.WindowSize { - if debugDecoder { - printf("compressed block too big: csize:%d block: %+v\n", uint64(cSize), b) - } - return ErrCompressedSizeTooBig - } - // Empty compressed blocks must at least be 2 bytes - // for Literals_Block_Type and one for Sequences_Section_Header. - if cSize < 2 { - return ErrBlockTooSmall - } - case blockTypeRaw: - if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) { - if debugDecoder { - printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b) - } - return ErrWindowSizeExceeded - } - - b.RLESize = 0 - // We do not need a destination for raw blocks. - maxSize = -1 - default: - panic("Invalid block type") - } - - // Read block data. - if _, ok := br.(*byteBuf); !ok && cap(b.dataStorage) < cSize { - // byteBuf doesn't need a destination buffer. - if b.lowMem || cSize > maxCompressedBlockSize { - b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc) - } else { - b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc) - } - } - b.data, err = br.readBig(cSize, b.dataStorage) - if err != nil { - if debugDecoder { - println("Reading block:", err, "(", cSize, ")", len(b.data)) - printf("%T", br) - } - return err - } - if cap(b.dst) <= maxSize { - b.dst = make([]byte, 0, maxSize+1) - } - return nil -} - -// sendEOF will make the decoder send EOF on this frame. -func (b *blockDec) sendErr(err error) { - b.Last = true - b.Type = blockTypeReserved - b.err = err -} - -// Close will release resources. -// Closed blockDec cannot be reset. -func (b *blockDec) Close() { -} - -// decodeBuf -func (b *blockDec) decodeBuf(hist *history) error { - switch b.Type { - case blockTypeRLE: - if cap(b.dst) < int(b.RLESize) { - if b.lowMem { - b.dst = make([]byte, b.RLESize) - } else { - b.dst = make([]byte, maxCompressedBlockSize) - } - } - b.dst = b.dst[:b.RLESize] - v := b.data[0] - for i := range b.dst { - b.dst[i] = v - } - hist.appendKeep(b.dst) - return nil - case blockTypeRaw: - hist.appendKeep(b.data) - return nil - case blockTypeCompressed: - saved := b.dst - // Append directly to history - if hist.ignoreBuffer == 0 { - b.dst = hist.b - hist.b = nil - } else { - b.dst = b.dst[:0] - } - err := b.decodeCompressed(hist) - if debugDecoder { - println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err) - } - if hist.ignoreBuffer == 0 { - hist.b = b.dst - b.dst = saved - } else { - hist.appendKeep(b.dst) - } - return err - case blockTypeReserved: - // Used for returning errors. - return b.err - default: - panic("Invalid block type") - } -} - -func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err error) { - // There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header - if len(in) < 2 { - return in, ErrBlockTooSmall - } - - litType := literalsBlockType(in[0] & 3) - var litRegenSize int - var litCompSize int - sizeFormat := (in[0] >> 2) & 3 - var fourStreams bool - var literals []byte - switch litType { - case literalsBlockRaw, literalsBlockRLE: - switch sizeFormat { - case 0, 2: - // Regenerated_Size uses 5 bits (0-31). Literals_Section_Header uses 1 byte. - litRegenSize = int(in[0] >> 3) - in = in[1:] - case 1: - // Regenerated_Size uses 12 bits (0-4095). Literals_Section_Header uses 2 bytes. - litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) - in = in[2:] - case 3: - // Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes. - if len(in) < 3 { - println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return in, ErrBlockTooSmall - } - litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12) - in = in[3:] - } - case literalsBlockCompressed, literalsBlockTreeless: - switch sizeFormat { - case 0, 1: - // Both Regenerated_Size and Compressed_Size use 10 bits (0-1023). - if len(in) < 3 { - println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return in, ErrBlockTooSmall - } - n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) - litRegenSize = int(n & 1023) - litCompSize = int(n >> 10) - fourStreams = sizeFormat == 1 - in = in[3:] - case 2: - fourStreams = true - if len(in) < 4 { - println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return in, ErrBlockTooSmall - } - n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) - litRegenSize = int(n & 16383) - litCompSize = int(n >> 14) - in = in[4:] - case 3: - fourStreams = true - if len(in) < 5 { - println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return in, ErrBlockTooSmall - } - n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28) - litRegenSize = int(n & 262143) - litCompSize = int(n >> 18) - in = in[5:] - } - } - if debugDecoder { - println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams) - } - if litRegenSize > int(b.WindowSize) || litRegenSize > maxCompressedBlockSize { - return in, ErrWindowSizeExceeded - } - - switch litType { - case literalsBlockRaw: - if len(in) < litRegenSize { - println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize) - return in, ErrBlockTooSmall - } - literals = in[:litRegenSize] - in = in[litRegenSize:] - //printf("Found %d uncompressed literals\n", litRegenSize) - case literalsBlockRLE: - if len(in) < 1 { - println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1) - return in, ErrBlockTooSmall - } - if cap(b.literalBuf) < litRegenSize { - if b.lowMem { - b.literalBuf = make([]byte, litRegenSize, litRegenSize+compressedBlockOverAlloc) - } else { - b.literalBuf = make([]byte, litRegenSize, maxCompressedBlockSize+compressedBlockOverAlloc) - } - } - literals = b.literalBuf[:litRegenSize] - v := in[0] - for i := range literals { - literals[i] = v - } - in = in[1:] - if debugDecoder { - printf("Found %d RLE compressed literals\n", litRegenSize) - } - case literalsBlockTreeless: - if len(in) < litCompSize { - println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) - return in, ErrBlockTooSmall - } - // Store compressed literals, so we defer decoding until we get history. - literals = in[:litCompSize] - in = in[litCompSize:] - if debugDecoder { - printf("Found %d compressed literals\n", litCompSize) - } - huff := hist.huffTree - if huff == nil { - return in, errors.New("literal block was treeless, but no history was defined") - } - // Ensure we have space to store it. - if cap(b.literalBuf) < litRegenSize { - if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) - } else { - b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) - } - } - var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize - if fourStreams { - literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) - } else { - literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) - } - // Make sure we don't leak our literals buffer - if err != nil { - println("decompressing literals:", err) - return in, err - } - if len(literals) != litRegenSize { - return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) - } - - case literalsBlockCompressed: - if len(in) < litCompSize { - println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) - return in, ErrBlockTooSmall - } - literals = in[:litCompSize] - in = in[litCompSize:] - // Ensure we have space to store it. - if cap(b.literalBuf) < litRegenSize { - if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) - } else { - b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) - } - } - huff := hist.huffTree - if huff == nil || (hist.dict != nil && huff == hist.dict.litEnc) { - huff = huffDecoderPool.Get().(*huff0.Scratch) - if huff == nil { - huff = &huff0.Scratch{} - } - } - var err error - if debugDecoder { - println("huff table input:", len(literals), "CRC:", crc32.ChecksumIEEE(literals)) - } - huff, literals, err = huff0.ReadTable(literals, huff) - if err != nil { - println("reading huffman table:", err) - return in, err - } - hist.huffTree = huff - huff.MaxDecodedSize = litRegenSize - // Use our out buffer. - if fourStreams { - literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) - } else { - literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) - } - if err != nil { - println("decoding compressed literals:", err) - return in, err - } - // Make sure we don't leak our literals buffer - if len(literals) != litRegenSize { - return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) - } - // Re-cap to get extra size. - literals = b.literalBuf[:len(literals)] - if debugDecoder { - printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) - } - } - hist.decoders.literals = literals - return in, nil -} - -// decodeCompressed will start decompressing a block. -func (b *blockDec) decodeCompressed(hist *history) error { - in := b.data - in, err := b.decodeLiterals(in, hist) - if err != nil { - return err - } - err = b.prepareSequences(in, hist) - if err != nil { - return err - } - if hist.decoders.nSeqs == 0 { - b.dst = append(b.dst, hist.decoders.literals...) - return nil - } - before := len(hist.decoders.out) - err = hist.decoders.decodeSync(hist.b[hist.ignoreBuffer:]) - if err != nil { - return err - } - if hist.decoders.maxSyncLen > 0 { - hist.decoders.maxSyncLen += uint64(before) - hist.decoders.maxSyncLen -= uint64(len(hist.decoders.out)) - } - b.dst = hist.decoders.out - hist.recentOffsets = hist.decoders.prevOffset - return nil -} - -func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { - if debugDecoder { - printf("prepareSequences: %d byte(s) input\n", len(in)) - } - // Decode Sequences - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section - if len(in) < 1 { - return ErrBlockTooSmall - } - var nSeqs int - seqHeader := in[0] - switch { - case seqHeader < 128: - nSeqs = int(seqHeader) - in = in[1:] - case seqHeader < 255: - if len(in) < 2 { - return ErrBlockTooSmall - } - nSeqs = int(seqHeader-128)<<8 | int(in[1]) - in = in[2:] - case seqHeader == 255: - if len(in) < 3 { - return ErrBlockTooSmall - } - nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8) - in = in[3:] - } - if nSeqs == 0 && len(in) != 0 { - // When no sequences, there should not be any more data... - if debugDecoder { - printf("prepareSequences: 0 sequences, but %d byte(s) left on stream\n", len(in)) - } - return ErrUnexpectedBlockSize - } - - var seqs = &hist.decoders - seqs.nSeqs = nSeqs - if nSeqs > 0 { - if len(in) < 1 { - return ErrBlockTooSmall - } - br := byteReader{b: in, off: 0} - compMode := br.Uint8() - br.advance(1) - if debugDecoder { - printf("Compression modes: 0b%b", compMode) - } - if compMode&3 != 0 { - return errors.New("corrupt block: reserved bits not zero") - } - for i := uint(0); i < 3; i++ { - mode := seqCompMode((compMode >> (6 - i*2)) & 3) - if debugDecoder { - println("Table", tableIndex(i), "is", mode) - } - var seq *sequenceDec - switch tableIndex(i) { - case tableLiteralLengths: - seq = &seqs.litLengths - case tableOffsets: - seq = &seqs.offsets - case tableMatchLengths: - seq = &seqs.matchLengths - default: - panic("unknown table") - } - switch mode { - case compModePredefined: - if seq.fse != nil && !seq.fse.preDefined { - fseDecoderPool.Put(seq.fse) - } - seq.fse = &fsePredef[i] - case compModeRLE: - if br.remain() < 1 { - return ErrBlockTooSmall - } - v := br.Uint8() - br.advance(1) - if seq.fse == nil || seq.fse.preDefined { - seq.fse = fseDecoderPool.Get().(*fseDecoder) - } - symb, err := decSymbolValue(v, symbolTableX[i]) - if err != nil { - printf("RLE Transform table (%v) error: %v", tableIndex(i), err) - return err - } - seq.fse.setRLE(symb) - if debugDecoder { - printf("RLE set to 0x%x, code: %v", symb, v) - } - case compModeFSE: - if debugDecoder { - println("Reading table for", tableIndex(i)) - } - if seq.fse == nil || seq.fse.preDefined { - seq.fse = fseDecoderPool.Get().(*fseDecoder) - } - err := seq.fse.readNCount(&br, uint16(maxTableSymbol[i])) - if err != nil { - println("Read table error:", err) - return err - } - err = seq.fse.transform(symbolTableX[i]) - if err != nil { - println("Transform table error:", err) - return err - } - if debugDecoder { - println("Read table ok", "symbolLen:", seq.fse.symbolLen) - } - case compModeRepeat: - seq.repeat = true - } - if br.overread() { - return io.ErrUnexpectedEOF - } - } - in = br.unread() - } - if debugDecoder { - println("Literals:", len(seqs.literals), "hash:", xxhash.Sum64(seqs.literals), "and", seqs.nSeqs, "sequences.") - } - - if nSeqs == 0 { - if len(b.sequence) > 0 { - b.sequence = b.sequence[:0] - } - return nil - } - br := seqs.br - if br == nil { - br = &bitReader{} - } - if err := br.init(in); err != nil { - return err - } - - if err := seqs.initialize(br, hist, b.dst); err != nil { - println("initializing sequences:", err) - return err - } - // Extract blocks... - if false && hist.dict == nil { - fatalErr := func(err error) { - if err != nil { - panic(err) - } - } - fn := fmt.Sprintf("n-%d-lits-%d-prev-%d-%d-%d-win-%d.blk", hist.decoders.nSeqs, len(hist.decoders.literals), hist.recentOffsets[0], hist.recentOffsets[1], hist.recentOffsets[2], hist.windowSize) - var buf bytes.Buffer - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.litLengths.fse)) - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse)) - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse)) - buf.Write(in) - os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) - } - - return nil -} - -func (b *blockDec) decodeSequences(hist *history) error { - if cap(b.sequence) < hist.decoders.nSeqs { - if b.lowMem { - b.sequence = make([]seqVals, 0, hist.decoders.nSeqs) - } else { - b.sequence = make([]seqVals, 0, 0x7F00+0xffff) - } - } - b.sequence = b.sequence[:hist.decoders.nSeqs] - if hist.decoders.nSeqs == 0 { - hist.decoders.seqSize = len(hist.decoders.literals) - return nil - } - hist.decoders.windowSize = hist.windowSize - hist.decoders.prevOffset = hist.recentOffsets - - err := hist.decoders.decode(b.sequence) - hist.recentOffsets = hist.decoders.prevOffset - return err -} - -func (b *blockDec) executeSequences(hist *history) error { - hbytes := hist.b - if len(hbytes) > hist.windowSize { - hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } - } - hist.decoders.windowSize = hist.windowSize - hist.decoders.out = b.dst[:0] - err := hist.decoders.execute(b.sequence, hbytes) - if err != nil { - return err - } - return b.updateHistory(hist) -} - -func (b *blockDec) updateHistory(hist *history) error { - if len(b.data) > maxCompressedBlockSize { - return fmt.Errorf("compressed block size too large (%d)", len(b.data)) - } - // Set output and release references. - b.dst = hist.decoders.out - hist.recentOffsets = hist.decoders.prevOffset - - if b.Last { - // if last block we don't care about history. - println("Last block, no history returned") - hist.b = hist.b[:0] - return nil - } else { - hist.append(b.dst) - if debugDecoder { - println("Finished block with ", len(b.sequence), "sequences. Added", len(b.dst), "to history, now length", len(hist.b)) - } - } - hist.decoders.out, hist.decoders.literals = nil, nil - - return nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go deleted file mode 100644 index 32a7f401d..000000000 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ /dev/null @@ -1,909 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "errors" - "fmt" - "math" - "math/bits" - - "github.com/klauspost/compress/huff0" -) - -type blockEnc struct { - size int - literals []byte - sequences []seq - coders seqCoders - litEnc *huff0.Scratch - dictLitEnc *huff0.Scratch - wr bitWriter - - extraLits int - output []byte - recentOffsets [3]uint32 - prevRecentOffsets [3]uint32 - - last bool - lowMem bool -} - -// init should be used once the block has been created. -// If called more than once, the effect is the same as calling reset. -func (b *blockEnc) init() { - if b.lowMem { - // 1K literals - if cap(b.literals) < 1<<10 { - b.literals = make([]byte, 0, 1<<10) - } - const defSeqs = 20 - if cap(b.sequences) < defSeqs { - b.sequences = make([]seq, 0, defSeqs) - } - // 1K - if cap(b.output) < 1<<10 { - b.output = make([]byte, 0, 1<<10) - } - } else { - if cap(b.literals) < maxCompressedBlockSize { - b.literals = make([]byte, 0, maxCompressedBlockSize) - } - const defSeqs = 2000 - if cap(b.sequences) < defSeqs { - b.sequences = make([]seq, 0, defSeqs) - } - if cap(b.output) < maxCompressedBlockSize { - b.output = make([]byte, 0, maxCompressedBlockSize) - } - } - - if b.coders.mlEnc == nil { - b.coders.mlEnc = &fseEncoder{} - b.coders.mlPrev = &fseEncoder{} - b.coders.ofEnc = &fseEncoder{} - b.coders.ofPrev = &fseEncoder{} - b.coders.llEnc = &fseEncoder{} - b.coders.llPrev = &fseEncoder{} - } - b.litEnc = &huff0.Scratch{WantLogLess: 4} - b.reset(nil) -} - -// initNewEncode can be used to reset offsets and encoders to the initial state. -func (b *blockEnc) initNewEncode() { - b.recentOffsets = [3]uint32{1, 4, 8} - b.litEnc.Reuse = huff0.ReusePolicyNone - b.coders.setPrev(nil, nil, nil) -} - -// reset will reset the block for a new encode, but in the same stream, -// meaning that state will be carried over, but the block content is reset. -// If a previous block is provided, the recent offsets are carried over. -func (b *blockEnc) reset(prev *blockEnc) { - b.extraLits = 0 - b.literals = b.literals[:0] - b.size = 0 - b.sequences = b.sequences[:0] - b.output = b.output[:0] - b.last = false - if prev != nil { - b.recentOffsets = prev.prevRecentOffsets - } - b.dictLitEnc = nil -} - -// reset will reset the block for a new encode, but in the same stream, -// meaning that state will be carried over, but the block content is reset. -// If a previous block is provided, the recent offsets are carried over. -func (b *blockEnc) swapEncoders(prev *blockEnc) { - b.coders.swap(&prev.coders) - b.litEnc, prev.litEnc = prev.litEnc, b.litEnc -} - -// blockHeader contains the information for a block header. -type blockHeader uint32 - -// setLast sets the 'last' indicator on a block. -func (h *blockHeader) setLast(b bool) { - if b { - *h = *h | 1 - } else { - const mask = (1 << 24) - 2 - *h = *h & mask - } -} - -// setSize will store the compressed size of a block. -func (h *blockHeader) setSize(v uint32) { - const mask = 7 - *h = (*h)&mask | blockHeader(v<<3) -} - -// setType sets the block type. -func (h *blockHeader) setType(t blockType) { - const mask = 1 | (((1 << 24) - 1) ^ 7) - *h = (*h & mask) | blockHeader(t<<1) -} - -// appendTo will append the block header to a slice. -func (h blockHeader) appendTo(b []byte) []byte { - return append(b, uint8(h), uint8(h>>8), uint8(h>>16)) -} - -// String returns a string representation of the block. -func (h blockHeader) String() string { - return fmt.Sprintf("Type: %d, Size: %d, Last:%t", (h>>1)&3, h>>3, h&1 == 1) -} - -// literalsHeader contains literals header information. -type literalsHeader uint64 - -// setType can be used to set the type of literal block. -func (h *literalsHeader) setType(t literalsBlockType) { - const mask = math.MaxUint64 - 3 - *h = (*h & mask) | literalsHeader(t) -} - -// setSize can be used to set a single size, for uncompressed and RLE content. -func (h *literalsHeader) setSize(regenLen int) { - inBits := bits.Len32(uint32(regenLen)) - // Only retain 2 bits - const mask = 3 - lh := uint64(*h & mask) - switch { - case inBits < 5: - lh |= (uint64(regenLen) << 3) | (1 << 60) - if debugEncoder { - got := int(lh>>3) & 0xff - if got != regenLen { - panic(fmt.Sprint("litRegenSize = ", regenLen, "(want) != ", got, "(got)")) - } - } - case inBits < 12: - lh |= (1 << 2) | (uint64(regenLen) << 4) | (2 << 60) - case inBits < 20: - lh |= (3 << 2) | (uint64(regenLen) << 4) | (3 << 60) - default: - panic(fmt.Errorf("internal error: block too big (%d)", regenLen)) - } - *h = literalsHeader(lh) -} - -// setSizes will set the size of a compressed literals section and the input length. -func (h *literalsHeader) setSizes(compLen, inLen int, single bool) { - compBits, inBits := bits.Len32(uint32(compLen)), bits.Len32(uint32(inLen)) - // Only retain 2 bits - const mask = 3 - lh := uint64(*h & mask) - switch { - case compBits <= 10 && inBits <= 10: - if !single { - lh |= 1 << 2 - } - lh |= (uint64(inLen) << 4) | (uint64(compLen) << (10 + 4)) | (3 << 60) - if debugEncoder { - const mmask = (1 << 24) - 1 - n := (lh >> 4) & mmask - if int(n&1023) != inLen { - panic(fmt.Sprint("regensize:", int(n&1023), "!=", inLen, inBits)) - } - if int(n>>10) != compLen { - panic(fmt.Sprint("compsize:", int(n>>10), "!=", compLen, compBits)) - } - } - case compBits <= 14 && inBits <= 14: - lh |= (2 << 2) | (uint64(inLen) << 4) | (uint64(compLen) << (14 + 4)) | (4 << 60) - if single { - panic("single stream used with more than 10 bits length.") - } - case compBits <= 18 && inBits <= 18: - lh |= (3 << 2) | (uint64(inLen) << 4) | (uint64(compLen) << (18 + 4)) | (5 << 60) - if single { - panic("single stream used with more than 10 bits length.") - } - default: - panic("internal error: block too big") - } - *h = literalsHeader(lh) -} - -// appendTo will append the literals header to a byte slice. -func (h literalsHeader) appendTo(b []byte) []byte { - size := uint8(h >> 60) - switch size { - case 1: - b = append(b, uint8(h)) - case 2: - b = append(b, uint8(h), uint8(h>>8)) - case 3: - b = append(b, uint8(h), uint8(h>>8), uint8(h>>16)) - case 4: - b = append(b, uint8(h), uint8(h>>8), uint8(h>>16), uint8(h>>24)) - case 5: - b = append(b, uint8(h), uint8(h>>8), uint8(h>>16), uint8(h>>24), uint8(h>>32)) - default: - panic(fmt.Errorf("internal error: literalsHeader has invalid size (%d)", size)) - } - return b -} - -// size returns the output size with currently set values. -func (h literalsHeader) size() int { - return int(h >> 60) -} - -func (h literalsHeader) String() string { - return fmt.Sprintf("Type: %d, SizeFormat: %d, Size: 0x%d, Bytes:%d", literalsBlockType(h&3), (h>>2)&3, h&((1<<60)-1)>>4, h>>60) -} - -// pushOffsets will push the recent offsets to the backup store. -func (b *blockEnc) pushOffsets() { - b.prevRecentOffsets = b.recentOffsets -} - -// pushOffsets will push the recent offsets to the backup store. -func (b *blockEnc) popOffsets() { - b.recentOffsets = b.prevRecentOffsets -} - -// matchOffset will adjust recent offsets and return the adjusted one, -// if it matches a previous offset. -func (b *blockEnc) matchOffset(offset, lits uint32) uint32 { - // Check if offset is one of the recent offsets. - // Adjusts the output offset accordingly. - // Gives a tiny bit of compression, typically around 1%. - if true { - if lits > 0 { - switch offset { - case b.recentOffsets[0]: - offset = 1 - case b.recentOffsets[1]: - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset = 2 - case b.recentOffsets[2]: - b.recentOffsets[2] = b.recentOffsets[1] - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset = 3 - default: - b.recentOffsets[2] = b.recentOffsets[1] - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset += 3 - } - } else { - switch offset { - case b.recentOffsets[1]: - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset = 1 - case b.recentOffsets[2]: - b.recentOffsets[2] = b.recentOffsets[1] - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset = 2 - case b.recentOffsets[0] - 1: - b.recentOffsets[2] = b.recentOffsets[1] - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset = 3 - default: - b.recentOffsets[2] = b.recentOffsets[1] - b.recentOffsets[1] = b.recentOffsets[0] - b.recentOffsets[0] = offset - offset += 3 - } - } - } else { - offset += 3 - } - return offset -} - -// encodeRaw can be used to set the output to a raw representation of supplied bytes. -func (b *blockEnc) encodeRaw(a []byte) { - var bh blockHeader - bh.setLast(b.last) - bh.setSize(uint32(len(a))) - bh.setType(blockTypeRaw) - b.output = bh.appendTo(b.output[:0]) - b.output = append(b.output, a...) - if debugEncoder { - println("Adding RAW block, length", len(a), "last:", b.last) - } -} - -// encodeRaw can be used to set the output to a raw representation of supplied bytes. -func (b *blockEnc) encodeRawTo(dst, src []byte) []byte { - var bh blockHeader - bh.setLast(b.last) - bh.setSize(uint32(len(src))) - bh.setType(blockTypeRaw) - dst = bh.appendTo(dst) - dst = append(dst, src...) - if debugEncoder { - println("Adding RAW block, length", len(src), "last:", b.last) - } - return dst -} - -// encodeLits can be used if the block is only litLen. -func (b *blockEnc) encodeLits(lits []byte, raw bool) error { - var bh blockHeader - bh.setLast(b.last) - bh.setSize(uint32(len(lits))) - - // Don't compress extremely small blocks - if len(lits) < 8 || (len(lits) < 32 && b.dictLitEnc == nil) || raw { - if debugEncoder { - println("Adding RAW block, length", len(lits), "last:", b.last) - } - bh.setType(blockTypeRaw) - b.output = bh.appendTo(b.output) - b.output = append(b.output, lits...) - return nil - } - - var ( - out []byte - reUsed, single bool - err error - ) - if b.dictLitEnc != nil { - b.litEnc.TransferCTable(b.dictLitEnc) - b.litEnc.Reuse = huff0.ReusePolicyAllow - b.dictLitEnc = nil - } - if len(lits) >= 1024 { - // Use 4 Streams. - out, reUsed, err = huff0.Compress4X(lits, b.litEnc) - } else if len(lits) > 16 { - // Use 1 stream - single = true - out, reUsed, err = huff0.Compress1X(lits, b.litEnc) - } else { - err = huff0.ErrIncompressible - } - if err == nil && len(out)+5 > len(lits) { - // If we are close, we may still be worse or equal to raw. - var lh literalsHeader - lh.setSizes(len(out), len(lits), single) - if len(out)+lh.size() >= len(lits) { - err = huff0.ErrIncompressible - } - } - switch err { - case huff0.ErrIncompressible: - if debugEncoder { - println("Adding RAW block, length", len(lits), "last:", b.last) - } - bh.setType(blockTypeRaw) - b.output = bh.appendTo(b.output) - b.output = append(b.output, lits...) - return nil - case huff0.ErrUseRLE: - if debugEncoder { - println("Adding RLE block, length", len(lits)) - } - bh.setType(blockTypeRLE) - b.output = bh.appendTo(b.output) - b.output = append(b.output, lits[0]) - return nil - case nil: - default: - return err - } - // Compressed... - // Now, allow reuse - b.litEnc.Reuse = huff0.ReusePolicyAllow - bh.setType(blockTypeCompressed) - var lh literalsHeader - if reUsed { - if debugEncoder { - println("Reused tree, compressed to", len(out)) - } - lh.setType(literalsBlockTreeless) - } else { - if debugEncoder { - println("New tree, compressed to", len(out), "tree size:", len(b.litEnc.OutTable)) - } - lh.setType(literalsBlockCompressed) - } - // Set sizes - lh.setSizes(len(out), len(lits), single) - bh.setSize(uint32(len(out) + lh.size() + 1)) - - // Write block headers. - b.output = bh.appendTo(b.output) - b.output = lh.appendTo(b.output) - // Add compressed data. - b.output = append(b.output, out...) - // No sequences. - b.output = append(b.output, 0) - return nil -} - -// encodeRLE will encode an RLE block. -func (b *blockEnc) encodeRLE(val byte, length uint32) { - var bh blockHeader - bh.setLast(b.last) - bh.setSize(length) - bh.setType(blockTypeRLE) - b.output = bh.appendTo(b.output) - b.output = append(b.output, val) -} - -// fuzzFseEncoder can be used to fuzz the FSE encoder. -func fuzzFseEncoder(data []byte) int { - if len(data) > maxSequences || len(data) < 2 { - return 0 - } - enc := fseEncoder{} - hist := enc.Histogram() - maxSym := uint8(0) - for i, v := range data { - v = v & 63 - data[i] = v - hist[v]++ - if v > maxSym { - maxSym = v - } - } - if maxSym == 0 { - // All 0 - return 0 - } - maxCount := func(a []uint32) int { - var max uint32 - for _, v := range a { - if v > max { - max = v - } - } - return int(max) - } - cnt := maxCount(hist[:maxSym]) - if cnt == len(data) { - // RLE - return 0 - } - enc.HistogramFinished(maxSym, cnt) - err := enc.normalizeCount(len(data)) - if err != nil { - return 0 - } - _, err = enc.writeCount(nil) - if err != nil { - panic(err) - } - return 1 -} - -// encode will encode the block and append the output in b.output. -// Previous offset codes must be pushed if more blocks are expected. -func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { - if len(b.sequences) == 0 { - return b.encodeLits(b.literals, rawAllLits) - } - if len(b.sequences) == 1 && len(org) > 0 && len(b.literals) <= 1 { - // Check common RLE cases. - seq := b.sequences[0] - if seq.litLen == uint32(len(b.literals)) && seq.offset-3 == 1 { - // Offset == 1 and 0 or 1 literals. - b.encodeRLE(org[0], b.sequences[0].matchLen+zstdMinMatch+seq.litLen) - return nil - } - } - - // We want some difference to at least account for the headers. - saved := b.size - len(b.literals) - (b.size >> 6) - if saved < 16 { - if org == nil { - return errIncompressible - } - b.popOffsets() - return b.encodeLits(org, rawAllLits) - } - - var bh blockHeader - var lh literalsHeader - bh.setLast(b.last) - bh.setType(blockTypeCompressed) - // Store offset of the block header. Needed when we know the size. - bhOffset := len(b.output) - b.output = bh.appendTo(b.output) - - var ( - out []byte - reUsed, single bool - err error - ) - if b.dictLitEnc != nil { - b.litEnc.TransferCTable(b.dictLitEnc) - b.litEnc.Reuse = huff0.ReusePolicyAllow - b.dictLitEnc = nil - } - if len(b.literals) >= 1024 && !raw { - // Use 4 Streams. - out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) - } else if len(b.literals) > 16 && !raw { - // Use 1 stream - single = true - out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc) - } else { - err = huff0.ErrIncompressible - } - - if err == nil && len(out)+5 > len(b.literals) { - // If we are close, we may still be worse or equal to raw. - var lh literalsHeader - lh.setSize(len(b.literals)) - szRaw := lh.size() - lh.setSizes(len(out), len(b.literals), single) - szComp := lh.size() - if len(out)+szComp >= len(b.literals)+szRaw { - err = huff0.ErrIncompressible - } - } - switch err { - case huff0.ErrIncompressible: - lh.setType(literalsBlockRaw) - lh.setSize(len(b.literals)) - b.output = lh.appendTo(b.output) - b.output = append(b.output, b.literals...) - if debugEncoder { - println("Adding literals RAW, length", len(b.literals)) - } - case huff0.ErrUseRLE: - lh.setType(literalsBlockRLE) - lh.setSize(len(b.literals)) - b.output = lh.appendTo(b.output) - b.output = append(b.output, b.literals[0]) - if debugEncoder { - println("Adding literals RLE") - } - case nil: - // Compressed litLen... - if reUsed { - if debugEncoder { - println("reused tree") - } - lh.setType(literalsBlockTreeless) - } else { - if debugEncoder { - println("new tree, size:", len(b.litEnc.OutTable)) - } - lh.setType(literalsBlockCompressed) - if debugEncoder { - _, _, err := huff0.ReadTable(out, nil) - if err != nil { - panic(err) - } - } - } - lh.setSizes(len(out), len(b.literals), single) - if debugEncoder { - printf("Compressed %d literals to %d bytes", len(b.literals), len(out)) - println("Adding literal header:", lh) - } - b.output = lh.appendTo(b.output) - b.output = append(b.output, out...) - b.litEnc.Reuse = huff0.ReusePolicyAllow - if debugEncoder { - println("Adding literals compressed") - } - default: - if debugEncoder { - println("Adding literals ERROR:", err) - } - return err - } - // Sequence compression - - // Write the number of sequences - switch { - case len(b.sequences) < 128: - b.output = append(b.output, uint8(len(b.sequences))) - case len(b.sequences) < 0x7f00: // TODO: this could be wrong - n := len(b.sequences) - b.output = append(b.output, 128+uint8(n>>8), uint8(n)) - default: - n := len(b.sequences) - 0x7f00 - b.output = append(b.output, 255, uint8(n), uint8(n>>8)) - } - if debugEncoder { - println("Encoding", len(b.sequences), "sequences") - } - b.genCodes() - llEnc := b.coders.llEnc - ofEnc := b.coders.ofEnc - mlEnc := b.coders.mlEnc - err = llEnc.normalizeCount(len(b.sequences)) - if err != nil { - return err - } - err = ofEnc.normalizeCount(len(b.sequences)) - if err != nil { - return err - } - err = mlEnc.normalizeCount(len(b.sequences)) - if err != nil { - return err - } - - // Choose the best compression mode for each type. - // Will evaluate the new vs predefined and previous. - chooseComp := func(cur, prev, preDef *fseEncoder) (*fseEncoder, seqCompMode) { - // See if predefined/previous is better - hist := cur.count[:cur.symbolLen] - nSize := cur.approxSize(hist) + cur.maxHeaderSize() - predefSize := preDef.approxSize(hist) - prevSize := prev.approxSize(hist) - - // Add a small penalty for new encoders. - // Don't bother with extremely small (<2 byte gains). - nSize = nSize + (nSize+2*8*16)>>4 - switch { - case predefSize <= prevSize && predefSize <= nSize || forcePreDef: - if debugEncoder { - println("Using predefined", predefSize>>3, "<=", nSize>>3) - } - return preDef, compModePredefined - case prevSize <= nSize: - if debugEncoder { - println("Using previous", prevSize>>3, "<=", nSize>>3) - } - return prev, compModeRepeat - default: - if debugEncoder { - println("Using new, predef", predefSize>>3, ". previous:", prevSize>>3, ">", nSize>>3, "header max:", cur.maxHeaderSize()>>3, "bytes") - println("tl:", cur.actualTableLog, "symbolLen:", cur.symbolLen, "norm:", cur.norm[:cur.symbolLen], "hist", cur.count[:cur.symbolLen]) - } - return cur, compModeFSE - } - } - - // Write compression mode - var mode uint8 - if llEnc.useRLE { - mode |= uint8(compModeRLE) << 6 - llEnc.setRLE(b.sequences[0].llCode) - if debugEncoder { - println("llEnc.useRLE") - } - } else { - var m seqCompMode - llEnc, m = chooseComp(llEnc, b.coders.llPrev, &fsePredefEnc[tableLiteralLengths]) - mode |= uint8(m) << 6 - } - if ofEnc.useRLE { - mode |= uint8(compModeRLE) << 4 - ofEnc.setRLE(b.sequences[0].ofCode) - if debugEncoder { - println("ofEnc.useRLE") - } - } else { - var m seqCompMode - ofEnc, m = chooseComp(ofEnc, b.coders.ofPrev, &fsePredefEnc[tableOffsets]) - mode |= uint8(m) << 4 - } - - if mlEnc.useRLE { - mode |= uint8(compModeRLE) << 2 - mlEnc.setRLE(b.sequences[0].mlCode) - if debugEncoder { - println("mlEnc.useRLE, code: ", b.sequences[0].mlCode, "value", b.sequences[0].matchLen) - } - } else { - var m seqCompMode - mlEnc, m = chooseComp(mlEnc, b.coders.mlPrev, &fsePredefEnc[tableMatchLengths]) - mode |= uint8(m) << 2 - } - b.output = append(b.output, mode) - if debugEncoder { - printf("Compression modes: 0b%b", mode) - } - b.output, err = llEnc.writeCount(b.output) - if err != nil { - return err - } - start := len(b.output) - b.output, err = ofEnc.writeCount(b.output) - if err != nil { - return err - } - if false { - println("block:", b.output[start:], "tablelog", ofEnc.actualTableLog, "maxcount:", ofEnc.maxCount) - fmt.Printf("selected TableLog: %d, Symbol length: %d\n", ofEnc.actualTableLog, ofEnc.symbolLen) - for i, v := range ofEnc.norm[:ofEnc.symbolLen] { - fmt.Printf("%3d: %5d -> %4d \n", i, ofEnc.count[i], v) - } - } - b.output, err = mlEnc.writeCount(b.output) - if err != nil { - return err - } - - // Maybe in block? - wr := &b.wr - wr.reset(b.output) - - var ll, of, ml cState - - // Current sequence - seq := len(b.sequences) - 1 - s := b.sequences[seq] - llEnc.setBits(llBitsTable[:]) - mlEnc.setBits(mlBitsTable[:]) - ofEnc.setBits(nil) - - llTT, ofTT, mlTT := llEnc.ct.symbolTT[:256], ofEnc.ct.symbolTT[:256], mlEnc.ct.symbolTT[:256] - - // We have 3 bounds checks here (and in the loop). - // Since we are iterating backwards it is kinda hard to avoid. - llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] - ll.init(wr, &llEnc.ct, llB) - of.init(wr, &ofEnc.ct, ofB) - wr.flush32() - ml.init(wr, &mlEnc.ct, mlB) - - // Each of these lookups also generates a bounds check. - wr.addBits32NC(s.litLen, llB.outBits) - wr.addBits32NC(s.matchLen, mlB.outBits) - wr.flush32() - wr.addBits32NC(s.offset, ofB.outBits) - if debugSequences { - println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB) - } - seq-- - // Store sequences in reverse... - for seq >= 0 { - s = b.sequences[seq] - - ofB := ofTT[s.ofCode] - wr.flush32() // tablelog max is below 8 for each, so it will fill max 24 bits. - //of.encode(ofB) - nbBitsOut := (uint32(of.state) + ofB.deltaNbBits) >> 16 - dstState := int32(of.state>>(nbBitsOut&15)) + int32(ofB.deltaFindState) - wr.addBits16NC(of.state, uint8(nbBitsOut)) - of.state = of.stateTable[dstState] - - // Accumulate extra bits. - outBits := ofB.outBits & 31 - extraBits := uint64(s.offset & bitMask32[outBits]) - extraBitsN := outBits - - mlB := mlTT[s.mlCode] - //ml.encode(mlB) - nbBitsOut = (uint32(ml.state) + mlB.deltaNbBits) >> 16 - dstState = int32(ml.state>>(nbBitsOut&15)) + int32(mlB.deltaFindState) - wr.addBits16NC(ml.state, uint8(nbBitsOut)) - ml.state = ml.stateTable[dstState] - - outBits = mlB.outBits & 31 - extraBits = extraBits<<outBits | uint64(s.matchLen&bitMask32[outBits]) - extraBitsN += outBits - - llB := llTT[s.llCode] - //ll.encode(llB) - nbBitsOut = (uint32(ll.state) + llB.deltaNbBits) >> 16 - dstState = int32(ll.state>>(nbBitsOut&15)) + int32(llB.deltaFindState) - wr.addBits16NC(ll.state, uint8(nbBitsOut)) - ll.state = ll.stateTable[dstState] - - outBits = llB.outBits & 31 - extraBits = extraBits<<outBits | uint64(s.litLen&bitMask32[outBits]) - extraBitsN += outBits - - wr.flush32() - wr.addBits64NC(extraBits, extraBitsN) - - if debugSequences { - println("Encoded seq", seq, s) - } - - seq-- - } - ml.flush(mlEnc.actualTableLog) - of.flush(ofEnc.actualTableLog) - ll.flush(llEnc.actualTableLog) - wr.close() - b.output = wr.out - - // Maybe even add a bigger margin. - if len(b.output)-3-bhOffset >= b.size { - // Discard and encode as raw block. - b.output = b.encodeRawTo(b.output[:bhOffset], org) - b.popOffsets() - b.litEnc.Reuse = huff0.ReusePolicyNone - return nil - } - - // Size is output minus block header. - bh.setSize(uint32(len(b.output)-bhOffset) - 3) - if debugEncoder { - println("Rewriting block header", bh) - } - _ = bh.appendTo(b.output[bhOffset:bhOffset]) - b.coders.setPrev(llEnc, mlEnc, ofEnc) - return nil -} - -var errIncompressible = errors.New("incompressible") - -func (b *blockEnc) genCodes() { - if len(b.sequences) == 0 { - // nothing to do - return - } - if len(b.sequences) > math.MaxUint16 { - panic("can only encode up to 64K sequences") - } - // No bounds checks after here: - llH := b.coders.llEnc.Histogram() - ofH := b.coders.ofEnc.Histogram() - mlH := b.coders.mlEnc.Histogram() - for i := range llH { - llH[i] = 0 - } - for i := range ofH { - ofH[i] = 0 - } - for i := range mlH { - mlH[i] = 0 - } - - var llMax, ofMax, mlMax uint8 - for i := range b.sequences { - seq := &b.sequences[i] - v := llCode(seq.litLen) - seq.llCode = v - llH[v]++ - if v > llMax { - llMax = v - } - - v = ofCode(seq.offset) - seq.ofCode = v - ofH[v]++ - if v > ofMax { - ofMax = v - } - - v = mlCode(seq.matchLen) - seq.mlCode = v - mlH[v]++ - if v > mlMax { - mlMax = v - if debugAsserts && mlMax > maxMatchLengthSymbol { - panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen)) - } - } - } - maxCount := func(a []uint32) int { - var max uint32 - for _, v := range a { - if v > max { - max = v - } - } - return int(max) - } - if debugAsserts && mlMax > maxMatchLengthSymbol { - panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d)", mlMax)) - } - if debugAsserts && ofMax > maxOffsetBits { - panic(fmt.Errorf("ofMax > maxOffsetBits (%d)", ofMax)) - } - if debugAsserts && llMax > maxLiteralLengthSymbol { - panic(fmt.Errorf("llMax > maxLiteralLengthSymbol (%d)", llMax)) - } - - b.coders.mlEnc.HistogramFinished(mlMax, maxCount(mlH[:mlMax+1])) - b.coders.ofEnc.HistogramFinished(ofMax, maxCount(ofH[:ofMax+1])) - b.coders.llEnc.HistogramFinished(llMax, maxCount(llH[:llMax+1])) -} diff --git a/vendor/github.com/klauspost/compress/zstd/blocktype_string.go b/vendor/github.com/klauspost/compress/zstd/blocktype_string.go deleted file mode 100644 index 01a01e486..000000000 --- a/vendor/github.com/klauspost/compress/zstd/blocktype_string.go +++ /dev/null @@ -1,85 +0,0 @@ -// Code generated by "stringer -type=blockType,literalsBlockType,seqCompMode,tableIndex"; DO NOT EDIT. - -package zstd - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[blockTypeRaw-0] - _ = x[blockTypeRLE-1] - _ = x[blockTypeCompressed-2] - _ = x[blockTypeReserved-3] -} - -const _blockType_name = "blockTypeRawblockTypeRLEblockTypeCompressedblockTypeReserved" - -var _blockType_index = [...]uint8{0, 12, 24, 43, 60} - -func (i blockType) String() string { - if i >= blockType(len(_blockType_index)-1) { - return "blockType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _blockType_name[_blockType_index[i]:_blockType_index[i+1]] -} -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[literalsBlockRaw-0] - _ = x[literalsBlockRLE-1] - _ = x[literalsBlockCompressed-2] - _ = x[literalsBlockTreeless-3] -} - -const _literalsBlockType_name = "literalsBlockRawliteralsBlockRLEliteralsBlockCompressedliteralsBlockTreeless" - -var _literalsBlockType_index = [...]uint8{0, 16, 32, 55, 76} - -func (i literalsBlockType) String() string { - if i >= literalsBlockType(len(_literalsBlockType_index)-1) { - return "literalsBlockType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _literalsBlockType_name[_literalsBlockType_index[i]:_literalsBlockType_index[i+1]] -} -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[compModePredefined-0] - _ = x[compModeRLE-1] - _ = x[compModeFSE-2] - _ = x[compModeRepeat-3] -} - -const _seqCompMode_name = "compModePredefinedcompModeRLEcompModeFSEcompModeRepeat" - -var _seqCompMode_index = [...]uint8{0, 18, 29, 40, 54} - -func (i seqCompMode) String() string { - if i >= seqCompMode(len(_seqCompMode_index)-1) { - return "seqCompMode(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _seqCompMode_name[_seqCompMode_index[i]:_seqCompMode_index[i+1]] -} -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[tableLiteralLengths-0] - _ = x[tableOffsets-1] - _ = x[tableMatchLengths-2] -} - -const _tableIndex_name = "tableLiteralLengthstableOffsetstableMatchLengths" - -var _tableIndex_index = [...]uint8{0, 19, 31, 48} - -func (i tableIndex) String() string { - if i >= tableIndex(len(_tableIndex_index)-1) { - return "tableIndex(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _tableIndex_name[_tableIndex_index[i]:_tableIndex_index[i+1]] -} diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go deleted file mode 100644 index 55a388553..000000000 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "fmt" - "io" -) - -type byteBuffer interface { - // Read up to 8 bytes. - // Returns io.ErrUnexpectedEOF if this cannot be satisfied. - readSmall(n int) ([]byte, error) - - // Read >8 bytes. - // MAY use the destination slice. - readBig(n int, dst []byte) ([]byte, error) - - // Read a single byte. - readByte() (byte, error) - - // Skip n bytes. - skipN(n int64) error -} - -// in-memory buffer -type byteBuf []byte - -func (b *byteBuf) readSmall(n int) ([]byte, error) { - if debugAsserts && n > 8 { - panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) - } - bb := *b - if len(bb) < n { - return nil, io.ErrUnexpectedEOF - } - r := bb[:n] - *b = bb[n:] - return r, nil -} - -func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { - bb := *b - if len(bb) < n { - return nil, io.ErrUnexpectedEOF - } - r := bb[:n] - *b = bb[n:] - return r, nil -} - -func (b *byteBuf) readByte() (byte, error) { - bb := *b - if len(bb) < 1 { - return 0, io.ErrUnexpectedEOF - } - r := bb[0] - *b = bb[1:] - return r, nil -} - -func (b *byteBuf) skipN(n int64) error { - bb := *b - if n < 0 { - return fmt.Errorf("negative skip (%d) requested", n) - } - if int64(len(bb)) < n { - return io.ErrUnexpectedEOF - } - *b = bb[n:] - return nil -} - -// wrapper around a reader. -type readerWrapper struct { - r io.Reader - tmp [8]byte -} - -func (r *readerWrapper) readSmall(n int) ([]byte, error) { - if debugAsserts && n > 8 { - panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) - } - n2, err := io.ReadFull(r.r, r.tmp[:n]) - // We only really care about the actual bytes read. - if err != nil { - if err == io.EOF { - return nil, io.ErrUnexpectedEOF - } - if debugDecoder { - println("readSmall: got", n2, "want", n, "err", err) - } - return nil, err - } - return r.tmp[:n], nil -} - -func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) { - if cap(dst) < n { - dst = make([]byte, n) - } - n2, err := io.ReadFull(r.r, dst[:n]) - if err == io.EOF && n > 0 { - err = io.ErrUnexpectedEOF - } - return dst[:n2], err -} - -func (r *readerWrapper) readByte() (byte, error) { - n2, err := io.ReadFull(r.r, r.tmp[:1]) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return 0, err - } - if n2 != 1 { - return 0, io.ErrUnexpectedEOF - } - return r.tmp[0], nil -} - -func (r *readerWrapper) skipN(n int64) error { - n2, err := io.CopyN(io.Discard, r.r, n) - if n2 != n { - err = io.ErrUnexpectedEOF - } - return err -} diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go deleted file mode 100644 index 0e59a242d..000000000 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -// byteReader provides a byte reader that reads -// little endian values from a byte stream. -// The input stream is manually advanced. -// The reader performs no bounds checks. -type byteReader struct { - b []byte - off int -} - -// advance the stream b n bytes. -func (b *byteReader) advance(n uint) { - b.off += int(n) -} - -// overread returns whether we have advanced too far. -func (b *byteReader) overread() bool { - return b.off > len(b.b) -} - -// Int32 returns a little endian int32 starting at current offset. -func (b byteReader) Int32() int32 { - b2 := b.b[b.off:] - b2 = b2[:4] - v3 := int32(b2[3]) - v2 := int32(b2[2]) - v1 := int32(b2[1]) - v0 := int32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) -} - -// Uint8 returns the next byte -func (b *byteReader) Uint8() uint8 { - v := b.b[b.off] - return v -} - -// Uint32 returns a little endian uint32 starting at current offset. -func (b byteReader) Uint32() uint32 { - if r := b.remain(); r < 4 { - // Very rare - v := uint32(0) - for i := 1; i <= r; i++ { - v = (v << 8) | uint32(b.b[len(b.b)-i]) - } - return v - } - b2 := b.b[b.off:] - b2 = b2[:4] - v3 := uint32(b2[3]) - v2 := uint32(b2[2]) - v1 := uint32(b2[1]) - v0 := uint32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) -} - -// Uint32NC returns a little endian uint32 starting at current offset. -// The caller must be sure if there are at least 4 bytes left. -func (b byteReader) Uint32NC() uint32 { - b2 := b.b[b.off:] - b2 = b2[:4] - v3 := uint32(b2[3]) - v2 := uint32(b2[2]) - v1 := uint32(b2[1]) - v0 := uint32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) -} - -// unread returns the unread portion of the input. -func (b byteReader) unread() []byte { - return b.b[b.off:] -} - -// remain will return the number of bytes remaining. -func (b byteReader) remain() int { - return len(b.b) - b.off -} diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go deleted file mode 100644 index 6a5a2988b..000000000 --- a/vendor/github.com/klauspost/compress/zstd/decodeheader.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2020+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package zstd - -import ( - "encoding/binary" - "errors" - "io" -) - -// HeaderMaxSize is the maximum size of a Frame and Block Header. -// If less is sent to Header.Decode it *may* still contain enough information. -const HeaderMaxSize = 14 + 3 - -// Header contains information about the first frame and block within that. -type Header struct { - // SingleSegment specifies whether the data is to be decompressed into a - // single contiguous memory segment. - // It implies that WindowSize is invalid and that FrameContentSize is valid. - SingleSegment bool - - // WindowSize is the window of data to keep while decoding. - // Will only be set if SingleSegment is false. - WindowSize uint64 - - // Dictionary ID. - // If 0, no dictionary. - DictionaryID uint32 - - // HasFCS specifies whether FrameContentSize has a valid value. - HasFCS bool - - // FrameContentSize is the expected uncompressed size of the entire frame. - FrameContentSize uint64 - - // Skippable will be true if the frame is meant to be skipped. - // This implies that FirstBlock.OK is false. - Skippable bool - - // SkippableID is the user-specific ID for the skippable frame. - // Valid values are between 0 to 15, inclusive. - SkippableID int - - // SkippableSize is the length of the user data to skip following - // the header. - SkippableSize uint32 - - // HeaderSize is the raw size of the frame header. - // - // For normal frames, it includes the size of the magic number and - // the size of the header (per section 3.1.1.1). - // It does not include the size for any data blocks (section 3.1.1.2) nor - // the size for the trailing content checksum. - // - // For skippable frames, this counts the size of the magic number - // along with the size of the size field of the payload. - // It does not include the size of the skippable payload itself. - // The total frame size is the HeaderSize plus the SkippableSize. - HeaderSize int - - // First block information. - FirstBlock struct { - // OK will be set if first block could be decoded. - OK bool - - // Is this the last block of a frame? - Last bool - - // Is the data compressed? - // If true CompressedSize will be populated. - // Unfortunately DecompressedSize cannot be determined - // without decoding the blocks. - Compressed bool - - // DecompressedSize is the expected decompressed size of the block. - // Will be 0 if it cannot be determined. - DecompressedSize int - - // CompressedSize of the data in the block. - // Does not include the block header. - // Will be equal to DecompressedSize if not Compressed. - CompressedSize int - } - - // If set there is a checksum present for the block content. - // The checksum field at the end is always 4 bytes long. - HasCheckSum bool -} - -// Decode the header from the beginning of the stream. -// This will decode the frame header and the first block header if enough bytes are provided. -// It is recommended to provide at least HeaderMaxSize bytes. -// If the frame header cannot be read an error will be returned. -// If there isn't enough input, io.ErrUnexpectedEOF is returned. -// The FirstBlock.OK will indicate if enough information was available to decode the first block header. -func (h *Header) Decode(in []byte) error { - _, err := h.DecodeAndStrip(in) - return err -} - -// DecodeAndStrip will decode the header from the beginning of the stream -// and on success return the remaining bytes. -// This will decode the frame header and the first block header if enough bytes are provided. -// It is recommended to provide at least HeaderMaxSize bytes. -// If the frame header cannot be read an error will be returned. -// If there isn't enough input, io.ErrUnexpectedEOF is returned. -// The FirstBlock.OK will indicate if enough information was available to decode the first block header. -func (h *Header) DecodeAndStrip(in []byte) (remain []byte, err error) { - *h = Header{} - if len(in) < 4 { - return nil, io.ErrUnexpectedEOF - } - h.HeaderSize += 4 - b, in := in[:4], in[4:] - if string(b) != frameMagic { - if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 { - return nil, ErrMagicMismatch - } - if len(in) < 4 { - return nil, io.ErrUnexpectedEOF - } - h.HeaderSize += 4 - h.Skippable = true - h.SkippableID = int(b[0] & 0xf) - h.SkippableSize = binary.LittleEndian.Uint32(in) - return in[4:], nil - } - - // Read Window_Descriptor - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor - if len(in) < 1 { - return nil, io.ErrUnexpectedEOF - } - fhd, in := in[0], in[1:] - h.HeaderSize++ - h.SingleSegment = fhd&(1<<5) != 0 - h.HasCheckSum = fhd&(1<<2) != 0 - if fhd&(1<<3) != 0 { - return nil, errors.New("reserved bit set on frame header") - } - - if !h.SingleSegment { - if len(in) < 1 { - return nil, io.ErrUnexpectedEOF - } - var wd byte - wd, in = in[0], in[1:] - h.HeaderSize++ - windowLog := 10 + (wd >> 3) - windowBase := uint64(1) << windowLog - windowAdd := (windowBase / 8) * uint64(wd&0x7) - h.WindowSize = windowBase + windowAdd - } - - // Read Dictionary_ID - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - if size := fhd & 3; size != 0 { - if size == 3 { - size = 4 - } - if len(in) < int(size) { - return nil, io.ErrUnexpectedEOF - } - b, in = in[:size], in[size:] - h.HeaderSize += int(size) - switch len(b) { - case 1: - h.DictionaryID = uint32(b[0]) - case 2: - h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) - case 4: - h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) - } - } - - // Read Frame_Content_Size - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size - var fcsSize int - v := fhd >> 6 - switch v { - case 0: - if h.SingleSegment { - fcsSize = 1 - } - default: - fcsSize = 1 << v - } - - if fcsSize > 0 { - h.HasFCS = true - if len(in) < fcsSize { - return nil, io.ErrUnexpectedEOF - } - b, in = in[:fcsSize], in[fcsSize:] - h.HeaderSize += int(fcsSize) - switch len(b) { - case 1: - h.FrameContentSize = uint64(b[0]) - case 2: - // When FCS_Field_Size is 2, the offset of 256 is added. - h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) + 256 - case 4: - h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) | (uint64(b[2]) << 16) | (uint64(b[3]) << 24) - case 8: - d1 := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) - d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24) - h.FrameContentSize = uint64(d1) | (uint64(d2) << 32) - } - } - - // Frame Header done, we will not fail from now on. - if len(in) < 3 { - return in, nil - } - tmp := in[:3] - bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) - h.FirstBlock.Last = bh&1 != 0 - blockType := blockType((bh >> 1) & 3) - // find size. - cSize := int(bh >> 3) - switch blockType { - case blockTypeReserved: - return in, nil - case blockTypeRLE: - h.FirstBlock.Compressed = true - h.FirstBlock.DecompressedSize = cSize - h.FirstBlock.CompressedSize = 1 - case blockTypeCompressed: - h.FirstBlock.Compressed = true - h.FirstBlock.CompressedSize = cSize - case blockTypeRaw: - h.FirstBlock.DecompressedSize = cSize - h.FirstBlock.CompressedSize = cSize - default: - panic("Invalid block type") - } - - h.FirstBlock.OK = true - return in, nil -} - -// AppendTo will append the encoded header to the dst slice. -// There is no error checking performed on the header values. -func (h *Header) AppendTo(dst []byte) ([]byte, error) { - if h.Skippable { - magic := [4]byte{0x50, 0x2a, 0x4d, 0x18} - magic[0] |= byte(h.SkippableID & 0xf) - dst = append(dst, magic[:]...) - f := h.SkippableSize - return append(dst, uint8(f), uint8(f>>8), uint8(f>>16), uint8(f>>24)), nil - } - f := frameHeader{ - ContentSize: h.FrameContentSize, - WindowSize: uint32(h.WindowSize), - SingleSegment: h.SingleSegment, - Checksum: h.HasCheckSum, - DictID: h.DictionaryID, - } - return f.appendTo(dst), nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go deleted file mode 100644 index bbca17234..000000000 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ /dev/null @@ -1,948 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "context" - "encoding/binary" - "io" - "sync" - - "github.com/klauspost/compress/zstd/internal/xxhash" -) - -// Decoder provides decoding of zstandard streams. -// The decoder has been designed to operate without allocations after a warmup. -// This means that you should store the decoder for best performance. -// To re-use a stream decoder, use the Reset(r io.Reader) error to switch to another stream. -// A decoder can safely be re-used even if the previous stream failed. -// To release the resources, you must call the Close() function on a decoder. -type Decoder struct { - o decoderOptions - - // Unreferenced decoders, ready for use. - decoders chan *blockDec - - // Current read position used for Reader functionality. - current decoderState - - // sync stream decoding - syncStream struct { - decodedFrame uint64 - br readerWrapper - enabled bool - inFrame bool - dstBuf []byte - } - - frame *frameDec - - // Custom dictionaries. - dicts map[uint32]*dict - - // streamWg is the waitgroup for all streams - streamWg sync.WaitGroup -} - -// decoderState is used for maintaining state when the decoder -// is used for streaming. -type decoderState struct { - // current block being written to stream. - decodeOutput - - // output in order to be written to stream. - output chan decodeOutput - - // cancel remaining output. - cancel context.CancelFunc - - // crc of current frame - crc *xxhash.Digest - - flushed bool -} - -var ( - // Check the interfaces we want to support. - _ = io.WriterTo(&Decoder{}) - _ = io.Reader(&Decoder{}) -) - -// NewReader creates a new decoder. -// A nil Reader can be provided in which case Reset can be used to start a decode. -// -// A Decoder can be used in two modes: -// -// 1) As a stream, or -// 2) For stateless decoding using DecodeAll. -// -// Only a single stream can be decoded concurrently, but the same decoder -// can run multiple concurrent stateless decodes. It is even possible to -// use stateless decodes while a stream is being decoded. -// -// The Reset function can be used to initiate a new stream, which will considerably -// reduce the allocations normally caused by NewReader. -func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { - initPredefined() - var d Decoder - d.o.setDefault() - for _, o := range opts { - err := o(&d.o) - if err != nil { - return nil, err - } - } - d.current.crc = xxhash.New() - d.current.flushed = true - - if r == nil { - d.current.err = ErrDecoderNilInput - } - - // Transfer option dicts. - d.dicts = make(map[uint32]*dict, len(d.o.dicts)) - for _, dc := range d.o.dicts { - d.dicts[dc.id] = dc - } - d.o.dicts = nil - - // Create decoders - d.decoders = make(chan *blockDec, d.o.concurrent) - for i := 0; i < d.o.concurrent; i++ { - dec := newBlockDec(d.o.lowMem) - dec.localFrame = newFrameDec(d.o) - d.decoders <- dec - } - - if r == nil { - return &d, nil - } - return &d, d.Reset(r) -} - -// Read bytes from the decompressed stream into p. -// Returns the number of bytes written and any error that occurred. -// When the stream is done, io.EOF will be returned. -func (d *Decoder) Read(p []byte) (int, error) { - var n int - for { - if len(d.current.b) > 0 { - filled := copy(p, d.current.b) - p = p[filled:] - d.current.b = d.current.b[filled:] - n += filled - } - if len(p) == 0 { - break - } - if len(d.current.b) == 0 { - // We have an error and no more data - if d.current.err != nil { - break - } - if !d.nextBlock(n == 0) { - return n, d.current.err - } - } - } - if len(d.current.b) > 0 { - if debugDecoder { - println("returning", n, "still bytes left:", len(d.current.b)) - } - // Only return error at end of block - return n, nil - } - if d.current.err != nil { - d.drainOutput() - } - if debugDecoder { - println("returning", n, d.current.err, len(d.decoders)) - } - return n, d.current.err -} - -// Reset will reset the decoder the supplied stream after the current has finished processing. -// Note that this functionality cannot be used after Close has been called. -// Reset can be called with a nil reader to release references to the previous reader. -// After being called with a nil reader, no other operations than Reset or DecodeAll or Close -// should be used. -func (d *Decoder) Reset(r io.Reader) error { - if d.current.err == ErrDecoderClosed { - return d.current.err - } - - d.drainOutput() - - d.syncStream.br.r = nil - if r == nil { - d.current.err = ErrDecoderNilInput - if len(d.current.b) > 0 { - d.current.b = d.current.b[:0] - } - d.current.flushed = true - return nil - } - - // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { - bb2 := bb - if debugDecoder { - println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) - } - b := bb2.Bytes() - var dst []byte - if cap(d.syncStream.dstBuf) > 0 { - dst = d.syncStream.dstBuf[:0] - } - - dst, err := d.DecodeAll(b, dst) - if err == nil { - err = io.EOF - } - // Save output buffer - d.syncStream.dstBuf = dst - d.current.b = dst - d.current.err = err - d.current.flushed = true - if debugDecoder { - println("sync decode to", len(dst), "bytes, err:", err) - } - return nil - } - // Remove current block. - d.stashDecoder() - d.current.decodeOutput = decodeOutput{} - d.current.err = nil - d.current.flushed = false - d.current.d = nil - d.syncStream.dstBuf = nil - - // Ensure no-one else is still running... - d.streamWg.Wait() - if d.frame == nil { - d.frame = newFrameDec(d.o) - } - - if d.o.concurrent == 1 { - return d.startSyncDecoder(r) - } - - d.current.output = make(chan decodeOutput, d.o.concurrent) - ctx, cancel := context.WithCancel(context.Background()) - d.current.cancel = cancel - d.streamWg.Add(1) - go d.startStreamDecoder(ctx, r, d.current.output) - - return nil -} - -// drainOutput will drain the output until errEndOfStream is sent. -func (d *Decoder) drainOutput() { - if d.current.cancel != nil { - if debugDecoder { - println("cancelling current") - } - d.current.cancel() - d.current.cancel = nil - } - if d.current.d != nil { - if debugDecoder { - printf("re-adding current decoder %p, decoders: %d", d.current.d, len(d.decoders)) - } - d.decoders <- d.current.d - d.current.d = nil - d.current.b = nil - } - if d.current.output == nil || d.current.flushed { - println("current already flushed") - return - } - for v := range d.current.output { - if v.d != nil { - if debugDecoder { - printf("re-adding decoder %p", v.d) - } - d.decoders <- v.d - } - } - d.current.output = nil - d.current.flushed = true -} - -// WriteTo writes data to w until there's no more data to write or when an error occurs. -// The return value n is the number of bytes written. -// Any error encountered during the write is also returned. -func (d *Decoder) WriteTo(w io.Writer) (int64, error) { - var n int64 - for { - if len(d.current.b) > 0 { - n2, err2 := w.Write(d.current.b) - n += int64(n2) - if err2 != nil && (d.current.err == nil || d.current.err == io.EOF) { - d.current.err = err2 - } else if n2 != len(d.current.b) { - d.current.err = io.ErrShortWrite - } - } - if d.current.err != nil { - break - } - d.nextBlock(true) - } - err := d.current.err - if err != nil { - d.drainOutput() - } - if err == io.EOF { - err = nil - } - return n, err -} - -// DecodeAll allows stateless decoding of a blob of bytes. -// Output will be appended to dst, so if the destination size is known -// you can pre-allocate the destination slice to avoid allocations. -// DecodeAll can be used concurrently. -// The Decoder concurrency limits will be respected. -func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { - if d.decoders == nil { - return dst, ErrDecoderClosed - } - - // Grab a block decoder and frame decoder. - block := <-d.decoders - frame := block.localFrame - initialSize := len(dst) - defer func() { - if debugDecoder { - printf("re-adding decoder: %p", block) - } - frame.rawInput = nil - frame.bBuf = nil - if frame.history.decoders.br != nil { - frame.history.decoders.br.in = nil - } - d.decoders <- block - }() - frame.bBuf = input - - for { - frame.history.reset() - err := frame.reset(&frame.bBuf) - if err != nil { - if err == io.EOF { - if debugDecoder { - println("frame reset return EOF") - } - return dst, nil - } - return dst, err - } - if err = d.setDict(frame); err != nil { - return nil, err - } - if frame.WindowSize > d.o.maxWindowSize { - if debugDecoder { - println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) - } - return dst, ErrWindowSizeExceeded - } - if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { - if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) - } - return dst, ErrDecoderSizeExceeded - } - if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { - if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) - } - return dst, ErrDecoderSizeExceeded - } - if cap(dst)-len(dst) < int(frame.FrameContentSize) { - dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)+compressedBlockOverAlloc) - copy(dst2, dst) - dst = dst2 - } - } - - if cap(dst) == 0 && !d.o.limitToCap { - // Allocate len(input) * 2 by default if nothing is provided - // and we didn't get frame content size. - size := len(input) * 2 - // Cap to 1 MB. - if size > 1<<20 { - size = 1 << 20 - } - if uint64(size) > d.o.maxDecodedSize { - size = int(d.o.maxDecodedSize) - } - dst = make([]byte, 0, size) - } - - dst, err = frame.runDecoder(dst, block) - if err != nil { - return dst, err - } - if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { - return dst, ErrDecoderSizeExceeded - } - if len(frame.bBuf) == 0 { - if debugDecoder { - println("frame dbuf empty") - } - break - } - } - return dst, nil -} - -// nextBlock returns the next block. -// If an error occurs d.err will be set. -// Optionally the function can block for new output. -// If non-blocking mode is used the returned boolean will be false -// if no data was available without blocking. -func (d *Decoder) nextBlock(blocking bool) (ok bool) { - if d.current.err != nil { - // Keep error state. - return false - } - d.current.b = d.current.b[:0] - - // SYNC: - if d.syncStream.enabled { - if !blocking { - return false - } - ok = d.nextBlockSync() - if !ok { - d.stashDecoder() - } - return ok - } - - //ASYNC: - d.stashDecoder() - if blocking { - d.current.decodeOutput, ok = <-d.current.output - } else { - select { - case d.current.decodeOutput, ok = <-d.current.output: - default: - return false - } - } - if !ok { - // This should not happen, so signal error state... - d.current.err = io.ErrUnexpectedEOF - return false - } - next := d.current.decodeOutput - if next.d != nil && next.d.async.newHist != nil { - d.current.crc.Reset() - } - if debugDecoder { - var tmp [4]byte - binary.LittleEndian.PutUint32(tmp[:], uint32(xxhash.Sum64(next.b))) - println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) - } - - if d.o.ignoreChecksum { - return true - } - - if len(next.b) > 0 { - d.current.crc.Write(next.b) - } - if next.err == nil && next.d != nil && next.d.hasCRC { - got := uint32(d.current.crc.Sum64()) - if got != next.d.checkCRC { - if debugDecoder { - printf("CRC Check Failed: %08x (got) != %08x (on stream)\n", got, next.d.checkCRC) - } - d.current.err = ErrCRCMismatch - } else { - if debugDecoder { - printf("CRC ok %08x\n", got) - } - } - } - - return true -} - -func (d *Decoder) nextBlockSync() (ok bool) { - if d.current.d == nil { - d.current.d = <-d.decoders - } - for len(d.current.b) == 0 { - if !d.syncStream.inFrame { - d.frame.history.reset() - d.current.err = d.frame.reset(&d.syncStream.br) - if d.current.err == nil { - d.current.err = d.setDict(d.frame) - } - if d.current.err != nil { - return false - } - if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { - d.current.err = ErrDecoderSizeExceeded - return false - } - - d.syncStream.decodedFrame = 0 - d.syncStream.inFrame = true - } - d.current.err = d.frame.next(d.current.d) - if d.current.err != nil { - return false - } - d.frame.history.ensureBlock() - if debugDecoder { - println("History trimmed:", len(d.frame.history.b), "decoded already:", d.syncStream.decodedFrame) - } - histBefore := len(d.frame.history.b) - d.current.err = d.current.d.decodeBuf(&d.frame.history) - - if d.current.err != nil { - println("error after:", d.current.err) - return false - } - d.current.b = d.frame.history.b[histBefore:] - if debugDecoder { - println("history after:", len(d.frame.history.b)) - } - - // Check frame size (before CRC) - d.syncStream.decodedFrame += uint64(len(d.current.b)) - if d.syncStream.decodedFrame > d.frame.FrameContentSize { - if debugDecoder { - printf("DecodedFrame (%d) > FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize) - } - d.current.err = ErrFrameSizeExceeded - return false - } - - // Check FCS - if d.current.d.Last && d.frame.FrameContentSize != fcsUnknown && d.syncStream.decodedFrame != d.frame.FrameContentSize { - if debugDecoder { - printf("DecodedFrame (%d) != FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize) - } - d.current.err = ErrFrameSizeMismatch - return false - } - - // Update/Check CRC - if d.frame.HasCheckSum { - if !d.o.ignoreChecksum { - d.frame.crc.Write(d.current.b) - } - if d.current.d.Last { - if !d.o.ignoreChecksum { - d.current.err = d.frame.checkCRC() - } else { - d.current.err = d.frame.consumeCRC() - } - if d.current.err != nil { - println("CRC error:", d.current.err) - return false - } - } - } - d.syncStream.inFrame = !d.current.d.Last - } - return true -} - -func (d *Decoder) stashDecoder() { - if d.current.d != nil { - if debugDecoder { - printf("re-adding current decoder %p", d.current.d) - } - d.decoders <- d.current.d - d.current.d = nil - } -} - -// Close will release all resources. -// It is NOT possible to reuse the decoder after this. -func (d *Decoder) Close() { - if d.current.err == ErrDecoderClosed { - return - } - d.drainOutput() - if d.current.cancel != nil { - d.current.cancel() - d.streamWg.Wait() - d.current.cancel = nil - } - if d.decoders != nil { - close(d.decoders) - for dec := range d.decoders { - dec.Close() - } - d.decoders = nil - } - if d.current.d != nil { - d.current.d.Close() - d.current.d = nil - } - d.current.err = ErrDecoderClosed -} - -// IOReadCloser returns the decoder as an io.ReadCloser for convenience. -// Any changes to the decoder will be reflected, so the returned ReadCloser -// can be reused along with the decoder. -// io.WriterTo is also supported by the returned ReadCloser. -func (d *Decoder) IOReadCloser() io.ReadCloser { - return closeWrapper{d: d} -} - -// closeWrapper wraps a function call as a closer. -type closeWrapper struct { - d *Decoder -} - -// WriteTo forwards WriteTo calls to the decoder. -func (c closeWrapper) WriteTo(w io.Writer) (n int64, err error) { - return c.d.WriteTo(w) -} - -// Read forwards read calls to the decoder. -func (c closeWrapper) Read(p []byte) (n int, err error) { - return c.d.Read(p) -} - -// Close closes the decoder. -func (c closeWrapper) Close() error { - c.d.Close() - return nil -} - -type decodeOutput struct { - d *blockDec - b []byte - err error -} - -func (d *Decoder) startSyncDecoder(r io.Reader) error { - d.frame.history.reset() - d.syncStream.br = readerWrapper{r: r} - d.syncStream.inFrame = false - d.syncStream.enabled = true - d.syncStream.decodedFrame = 0 - return nil -} - -// Create Decoder: -// ASYNC: -// Spawn 3 go routines. -// 0: Read frames and decode block literals. -// 1: Decode sequences. -// 2: Execute sequences, send to output. -func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output chan decodeOutput) { - defer d.streamWg.Done() - br := readerWrapper{r: r} - - var seqDecode = make(chan *blockDec, d.o.concurrent) - var seqExecute = make(chan *blockDec, d.o.concurrent) - - // Async 1: Decode sequences... - go func() { - var hist history - var hasErr bool - - for block := range seqDecode { - if hasErr { - if block != nil { - seqExecute <- block - } - continue - } - if block.async.newHist != nil { - if debugDecoder { - println("Async 1: new history, recent:", block.async.newHist.recentOffsets) - } - hist.reset() - hist.decoders = block.async.newHist.decoders - hist.recentOffsets = block.async.newHist.recentOffsets - hist.windowSize = block.async.newHist.windowSize - if block.async.newHist.dict != nil { - hist.setDict(block.async.newHist.dict) - } - } - if block.err != nil || block.Type != blockTypeCompressed { - hasErr = block.err != nil - seqExecute <- block - continue - } - - hist.decoders.literals = block.async.literals - block.err = block.prepareSequences(block.async.seqData, &hist) - if debugDecoder && block.err != nil { - println("prepareSequences returned:", block.err) - } - hasErr = block.err != nil - if block.err == nil { - block.err = block.decodeSequences(&hist) - if debugDecoder && block.err != nil { - println("decodeSequences returned:", block.err) - } - hasErr = block.err != nil - // block.async.sequence = hist.decoders.seq[:hist.decoders.nSeqs] - block.async.seqSize = hist.decoders.seqSize - } - seqExecute <- block - } - close(seqExecute) - hist.reset() - }() - - var wg sync.WaitGroup - wg.Add(1) - - // Async 3: Execute sequences... - frameHistCache := d.frame.history.b - go func() { - var hist history - var decodedFrame uint64 - var fcs uint64 - var hasErr bool - for block := range seqExecute { - out := decodeOutput{err: block.err, d: block} - if block.err != nil || hasErr { - hasErr = true - output <- out - continue - } - if block.async.newHist != nil { - if debugDecoder { - println("Async 2: new history") - } - hist.reset() - hist.windowSize = block.async.newHist.windowSize - hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer - if block.async.newHist.dict != nil { - hist.setDict(block.async.newHist.dict) - } - - if cap(hist.b) < hist.allocFrameBuffer { - if cap(frameHistCache) >= hist.allocFrameBuffer { - hist.b = frameHistCache - } else { - hist.b = make([]byte, 0, hist.allocFrameBuffer) - println("Alloc history sized", hist.allocFrameBuffer) - } - } - hist.b = hist.b[:0] - fcs = block.async.fcs - decodedFrame = 0 - } - do := decodeOutput{err: block.err, d: block} - switch block.Type { - case blockTypeRLE: - if debugDecoder { - println("add rle block length:", block.RLESize) - } - - if cap(block.dst) < int(block.RLESize) { - if block.lowMem { - block.dst = make([]byte, block.RLESize) - } else { - block.dst = make([]byte, maxCompressedBlockSize) - } - } - block.dst = block.dst[:block.RLESize] - v := block.data[0] - for i := range block.dst { - block.dst[i] = v - } - hist.append(block.dst) - do.b = block.dst - case blockTypeRaw: - if debugDecoder { - println("add raw block length:", len(block.data)) - } - hist.append(block.data) - do.b = block.data - case blockTypeCompressed: - if debugDecoder { - println("execute with history length:", len(hist.b), "window:", hist.windowSize) - } - hist.decoders.seqSize = block.async.seqSize - hist.decoders.literals = block.async.literals - do.err = block.executeSequences(&hist) - hasErr = do.err != nil - if debugDecoder && hasErr { - println("executeSequences returned:", do.err) - } - do.b = block.dst - } - if !hasErr { - decodedFrame += uint64(len(do.b)) - if decodedFrame > fcs { - println("fcs exceeded", block.Last, fcs, decodedFrame) - do.err = ErrFrameSizeExceeded - hasErr = true - } else if block.Last && fcs != fcsUnknown && decodedFrame != fcs { - do.err = ErrFrameSizeMismatch - hasErr = true - } else { - if debugDecoder { - println("fcs ok", block.Last, fcs, decodedFrame) - } - } - } - output <- do - } - close(output) - frameHistCache = hist.b - wg.Done() - if debugDecoder { - println("decoder goroutines finished") - } - hist.reset() - }() - - var hist history -decodeStream: - for { - var hasErr bool - hist.reset() - decodeBlock := func(block *blockDec) { - if hasErr { - if block != nil { - seqDecode <- block - } - return - } - if block.err != nil || block.Type != blockTypeCompressed { - hasErr = block.err != nil - seqDecode <- block - return - } - - remain, err := block.decodeLiterals(block.data, &hist) - block.err = err - hasErr = block.err != nil - if err == nil { - block.async.literals = hist.decoders.literals - block.async.seqData = remain - } else if debugDecoder { - println("decodeLiterals error:", err) - } - seqDecode <- block - } - frame := d.frame - if debugDecoder { - println("New frame...") - } - var historySent bool - frame.history.reset() - err := frame.reset(&br) - if debugDecoder && err != nil { - println("Frame decoder returned", err) - } - if err == nil { - err = d.setDict(frame) - } - if err == nil && d.frame.WindowSize > d.o.maxWindowSize { - if debugDecoder { - println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) - } - - err = ErrDecoderSizeExceeded - } - if err != nil { - select { - case <-ctx.Done(): - case dec := <-d.decoders: - dec.sendErr(err) - decodeBlock(dec) - } - break decodeStream - } - - // Go through all blocks of the frame. - for { - var dec *blockDec - select { - case <-ctx.Done(): - break decodeStream - case dec = <-d.decoders: - // Once we have a decoder, we MUST return it. - } - err := frame.next(dec) - if !historySent { - h := frame.history - if debugDecoder { - println("Alloc History:", h.allocFrameBuffer) - } - hist.reset() - if h.dict != nil { - hist.setDict(h.dict) - } - dec.async.newHist = &h - dec.async.fcs = frame.FrameContentSize - historySent = true - } else { - dec.async.newHist = nil - } - if debugDecoder && err != nil { - println("next block returned error:", err) - } - dec.err = err - dec.hasCRC = false - if dec.Last && frame.HasCheckSum && err == nil { - crc, err := frame.rawInput.readSmall(4) - if len(crc) < 4 { - if err == nil { - err = io.ErrUnexpectedEOF - - } - println("CRC missing?", err) - dec.err = err - } else { - dec.checkCRC = binary.LittleEndian.Uint32(crc) - dec.hasCRC = true - if debugDecoder { - printf("found crc to check: %08x\n", dec.checkCRC) - } - } - } - err = dec.err - last := dec.Last - decodeBlock(dec) - if err != nil { - break decodeStream - } - if last { - break - } - } - } - close(seqDecode) - wg.Wait() - hist.reset() - d.frame.history.b = frameHistCache -} - -func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.dicts[frame.DictionaryID] - if ok { - if debugDecoder { - println("setting dict", frame.DictionaryID) - } - frame.history.setDict(dict) - } else if frame.DictionaryID != 0 { - // A zero or missing dictionary id is ambiguous: - // either dictionary zero, or no dictionary. In particular, - // zstd --patch-from uses this id for the source file, - // so only return an error if the dictionary id is not zero. - err = ErrUnknownDictionary - } - return err -} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go deleted file mode 100644 index 774c5f00f..000000000 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "errors" - "fmt" - "math/bits" - "runtime" -) - -// DOption is an option for creating a decoder. -type DOption func(*decoderOptions) error - -// options retains accumulated state of multiple options. -type decoderOptions struct { - lowMem bool - concurrent int - maxDecodedSize uint64 - maxWindowSize uint64 - dicts []*dict - ignoreChecksum bool - limitToCap bool - decodeBufsBelow int -} - -func (o *decoderOptions) setDefault() { - *o = decoderOptions{ - // use less ram: true for now, but may change. - lowMem: true, - concurrent: runtime.GOMAXPROCS(0), - maxWindowSize: MaxWindowSize, - decodeBufsBelow: 128 << 10, - } - if o.concurrent > 4 { - o.concurrent = 4 - } - o.maxDecodedSize = 64 << 30 -} - -// WithDecoderLowmem will set whether to use a lower amount of memory, -// but possibly have to allocate more while running. -func WithDecoderLowmem(b bool) DOption { - return func(o *decoderOptions) error { o.lowMem = b; return nil } -} - -// WithDecoderConcurrency sets the number of created decoders. -// When decoding block with DecodeAll, this will limit the number -// of possible concurrently running decodes. -// When decoding streams, this will limit the number of -// inflight blocks. -// When decoding streams and setting maximum to 1, -// no async decoding will be done. -// When a value of 0 is provided GOMAXPROCS will be used. -// By default this will be set to 4 or GOMAXPROCS, whatever is lower. -func WithDecoderConcurrency(n int) DOption { - return func(o *decoderOptions) error { - if n < 0 { - return errors.New("concurrency must be at least 1") - } - if n == 0 { - o.concurrent = runtime.GOMAXPROCS(0) - } else { - o.concurrent = n - } - return nil - } -} - -// WithDecoderMaxMemory allows to set a maximum decoded size for in-memory -// non-streaming operations or maximum window size for streaming operations. -// This can be used to control memory usage of potentially hostile content. -// Maximum is 1 << 63 bytes. Default is 64GiB. -func WithDecoderMaxMemory(n uint64) DOption { - return func(o *decoderOptions) error { - if n == 0 { - return errors.New("WithDecoderMaxMemory must be at least 1") - } - if n > 1<<63 { - return errors.New("WithDecoderMaxmemory must be less than 1 << 63") - } - o.maxDecodedSize = n - return nil - } -} - -// WithDecoderDicts allows to register one or more dictionaries for the decoder. -// -// Each slice in dict must be in the [dictionary format] produced by -// "zstd --train" from the Zstandard reference implementation. -// -// If several dictionaries with the same ID are provided, the last one will be used. -// -// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format -func WithDecoderDicts(dicts ...[]byte) DOption { - return func(o *decoderOptions) error { - for _, b := range dicts { - d, err := loadDict(b) - if err != nil { - return err - } - o.dicts = append(o.dicts, d) - } - return nil - } -} - -// WithDecoderDictRaw registers a dictionary that may be used by the decoder. -// The slice content can be arbitrary data. -func WithDecoderDictRaw(id uint32, content []byte) DOption { - return func(o *decoderOptions) error { - if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { - return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) - } - o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) - return nil - } -} - -// WithDecoderMaxWindow allows to set a maximum window size for decodes. -// This allows rejecting packets that will cause big memory usage. -// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. -// If WithDecoderMaxMemory is set to a lower value, that will be used. -// Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. -func WithDecoderMaxWindow(size uint64) DOption { - return func(o *decoderOptions) error { - if size < MinWindowSize { - return errors.New("WithMaxWindowSize must be at least 1KB, 1024 bytes") - } - if size > (1<<41)+7*(1<<38) { - return errors.New("WithMaxWindowSize must be less than (1<<41) + 7*(1<<38) ~ 3.75TB") - } - o.maxWindowSize = size - return nil - } -} - -// WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes, -// or any size set in WithDecoderMaxMemory. -// This can be used to limit decoding to a specific maximum output size. -// Disabled by default. -func WithDecodeAllCapLimit(b bool) DOption { - return func(o *decoderOptions) error { - o.limitToCap = b - return nil - } -} - -// WithDecodeBuffersBelow will fully decode readers that have a -// `Bytes() []byte` and `Len() int` interface similar to bytes.Buffer. -// This typically uses less allocations but will have the full decompressed object in memory. -// Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. -// Default is 128KiB. -func WithDecodeBuffersBelow(size int) DOption { - return func(o *decoderOptions) error { - o.decodeBufsBelow = size - return nil - } -} - -// IgnoreChecksum allows to forcibly ignore checksum checking. -func IgnoreChecksum(b bool) DOption { - return func(o *decoderOptions) error { - o.ignoreChecksum = b - return nil - } -} diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go deleted file mode 100644 index b7b83164b..000000000 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ /dev/null @@ -1,565 +0,0 @@ -package zstd - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "math" - "sort" - - "github.com/klauspost/compress/huff0" -) - -type dict struct { - id uint32 - - litEnc *huff0.Scratch - llDec, ofDec, mlDec sequenceDec - offsets [3]int - content []byte -} - -const dictMagic = "\x37\xa4\x30\xec" - -// Maximum dictionary size for the reference implementation (1.5.3) is 2 GiB. -const dictMaxLength = 1 << 31 - -// ID returns the dictionary id or 0 if d is nil. -func (d *dict) ID() uint32 { - if d == nil { - return 0 - } - return d.id -} - -// ContentSize returns the dictionary content size or 0 if d is nil. -func (d *dict) ContentSize() int { - if d == nil { - return 0 - } - return len(d.content) -} - -// Content returns the dictionary content. -func (d *dict) Content() []byte { - if d == nil { - return nil - } - return d.content -} - -// Offsets returns the initial offsets. -func (d *dict) Offsets() [3]int { - if d == nil { - return [3]int{} - } - return d.offsets -} - -// LitEncoder returns the literal encoder. -func (d *dict) LitEncoder() *huff0.Scratch { - if d == nil { - return nil - } - return d.litEnc -} - -// Load a dictionary as described in -// https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format -func loadDict(b []byte) (*dict, error) { - // Check static field size. - if len(b) <= 8+(3*4) { - return nil, io.ErrUnexpectedEOF - } - d := dict{ - llDec: sequenceDec{fse: &fseDecoder{}}, - ofDec: sequenceDec{fse: &fseDecoder{}}, - mlDec: sequenceDec{fse: &fseDecoder{}}, - } - if string(b[:4]) != dictMagic { - return nil, ErrMagicMismatch - } - d.id = binary.LittleEndian.Uint32(b[4:8]) - if d.id == 0 { - return nil, errors.New("dictionaries cannot have ID 0") - } - - // Read literal table - var err error - d.litEnc, b, err = huff0.ReadTable(b[8:], nil) - if err != nil { - return nil, fmt.Errorf("loading literal table: %w", err) - } - d.litEnc.Reuse = huff0.ReusePolicyMust - - br := byteReader{ - b: b, - off: 0, - } - readDec := func(i tableIndex, dec *fseDecoder) error { - if err := dec.readNCount(&br, uint16(maxTableSymbol[i])); err != nil { - return err - } - if br.overread() { - return io.ErrUnexpectedEOF - } - err = dec.transform(symbolTableX[i]) - if err != nil { - println("Transform table error:", err) - return err - } - if debugDecoder || debugEncoder { - println("Read table ok", "symbolLen:", dec.symbolLen) - } - // Set decoders as predefined so they aren't reused. - dec.preDefined = true - return nil - } - - if err := readDec(tableOffsets, d.ofDec.fse); err != nil { - return nil, err - } - if err := readDec(tableMatchLengths, d.mlDec.fse); err != nil { - return nil, err - } - if err := readDec(tableLiteralLengths, d.llDec.fse); err != nil { - return nil, err - } - if br.remain() < 12 { - return nil, io.ErrUnexpectedEOF - } - - d.offsets[0] = int(br.Uint32()) - br.advance(4) - d.offsets[1] = int(br.Uint32()) - br.advance(4) - d.offsets[2] = int(br.Uint32()) - br.advance(4) - if d.offsets[0] <= 0 || d.offsets[1] <= 0 || d.offsets[2] <= 0 { - return nil, errors.New("invalid offset in dictionary") - } - d.content = make([]byte, br.remain()) - copy(d.content, br.unread()) - if d.offsets[0] > len(d.content) || d.offsets[1] > len(d.content) || d.offsets[2] > len(d.content) { - return nil, fmt.Errorf("initial offset bigger than dictionary content size %d, offsets: %v", len(d.content), d.offsets) - } - - return &d, nil -} - -// InspectDictionary loads a zstd dictionary and provides functions to inspect the content. -func InspectDictionary(b []byte) (interface { - ID() uint32 - ContentSize() int - Content() []byte - Offsets() [3]int - LitEncoder() *huff0.Scratch -}, error) { - initPredefined() - d, err := loadDict(b) - return d, err -} - -type BuildDictOptions struct { - // Dictionary ID. - ID uint32 - - // Content to use to create dictionary tables. - Contents [][]byte - - // History to use for all blocks. - History []byte - - // Offsets to use. - Offsets [3]int - - // CompatV155 will make the dictionary compatible with Zstd v1.5.5 and earlier. - // See https://github.com/facebook/zstd/issues/3724 - CompatV155 bool - - // Use the specified encoder level. - // The dictionary will be built using the specified encoder level, - // which will reflect speed and make the dictionary tailored for that level. - // If not set SpeedBestCompression will be used. - Level EncoderLevel - - // DebugOut will write stats and other details here if set. - DebugOut io.Writer -} - -func BuildDict(o BuildDictOptions) ([]byte, error) { - initPredefined() - hist := o.History - contents := o.Contents - debug := o.DebugOut != nil - println := func(args ...interface{}) { - if o.DebugOut != nil { - fmt.Fprintln(o.DebugOut, args...) - } - } - printf := func(s string, args ...interface{}) { - if o.DebugOut != nil { - fmt.Fprintf(o.DebugOut, s, args...) - } - } - print := func(args ...interface{}) { - if o.DebugOut != nil { - fmt.Fprint(o.DebugOut, args...) - } - } - - if int64(len(hist)) > dictMaxLength { - return nil, fmt.Errorf("dictionary of size %d > %d", len(hist), int64(dictMaxLength)) - } - if len(hist) < 8 { - return nil, fmt.Errorf("dictionary of size %d < %d", len(hist), 8) - } - if len(contents) == 0 { - return nil, errors.New("no content provided") - } - d := dict{ - id: o.ID, - litEnc: nil, - llDec: sequenceDec{}, - ofDec: sequenceDec{}, - mlDec: sequenceDec{}, - offsets: o.Offsets, - content: hist, - } - block := blockEnc{lowMem: false} - block.init() - enc := encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}}) - if o.Level != 0 { - eOpts := encoderOptions{ - level: o.Level, - blockSize: maxMatchLen, - windowSize: maxMatchLen, - dict: &d, - lowMem: false, - } - enc = eOpts.encoder() - } else { - o.Level = SpeedBestCompression - } - var ( - remain [256]int - ll [256]int - ml [256]int - of [256]int - ) - addValues := func(dst *[256]int, src []byte) { - for _, v := range src { - dst[v]++ - } - } - addHist := func(dst *[256]int, src *[256]uint32) { - for i, v := range src { - dst[i] += int(v) - } - } - seqs := 0 - nUsed := 0 - litTotal := 0 - newOffsets := make(map[uint32]int, 1000) - for _, b := range contents { - block.reset(nil) - if len(b) < 8 { - continue - } - nUsed++ - enc.Reset(&d, true) - enc.Encode(&block, b) - addValues(&remain, block.literals) - litTotal += len(block.literals) - if len(block.sequences) == 0 { - continue - } - seqs += len(block.sequences) - block.genCodes() - addHist(&ll, block.coders.llEnc.Histogram()) - addHist(&ml, block.coders.mlEnc.Histogram()) - addHist(&of, block.coders.ofEnc.Histogram()) - for i, seq := range block.sequences { - if i > 3 { - break - } - offset := seq.offset - if offset == 0 { - continue - } - if int(offset) >= len(o.History) { - continue - } - if offset > 3 { - newOffsets[offset-3]++ - } else { - newOffsets[uint32(o.Offsets[offset-1])]++ - } - } - } - // Find most used offsets. - var sortedOffsets []uint32 - for k := range newOffsets { - sortedOffsets = append(sortedOffsets, k) - } - sort.Slice(sortedOffsets, func(i, j int) bool { - a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { - // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] - } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] - }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) - } - println("") - } - - sortedOffsets = sortedOffsets[:3] - } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) - } - if debug { - println("New repeat offsets", o.Offsets) - } - - if nUsed == 0 || seqs == 0 { - return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) - } - if debug { - println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) - } - if seqs/nUsed < 512 { - // Use 512 as minimum. - nUsed = seqs / 512 - if nUsed == 0 { - nUsed = 1 - } - } - copyHist := func(dst *fseEncoder, src *[256]int) ([]byte, error) { - hist := dst.Histogram() - var maxSym uint8 - var maxCount int - var fakeLength int - for i, v := range src { - if v > 0 { - v = v / nUsed - if v == 0 { - v = 1 - } - } - if v > maxCount { - maxCount = v - } - if v != 0 { - maxSym = uint8(i) - } - fakeLength += v - hist[i] = uint32(v) - } - - // Ensure we aren't trying to represent RLE. - if maxCount == fakeLength { - for i := range hist { - if uint8(i) == maxSym { - fakeLength++ - maxSym++ - hist[i+1] = 1 - if maxSym > 1 { - break - } - } - if hist[0] == 0 { - fakeLength++ - hist[i] = 1 - if maxSym > 1 { - break - } - } - } - } - - dst.HistogramFinished(maxSym, maxCount) - dst.reUsed = false - dst.useRLE = false - err := dst.normalizeCount(fakeLength) - if err != nil { - return nil, err - } - if debug { - println("RAW:", dst.count[:maxSym+1], "NORM:", dst.norm[:maxSym+1], "LEN:", fakeLength) - } - return dst.writeCount(nil) - } - if debug { - print("Literal lengths: ") - } - llTable, err := copyHist(block.coders.llEnc, &ll) - if err != nil { - return nil, err - } - if debug { - print("Match lengths: ") - } - mlTable, err := copyHist(block.coders.mlEnc, &ml) - if err != nil { - return nil, err - } - if debug { - print("Offsets: ") - } - ofTable, err := copyHist(block.coders.ofEnc, &of) - if err != nil { - return nil, err - } - - // Literal table - avgSize := litTotal - if avgSize > huff0.BlockSizeMax/2 { - avgSize = huff0.BlockSizeMax / 2 - } - huffBuff := make([]byte, 0, avgSize) - // Target size - div := litTotal / avgSize - if div < 1 { - div = 1 - } - if debug { - println("Huffman weights:") - } - for i, n := range remain[:] { - if n > 0 { - n = n / div - // Allow all entries to be represented. - if n == 0 { - n = 1 - } - huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...) - if debug { - printf("[%d: %d], ", i, n) - } - } - } - if o.CompatV155 && remain[255]/div == 0 { - huffBuff = append(huffBuff, 255) - } - scratch := &huff0.Scratch{TableLog: 11} - for tries := 0; tries < 255; tries++ { - scratch = &huff0.Scratch{TableLog: 11} - _, _, err = huff0.Compress1X(huffBuff, scratch) - if err == nil { - break - } - if debug { - printf("Try %d: Huffman error: %v\n", tries+1, err) - } - huffBuff = huffBuff[:0] - if tries == 250 { - if debug { - println("Huffman: Bailing out with predefined table") - } - - // Bail out.... Just generate something - huffBuff = append(huffBuff, bytes.Repeat([]byte{255}, 10000)...) - for i := 0; i < 128; i++ { - huffBuff = append(huffBuff, byte(i)) - } - continue - } - if errors.Is(err, huff0.ErrIncompressible) { - // Try truncating least common. - for i, n := range remain[:] { - if n > 0 { - n = n / (div * (i + 1)) - if n > 0 { - huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...) - } - } - } - if o.CompatV155 && len(huffBuff) > 0 && huffBuff[len(huffBuff)-1] != 255 { - huffBuff = append(huffBuff, 255) - } - if len(huffBuff) == 0 { - huffBuff = append(huffBuff, 0, 255) - } - } - if errors.Is(err, huff0.ErrUseRLE) { - for i, n := range remain[:] { - n = n / (div * (i + 1)) - // Allow all entries to be represented. - if n == 0 { - n = 1 - } - huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...) - } - } - } - - var out bytes.Buffer - out.Write([]byte(dictMagic)) - out.Write(binary.LittleEndian.AppendUint32(nil, o.ID)) - out.Write(scratch.OutTable) - if debug { - println("huff table:", len(scratch.OutTable), "bytes") - println("of table:", len(ofTable), "bytes") - println("ml table:", len(mlTable), "bytes") - println("ll table:", len(llTable), "bytes") - } - out.Write(ofTable) - out.Write(mlTable) - out.Write(llTable) - out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[0]))) - out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) - out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) - out.Write(hist) - if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } - i, err := InspectDictionary(out.Bytes()) - if err != nil { - panic(err) - } - println("ID:", i.ID()) - println("Content size:", i.ContentSize()) - println("Encoder:", i.LitEncoder() != nil) - println("Offsets:", i.Offsets()) - var totalSize int - for _, b := range contents { - totalSize += len(b) - } - - encWith := func(opts ...EOption) int { - enc, err := NewWriter(nil, opts...) - if err != nil { - panic(err) - } - defer enc.Close() - var dst []byte - var totalSize int - for _, b := range contents { - dst = enc.EncodeAll(b, dst[:0]) - totalSize += len(dst) - } - return totalSize - } - plain := encWith(WithEncoderLevel(o.Level)) - withDict := encWith(WithEncoderLevel(o.Level), WithEncoderDict(out.Bytes())) - println("Input size:", totalSize) - println("Plain Compressed:", plain) - println("Dict Compressed:", withDict) - println("Saved:", plain-withDict, (plain-withDict)/len(contents), "bytes per input (rounded down)") - } - return out.Bytes(), nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go deleted file mode 100644 index 5ca46038a..000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ /dev/null @@ -1,173 +0,0 @@ -package zstd - -import ( - "fmt" - "math/bits" - - "github.com/klauspost/compress/zstd/internal/xxhash" -) - -const ( - dictShardBits = 6 -) - -type fastBase struct { - // cur is the offset at the start of hist - cur int32 - // maximum offset. Should be at least 2x block size. - maxMatchOff int32 - bufferReset int32 - hist []byte - crc *xxhash.Digest - tmp [8]byte - blk *blockEnc - lastDictID uint32 - lowMem bool -} - -// CRC returns the underlying CRC writer. -func (e *fastBase) CRC() *xxhash.Digest { - return e.crc -} - -// AppendCRC will append the CRC to the destination slice and return it. -func (e *fastBase) AppendCRC(dst []byte) []byte { - crc := e.crc.Sum(e.tmp[:0]) - dst = append(dst, crc[7], crc[6], crc[5], crc[4]) - return dst -} - -// WindowSize returns the window size of the encoder, -// or a window size small enough to contain the input size, if > 0. -func (e *fastBase) WindowSize(size int64) int32 { - if size > 0 && size < int64(e.maxMatchOff) { - b := int32(1) << uint(bits.Len(uint(size))) - // Keep minimum window. - if b < 1024 { - b = 1024 - } - return b - } - return e.maxMatchOff -} - -// Block returns the current block. -func (e *fastBase) Block() *blockEnc { - return e.blk -} - -func (e *fastBase) addBlock(src []byte) int32 { - if debugAsserts && e.cur > e.bufferReset { - panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, e.bufferReset)) - } - // check if we have space already - if len(e.hist)+len(src) > cap(e.hist) { - if cap(e.hist) == 0 { - e.ensureHist(len(src)) - } else { - if cap(e.hist) < int(e.maxMatchOff+maxCompressedBlockSize) { - panic(fmt.Errorf("unexpected buffer cap %d, want at least %d with window %d", cap(e.hist), e.maxMatchOff+maxCompressedBlockSize, e.maxMatchOff)) - } - // Move down - offset := int32(len(e.hist)) - e.maxMatchOff - copy(e.hist[0:e.maxMatchOff], e.hist[offset:]) - e.cur += offset - e.hist = e.hist[:e.maxMatchOff] - } - } - s := int32(len(e.hist)) - e.hist = append(e.hist, src...) - return s -} - -// ensureHist will ensure that history can keep at least this many bytes. -func (e *fastBase) ensureHist(n int) { - if cap(e.hist) >= n { - return - } - l := e.maxMatchOff - if (e.lowMem && e.maxMatchOff > maxCompressedBlockSize) || e.maxMatchOff <= maxCompressedBlockSize { - l += maxCompressedBlockSize - } else { - l += e.maxMatchOff - } - // Make it at least 1MB. - if l < 1<<20 && !e.lowMem { - l = 1 << 20 - } - // Make it at least the requested size. - if l < int32(n) { - l = int32(n) - } - e.hist = make([]byte, 0, l) -} - -// useBlock will replace the block with the provided one, -// but transfer recent offsets from the previous. -func (e *fastBase) UseBlock(enc *blockEnc) { - enc.reset(e.blk) - e.blk = enc -} - -func (e *fastBase) matchlen(s, t int32, src []byte) int32 { - if debugAsserts { - if s < 0 { - err := fmt.Sprintf("s (%d) < 0", s) - panic(err) - } - if t < 0 { - err := fmt.Sprintf("s (%d) < 0", s) - panic(err) - } - if s-t > e.maxMatchOff { - err := fmt.Sprintf("s (%d) - t (%d) > maxMatchOff (%d)", s, t, e.maxMatchOff) - panic(err) - } - if len(src)-int(s) > maxCompressedBlockSize { - panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) - } - } - return int32(matchLen(src[s:], src[t:])) -} - -// Reset the encoding table. -func (e *fastBase) resetBase(d *dict, singleBlock bool) { - if e.blk == nil { - e.blk = &blockEnc{lowMem: e.lowMem} - e.blk.init() - } else { - e.blk.reset(nil) - } - e.blk.initNewEncode() - if e.crc == nil { - e.crc = xxhash.New() - } else { - e.crc.Reset() - } - e.blk.dictLitEnc = nil - if d != nil { - low := e.lowMem - if singleBlock { - e.lowMem = true - } - e.ensureHist(d.ContentSize() + maxCompressedBlockSize) - e.lowMem = low - } - - // We offset current position so everything will be out of reach. - // If above reset line, history will be purged. - if e.cur < e.bufferReset { - e.cur += e.maxMatchOff + int32(len(e.hist)) - } - e.hist = e.hist[:0] - if d != nil { - // Set offsets (currently not used) - for i, off := range d.offsets { - e.blk.recentOffsets[i] = uint32(off) - e.blk.prevRecentOffsets[i] = e.blk.recentOffsets[i] - } - // Transfer litenc. - e.blk.dictLitEnc = d.litEnc - e.hist = append(e.hist, d.content...) - } -} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go deleted file mode 100644 index 4613724e9..000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ /dev/null @@ -1,560 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "bytes" - "fmt" - - "github.com/klauspost/compress" -) - -const ( - bestLongTableBits = 22 // Bits used in the long match table - bestLongTableSize = 1 << bestLongTableBits // Size of the table - bestLongLen = 8 // Bytes used for table hash - - // Note: Increasing the short table bits or making the hash shorter - // can actually lead to compression degradation since it will 'steal' more from the - // long match table and match offsets are quite big. - // This greatly depends on the type of input. - bestShortTableBits = 18 // Bits used in the short match table - bestShortTableSize = 1 << bestShortTableBits // Size of the table - bestShortLen = 4 // Bytes used for table hash - -) - -type match struct { - offset int32 - s int32 - length int32 - rep int32 - est int32 -} - -const highScore = maxMatchLen * 8 - -// estBits will estimate output bits from predefined tables. -func (m *match) estBits(bitsPerByte int32) { - mlc := mlCode(uint32(m.length - zstdMinMatch)) - var ofc uint8 - if m.rep < 0 { - ofc = ofCode(uint32(m.s-m.offset) + 3) - } else { - ofc = ofCode(uint32(m.rep) & 3) - } - // Cost, excluding - ofTT, mlTT := fsePredefEnc[tableOffsets].ct.symbolTT[ofc], fsePredefEnc[tableMatchLengths].ct.symbolTT[mlc] - - // Add cost of match encoding... - m.est = int32(ofTT.outBits + mlTT.outBits) - m.est += int32(ofTT.deltaNbBits>>16 + mlTT.deltaNbBits>>16) - // Subtract savings compared to literal encoding... - m.est -= (m.length * bitsPerByte) >> 10 - if m.est > 0 { - // Unlikely gain.. - m.length = 0 - m.est = highScore - } -} - -// bestFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. -// The long match table contains the previous entry with the same hash, -// effectively making it a "chain" of length 2. -// When we find a long match we choose between the two values and select the longest. -// When we find a short match, after checking the long, we check if we can find a long at n+1 -// and that it is longer (lazy matching). -type bestFastEncoder struct { - fastBase - table [bestShortTableSize]prevEntry - longTable [bestLongTableSize]prevEntry - dictTable []prevEntry - dictLongTable []prevEntry -} - -// Encode improves compression... -func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 4 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - e.table = [bestShortTableSize]prevEntry{} - e.longTable = [bestLongTableSize]prevEntry{} - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - v2 := e.table[i].prev - if v < minOff { - v = 0 - v2 = 0 - } else { - v = v - e.cur + e.maxMatchOff - if v2 < minOff { - v2 = 0 - } else { - v2 = v2 - e.cur + e.maxMatchOff - } - } - e.table[i] = prevEntry{ - offset: v, - prev: v2, - } - } - for i := range e.longTable[:] { - v := e.longTable[i].offset - v2 := e.longTable[i].prev - if v < minOff { - v = 0 - v2 = 0 - } else { - v = v - e.cur + e.maxMatchOff - if v2 < minOff { - v2 = 0 - } else { - v2 = v2 - e.cur + e.maxMatchOff - } - } - e.longTable[i] = prevEntry{ - offset: v, - prev: v2, - } - } - e.cur = e.maxMatchOff - break - } - - // Add block to history - s := e.addBlock(src) - blk.size = len(src) - - // Check RLE first - if len(src) > zstdMinMatch { - ml := matchLen(src[1:], src) - if ml == len(src)-1 { - blk.literals = append(blk.literals, src[0]) - blk.sequences = append(blk.sequences, seq{litLen: 1, matchLen: uint32(len(src)-1) - zstdMinMatch, offset: 1 + 3}) - return - } - } - - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Use this to estimate literal cost. - // Scaled by 10 bits. - bitsPerByte := int32((compress.ShannonEntropyBits(src) * 1024) / len(src)) - // Huffman can never go < 1 bit/byte - if bitsPerByte < 1024 { - bitsPerByte = 1024 - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - const kSearchStrength = 10 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - offset3 := int32(blk.recentOffsets[2]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - // We allow the encoder to optionally turn off repeat offsets across blocks - canRepeat := len(blk.sequences) > 2 - - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - const goodEnough = 250 - - cv := load6432(src, s) - - nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) - nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - // Set m to a match at offset if it looks like that will improve compression. - improve := func(m *match, offset int32, s int32, first uint32, rep int32) { - delta := s - offset - if delta >= e.maxMatchOff || delta <= 0 || load3232(src, offset) != first { - return - } - // Try to quick reject if we already have a long match. - if m.length > 16 { - left := len(src) - int(m.s+m.length) - // If we are too close to the end, keep as is. - if left <= 0 { - return - } - checkLen := m.length - (s - m.s) - 8 - if left > 2 && checkLen > 4 { - // Check 4 bytes, 4 bytes from the end of the current match. - a := load3232(src, offset+checkLen) - b := load3232(src, s+checkLen) - if a != b { - return - } - } - } - l := 4 + e.matchlen(s+4, offset+4, src) - if m.rep <= 0 { - // Extend candidate match backwards as far as possible. - // Do not extend repeats as we can assume they are optimal - // and offsets change if s == nextEmit. - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for offset > tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength { - s-- - offset-- - l++ - } - } - if debugAsserts { - if offset >= s { - panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff)) - } - if !bytes.Equal(src[s:s+l], src[offset:offset+l]) { - panic(fmt.Sprintf("second match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) - } - } - cand := match{offset: offset, s: s, length: l, rep: rep} - cand.estBits(bitsPerByte) - if m.est >= highScore || cand.est-m.est+(cand.s-m.s)*bitsPerByte>>10 < 0 { - *m = cand - } - } - - best := match{s: s, est: highScore} - improve(&best, candidateL.offset-e.cur, s, uint32(cv), -1) - improve(&best, candidateL.prev-e.cur, s, uint32(cv), -1) - improve(&best, candidateS.offset-e.cur, s, uint32(cv), -1) - improve(&best, candidateS.prev-e.cur, s, uint32(cv), -1) - - if canRepeat && best.length < goodEnough { - if s == nextEmit { - // Check repeats straight after a match. - improve(&best, s-offset2, s, uint32(cv), 1|4) - improve(&best, s-offset3, s, uint32(cv), 2|4) - if offset1 > 1 { - improve(&best, s-(offset1-1), s, uint32(cv), 3|4) - } - } - - // If either no match or a non-repeat match, check at + 1 - if best.rep <= 0 { - cv32 := uint32(cv >> 8) - spp := s + 1 - improve(&best, spp-offset1, spp, cv32, 1) - improve(&best, spp-offset2, spp, cv32, 2) - improve(&best, spp-offset3, spp, cv32, 3) - if best.rep < 0 { - cv32 = uint32(cv >> 24) - spp += 2 - improve(&best, spp-offset1, spp, cv32, 1) - improve(&best, spp-offset2, spp, cv32, 2) - improve(&best, spp-offset3, spp, cv32, 3) - } - } - } - // Load next and check... - e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: candidateL.offset} - e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: candidateS.offset} - index0 := s + 1 - - // Look far ahead, unless we have a really long match already... - if best.length < goodEnough { - // No match found, move forward on input, no need to check forward... - if best.length < 4 { - s += 1 + (s-nextEmit)>>(kSearchStrength-1) - if s >= sLimit { - break encodeLoop - } - continue - } - - candidateS = e.table[hashLen(cv>>8, bestShortTableBits, bestShortLen)] - cv = load6432(src, s+1) - cv2 := load6432(src, s+2) - candidateL = e.longTable[hashLen(cv, bestLongTableBits, bestLongLen)] - candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)] - - // Short at s+1 - improve(&best, candidateS.offset-e.cur, s+1, uint32(cv), -1) - // Long at s+1, s+2 - improve(&best, candidateL.offset-e.cur, s+1, uint32(cv), -1) - improve(&best, candidateL.prev-e.cur, s+1, uint32(cv), -1) - improve(&best, candidateL2.offset-e.cur, s+2, uint32(cv2), -1) - improve(&best, candidateL2.prev-e.cur, s+2, uint32(cv2), -1) - if false { - // Short at s+3. - // Too often worse... - improve(&best, e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+3, uint32(cv2>>8), -1) - } - - // Start check at a fixed offset to allow for a few mismatches. - // For this compression level 2 yields the best results. - // We cannot do this if we have already indexed this position. - const skipBeginning = 2 - if best.s > s-skipBeginning { - // See if we can find a better match by checking where the current best ends. - // Use that offset to see if we can find a better full match. - if sAt := best.s + best.length; sAt < sLimit { - nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) - candidateEnd := e.longTable[nextHashL] - - if off := candidateEnd.offset - e.cur - best.length + skipBeginning; off >= 0 { - improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) - if off := candidateEnd.prev - e.cur - best.length + skipBeginning; off >= 0 { - improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) - } - } - } - } - } - - if debugAsserts { - if best.offset >= best.s { - panic(fmt.Sprintf("best.offset > s: %d >= %d", best.offset, best.s)) - } - if best.s < nextEmit { - panic(fmt.Sprintf("s %d < nextEmit %d", best.s, nextEmit)) - } - if best.offset < s-e.maxMatchOff { - panic(fmt.Sprintf("best.offset < s-e.maxMatchOff: %d < %d", best.offset, s-e.maxMatchOff)) - } - if !bytes.Equal(src[best.s:best.s+best.length], src[best.offset:best.offset+best.length]) { - panic(fmt.Sprintf("match mismatch: %v != %v", src[best.s:best.s+best.length], src[best.offset:best.offset+best.length])) - } - } - - // We have a match, we can store the forward value - s = best.s - if best.rep > 0 { - var seq seq - seq.matchLen = uint32(best.length - zstdMinMatch) - addLiterals(&seq, best.s) - - // Repeat. If bit 4 is set, this is a non-lit repeat. - seq.offset = uint32(best.rep & 3) - if debugSequences { - println("repeat sequence", seq, "next s:", best.s, "off:", best.s-best.offset) - } - blk.sequences = append(blk.sequences, seq) - - // Index old s + 1 -> s - 1 - s = best.s + best.length - nextEmit = s - - // Index skipped... - end := s - if s > sLimit+4 { - end = sLimit + 4 - } - off := index0 + e.cur - for index0 < end { - cv0 := load6432(src, index0) - h0 := hashLen(cv0, bestLongTableBits, bestLongLen) - h1 := hashLen(cv0, bestShortTableBits, bestShortLen) - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} - off++ - index0++ - } - - switch best.rep { - case 2, 4 | 1: - offset1, offset2 = offset2, offset1 - case 3, 4 | 2: - offset1, offset2, offset3 = offset3, offset1, offset2 - case 4 | 3: - offset1, offset2, offset3 = offset1-1, offset1, offset2 - } - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, best.length) - } - break encodeLoop - } - continue - } - - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. - t := best.offset - offset1, offset2, offset3 = s-t, offset1, offset2 - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && int(offset1) > len(src) { - panic("invalid offset") - } - - // Write our sequence - var seq seq - l := best.length - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - - // Index old s + 1 -> s - 1 or sLimit - end := s - if s > sLimit-4 { - end = sLimit - 4 - } - - off := index0 + e.cur - for index0 < end { - cv0 := load6432(src, index0) - h0 := hashLen(cv0, bestLongTableBits, bestLongLen) - h1 := hashLen(cv0, bestShortTableBits, bestShortLen) - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} - index0++ - off++ - } - if s >= sLimit { - break encodeLoop - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - blk.recentOffsets[2] = uint32(offset3) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// EncodeNoHist will encode a block with no history and no following blocks. -// Most notable difference is that src will not be copied for history and -// we do not need to check for max match length. -func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { - e.ensureHist(len(src)) - e.Encode(blk, src) -} - -// Reset will reset and set a dictionary if not nil -func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { - e.resetBase(d, singleBlock) - if d == nil { - return - } - // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { - if len(e.dictTable) != len(e.table) { - e.dictTable = make([]prevEntry, len(e.table)) - } - end := int32(len(d.content)) - 8 + e.maxMatchOff - for i := e.maxMatchOff; i < end; i += 4 { - const hashLog = bestShortTableBits - - cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hashLen(cv, hashLog, bestShortLen) // 0 -> 4 - nextHash1 := hashLen(cv>>8, hashLog, bestShortLen) // 1 -> 5 - nextHash2 := hashLen(cv>>16, hashLog, bestShortLen) // 2 -> 6 - nextHash3 := hashLen(cv>>24, hashLog, bestShortLen) // 3 -> 7 - e.dictTable[nextHash] = prevEntry{ - prev: e.dictTable[nextHash].offset, - offset: i, - } - e.dictTable[nextHash1] = prevEntry{ - prev: e.dictTable[nextHash1].offset, - offset: i + 1, - } - e.dictTable[nextHash2] = prevEntry{ - prev: e.dictTable[nextHash2].offset, - offset: i + 2, - } - e.dictTable[nextHash3] = prevEntry{ - prev: e.dictTable[nextHash3].offset, - offset: i + 3, - } - } - e.lastDictID = d.id - } - - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { - if len(e.dictLongTable) != len(e.longTable) { - e.dictLongTable = make([]prevEntry, len(e.longTable)) - } - if len(d.content) >= 8 { - cv := load6432(d.content, 0) - h := hashLen(cv, bestLongTableBits, bestLongLen) - e.dictLongTable[h] = prevEntry{ - offset: e.maxMatchOff, - prev: e.dictLongTable[h].offset, - } - - end := int32(len(d.content)) - 8 + e.maxMatchOff - off := 8 // First to read - for i := e.maxMatchOff + 1; i < end; i++ { - cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hashLen(cv, bestLongTableBits, bestLongLen) - e.dictLongTable[h] = prevEntry{ - offset: i, - prev: e.dictLongTable[h].offset, - } - off++ - } - } - e.lastDictID = d.id - } - // Reset table to initial state - copy(e.longTable[:], e.dictLongTable) - - e.cur = e.maxMatchOff - // Reset table to initial state - copy(e.table[:], e.dictTable) -} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go deleted file mode 100644 index 84a79fde7..000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ /dev/null @@ -1,1252 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import "fmt" - -const ( - betterLongTableBits = 19 // Bits used in the long match table - betterLongTableSize = 1 << betterLongTableBits // Size of the table - betterLongLen = 8 // Bytes used for table hash - - // Note: Increasing the short table bits or making the hash shorter - // can actually lead to compression degradation since it will 'steal' more from the - // long match table and match offsets are quite big. - // This greatly depends on the type of input. - betterShortTableBits = 13 // Bits used in the short match table - betterShortTableSize = 1 << betterShortTableBits // Size of the table - betterShortLen = 5 // Bytes used for table hash - - betterLongTableShardCnt = 1 << (betterLongTableBits - dictShardBits) // Number of shards in the table - betterLongTableShardSize = betterLongTableSize / betterLongTableShardCnt // Size of an individual shard - - betterShortTableShardCnt = 1 << (betterShortTableBits - dictShardBits) // Number of shards in the table - betterShortTableShardSize = betterShortTableSize / betterShortTableShardCnt // Size of an individual shard -) - -type prevEntry struct { - offset int32 - prev int32 -} - -// betterFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. -// The long match table contains the previous entry with the same hash, -// effectively making it a "chain" of length 2. -// When we find a long match we choose between the two values and select the longest. -// When we find a short match, after checking the long, we check if we can find a long at n+1 -// and that it is longer (lazy matching). -type betterFastEncoder struct { - fastBase - table [betterShortTableSize]tableEntry - longTable [betterLongTableSize]prevEntry -} - -type betterFastEncoderDict struct { - betterFastEncoder - dictTable []tableEntry - dictLongTable []prevEntry - shortTableShardDirty [betterShortTableShardCnt]bool - longTableShardDirty [betterLongTableShardCnt]bool - allDirty bool -} - -// Encode improves compression... -func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 2 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - e.table = [betterShortTableSize]tableEntry{} - e.longTable = [betterLongTableSize]prevEntry{} - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - for i := range e.longTable[:] { - v := e.longTable[i].offset - v2 := e.longTable[i].prev - if v < minOff { - v = 0 - v2 = 0 - } else { - v = v - e.cur + e.maxMatchOff - if v2 < minOff { - v2 = 0 - } else { - v2 = v2 - e.cur + e.maxMatchOff - } - } - e.longTable[i] = prevEntry{ - offset: v, - prev: v2, - } - } - e.cur = e.maxMatchOff - break - } - // Add block to history - s := e.addBlock(src) - blk.size = len(src) - - // Check RLE first - if len(src) > zstdMinMatch { - ml := matchLen(src[1:], src) - if ml == len(src)-1 { - blk.literals = append(blk.literals, src[0]) - blk.sequences = append(blk.sequences, seq{litLen: 1, matchLen: uint32(len(src)-1) - zstdMinMatch, offset: 1 + 3}) - return - } - } - - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 1. - const stepSize = 1 - - const kSearchStrength = 9 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - var t int32 - // We allow the encoder to optionally turn off repeat offsets across blocks - canRepeat := len(blk.sequences) > 2 - var matched, index0 int32 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - const repOff = 1 - repIndex := s - offset1 + repOff - off := s + e.cur - e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset} - e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)} - index0 = s + 1 - - if canRepeat { - if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Index match start+1 (long) -> s - 1 - index0 := s + repOff - s += length + repOff - - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - // Index skipped... - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 - } - cv = load6432(src, s) - continue - } - const repOff2 = 1 - - // We deviate from the reference encoder and also check offset 2. - // Still slower and not much better, so disabled. - // repIndex = s - offset2 + repOff2 - if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { - // Consider history as well. - var seq seq - length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 2 - seq.offset = 2 - if debugSequences { - println("repeat sequence 2", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - s += length + repOff2 - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - - // Index skipped... - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 - } - cv = load6432(src, s) - // Swap offsets - offset1, offset2 = offset2, offset1 - continue - } - } - // Find the offsets of our two matches. - coffsetL := candidateL.offset - e.cur - coffsetLP := candidateL.prev - e.cur - - // Check if we have a long match. - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matched = e.matchlen(s+8, coffsetL+8, src) + 8 - t = coffsetL - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - - if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { - // Found a long match, at least 8 bytes. - prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8 - if prevMatch > matched { - matched = prevMatch - t = coffsetLP - } - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - } - break - } - - // Check if we have a long match on prev. - if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { - // Found a long match, at least 8 bytes. - matched = e.matchlen(s+8, coffsetLP+8, src) + 8 - t = coffsetLP - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - break - } - - coffsetS := candidateS.offset - e.cur - - // Check if we have a short match. - if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { - // found a regular match - matched = e.matchlen(s+4, coffsetS+4, src) + 4 - - // See if we can find a long match at s+1 - const checkAt = 1 - cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) - candidateL = e.longTable[nextHashL] - coffsetL = candidateL.offset - e.cur - - // We can store it, since we have at least a 4 byte match. - e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset} - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 - if matchedNext > matched { - t = coffsetL - s += checkAt - matched = matchedNext - if debugMatches { - println("long match (after short)") - } - break - } - } - - // Check prev long... - coffsetL = candidateL.prev - e.cur - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 - if matchedNext > matched { - t = coffsetL - s += checkAt - matched = matchedNext - if debugMatches { - println("prev long match (after short)") - } - break - } - } - t = coffsetS - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - if debugMatches { - println("short match") - } - break - } - - // No match found, move forward in input. - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - - // Try to find a better match by searching for a long match at the end of the current best match - if s+matched < sLimit { - // Allow some bytes at the beginning to mismatch. - // Sweet spot is around 3 bytes, but depends on input. - // The skipped bytes are tested in Extend backwards, - // and still picked up as part of the match if they do. - const skipBeginning = 3 - - nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) - s2 := s + skipBeginning - cv := load3232(src, s2) - candidateL := e.longTable[nextHashL] - coffsetL := candidateL.offset - e.cur - matched + skipBeginning - if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { - // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 - if matchedNext > matched { - t = coffsetL - s = s2 - matched = matchedNext - if debugMatches { - println("long match at end-of-match") - } - } - } - - // Check prev long... - if true { - coffsetL = candidateL.prev - e.cur - matched + skipBeginning - if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { - // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 - if matchedNext > matched { - t = coffsetL - s = s2 - matched = matchedNext - if debugMatches { - println("prev long match at end-of-match") - } - } - } - } - } - // A match has been found. Update recent offsets. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the n-byte match as long as possible. - l := matched - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - - // Index match start+1 (long) -> s - 1 - off := index0 + e.cur - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 - off += 2 - } - - cv = load6432(src, s) - if !canRepeat { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} - e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)} - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// EncodeNoHist will encode a block with no history and no following blocks. -// Most notable difference is that src will not be copied for history and -// we do not need to check for max match length. -func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { - e.ensureHist(len(src)) - e.Encode(blk, src) -} - -// Encode improves compression... -func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 2 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = prevEntry{} - } - e.cur = e.maxMatchOff - e.allDirty = true - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - for i := range e.longTable[:] { - v := e.longTable[i].offset - v2 := e.longTable[i].prev - if v < minOff { - v = 0 - v2 = 0 - } else { - v = v - e.cur + e.maxMatchOff - if v2 < minOff { - v2 = 0 - } else { - v2 = v2 - e.cur + e.maxMatchOff - } - } - e.longTable[i] = prevEntry{ - offset: v, - prev: v2, - } - } - e.allDirty = true - e.cur = e.maxMatchOff - break - } - - s := e.addBlock(src) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 1. - const stepSize = 1 - - const kSearchStrength = 9 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - var t int32 - // We allow the encoder to optionally turn off repeat offsets across blocks - canRepeat := len(blk.sequences) > 2 - var matched, index0 int32 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - const repOff = 1 - repIndex := s - offset1 + repOff - off := s + e.cur - e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset} - e.markLongShardDirty(nextHashL) - e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)} - e.markShortShardDirty(nextHashS) - index0 = s + 1 - - if canRepeat { - if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Index match start+1 (long) -> s - 1 - s += length + repOff - - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - // Index skipped... - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) - e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} - e.markShortShardDirty(h1) - index0 += 2 - } - cv = load6432(src, s) - continue - } - const repOff2 = 1 - - // We deviate from the reference encoder and also check offset 2. - // Still slower and not much better, so disabled. - // repIndex = s - offset2 + repOff2 - if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { - // Consider history as well. - var seq seq - length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 2 - seq.offset = 2 - if debugSequences { - println("repeat sequence 2", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - s += length + repOff2 - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - - // Index skipped... - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) - e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} - e.markShortShardDirty(h1) - index0 += 2 - } - cv = load6432(src, s) - // Swap offsets - offset1, offset2 = offset2, offset1 - continue - } - } - // Find the offsets of our two matches. - coffsetL := candidateL.offset - e.cur - coffsetLP := candidateL.prev - e.cur - - // Check if we have a long match. - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matched = e.matchlen(s+8, coffsetL+8, src) + 8 - t = coffsetL - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - - if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { - // Found a long match, at least 8 bytes. - prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8 - if prevMatch > matched { - matched = prevMatch - t = coffsetLP - } - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - } - break - } - - // Check if we have a long match on prev. - if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { - // Found a long match, at least 8 bytes. - matched = e.matchlen(s+8, coffsetLP+8, src) + 8 - t = coffsetLP - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - break - } - - coffsetS := candidateS.offset - e.cur - - // Check if we have a short match. - if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { - // found a regular match - matched = e.matchlen(s+4, coffsetS+4, src) + 4 - - // See if we can find a long match at s+1 - const checkAt = 1 - cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) - candidateL = e.longTable[nextHashL] - coffsetL = candidateL.offset - e.cur - - // We can store it, since we have at least a 4 byte match. - e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset} - e.markLongShardDirty(nextHashL) - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 - if matchedNext > matched { - t = coffsetL - s += checkAt - matched = matchedNext - if debugMatches { - println("long match (after short)") - } - break - } - } - - // Check prev long... - coffsetL = candidateL.prev - e.cur - if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { - // Found a long match, at least 8 bytes. - matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 - if matchedNext > matched { - t = coffsetL - s += checkAt - matched = matchedNext - if debugMatches { - println("prev long match (after short)") - } - break - } - } - t = coffsetS - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - if debugMatches { - println("short match") - } - break - } - - // No match found, move forward in input. - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - // Try to find a better match by searching for a long match at the end of the current best match - if s+matched < sLimit { - nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) - cv := load3232(src, s) - candidateL := e.longTable[nextHashL] - coffsetL := candidateL.offset - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { - // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 - if matchedNext > matched { - t = coffsetL - matched = matchedNext - if debugMatches { - println("long match at end-of-match") - } - } - } - - // Check prev long... - if true { - coffsetL = candidateL.prev - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { - // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 - if matchedNext > matched { - t = coffsetL - matched = matchedNext - if debugMatches { - println("prev long match at end-of-match") - } - } - } - } - } - // A match has been found. Update recent offsets. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the n-byte match as long as possible. - l := matched - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - - // Index match start+1 (long) -> s - 1 - off := index0 + e.cur - for index0 < s-1 { - cv0 := load6432(src, index0) - cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) - e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} - e.markShortShardDirty(h1) - index0 += 2 - off += 2 - } - - cv = load6432(src, s) - if !canRepeat { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} - e.markLongShardDirty(nextHashL) - e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)} - e.markShortShardDirty(nextHashS) - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { - e.resetBase(d, singleBlock) - if d != nil { - panic("betterFastEncoder: Reset with dict") - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { - e.resetBase(d, singleBlock) - if d == nil { - return - } - // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { - if len(e.dictTable) != len(e.table) { - e.dictTable = make([]tableEntry, len(e.table)) - } - end := int32(len(d.content)) - 8 + e.maxMatchOff - for i := e.maxMatchOff; i < end; i += 4 { - const hashLog = betterShortTableBits - - cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hashLen(cv, hashLog, betterShortLen) // 0 -> 4 - nextHash1 := hashLen(cv>>8, hashLog, betterShortLen) // 1 -> 5 - nextHash2 := hashLen(cv>>16, hashLog, betterShortLen) // 2 -> 6 - nextHash3 := hashLen(cv>>24, hashLog, betterShortLen) // 3 -> 7 - e.dictTable[nextHash] = tableEntry{ - val: uint32(cv), - offset: i, - } - e.dictTable[nextHash1] = tableEntry{ - val: uint32(cv >> 8), - offset: i + 1, - } - e.dictTable[nextHash2] = tableEntry{ - val: uint32(cv >> 16), - offset: i + 2, - } - e.dictTable[nextHash3] = tableEntry{ - val: uint32(cv >> 24), - offset: i + 3, - } - } - e.lastDictID = d.id - e.allDirty = true - } - - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { - if len(e.dictLongTable) != len(e.longTable) { - e.dictLongTable = make([]prevEntry, len(e.longTable)) - } - if len(d.content) >= 8 { - cv := load6432(d.content, 0) - h := hashLen(cv, betterLongTableBits, betterLongLen) - e.dictLongTable[h] = prevEntry{ - offset: e.maxMatchOff, - prev: e.dictLongTable[h].offset, - } - - end := int32(len(d.content)) - 8 + e.maxMatchOff - off := 8 // First to read - for i := e.maxMatchOff + 1; i < end; i++ { - cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hashLen(cv, betterLongTableBits, betterLongLen) - e.dictLongTable[h] = prevEntry{ - offset: i, - prev: e.dictLongTable[h].offset, - } - off++ - } - } - e.lastDictID = d.id - e.allDirty = true - } - - // Reset table to initial state - { - dirtyShardCnt := 0 - if !e.allDirty { - for i := range e.shortTableShardDirty { - if e.shortTableShardDirty[i] { - dirtyShardCnt++ - } - } - } - const shardCnt = betterShortTableShardCnt - const shardSize = betterShortTableShardSize - if e.allDirty || dirtyShardCnt > shardCnt*4/6 { - copy(e.table[:], e.dictTable) - for i := range e.shortTableShardDirty { - e.shortTableShardDirty[i] = false - } - } else { - for i := range e.shortTableShardDirty { - if !e.shortTableShardDirty[i] { - continue - } - - copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) - e.shortTableShardDirty[i] = false - } - } - } - { - dirtyShardCnt := 0 - if !e.allDirty { - for i := range e.shortTableShardDirty { - if e.shortTableShardDirty[i] { - dirtyShardCnt++ - } - } - } - const shardCnt = betterLongTableShardCnt - const shardSize = betterLongTableShardSize - if e.allDirty || dirtyShardCnt > shardCnt*4/6 { - copy(e.longTable[:], e.dictLongTable) - for i := range e.longTableShardDirty { - e.longTableShardDirty[i] = false - } - } else { - for i := range e.longTableShardDirty { - if !e.longTableShardDirty[i] { - continue - } - - copy(e.longTable[i*shardSize:(i+1)*shardSize], e.dictLongTable[i*shardSize:(i+1)*shardSize]) - e.longTableShardDirty[i] = false - } - } - } - e.cur = e.maxMatchOff - e.allDirty = false -} - -func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) { - e.longTableShardDirty[entryNum/betterLongTableShardSize] = true -} - -func (e *betterFastEncoderDict) markShortShardDirty(entryNum uint32) { - e.shortTableShardDirty[entryNum/betterShortTableShardSize] = true -} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go deleted file mode 100644 index d36be7bd8..000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ /dev/null @@ -1,1123 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import "fmt" - -const ( - dFastLongTableBits = 17 // Bits used in the long match table - dFastLongTableSize = 1 << dFastLongTableBits // Size of the table - dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - dFastLongLen = 8 // Bytes used for table hash - - dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table - dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard - - dFastShortTableBits = tableBits // Bits used in the short match table - dFastShortTableSize = 1 << dFastShortTableBits // Size of the table - dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - dFastShortLen = 5 // Bytes used for table hash - -) - -type doubleFastEncoder struct { - fastEncoder - longTable [dFastLongTableSize]tableEntry -} - -type doubleFastEncoderDict struct { - fastEncoderDict - longTable [dFastLongTableSize]tableEntry - dictLongTable []tableEntry - longTableShardDirty [dLongTableShardCnt]bool -} - -// Encode mimmics functionality in zstd_dfast.c -func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 2 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - e.table = [dFastShortTableSize]tableEntry{} - e.longTable = [dFastLongTableSize]tableEntry{} - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - for i := range e.longTable[:] { - v := e.longTable[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.longTable[i].offset = v - } - e.cur = e.maxMatchOff - break - } - - s := e.addBlock(src) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 1. - const stepSize = 1 - - const kSearchStrength = 8 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - var t int32 - // We allow the encoder to optionally turn off repeat offsets across blocks - canRepeat := len(blk.sequences) > 2 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - const repOff = 1 - repIndex := s - offset1 + repOff - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.table[nextHashS] = entry - - if canRepeat { - if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + repOff - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - } - // Find the offsets of our two matches. - coffsetL := s - (candidateL.offset - e.cur) - coffsetS := s - (candidateS.offset - e.cur) - - // Check if we have a long match. - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - break - } - - // Check if we have a short match. - if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { - // found a regular match - // See if we can find a long match at s+1 - const checkAt = 1 - cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) - candidateL = e.longTable[nextHashL] - coffsetL = s - (candidateL.offset - e.cur) + checkAt - - // We can store it, since we have at least a 4 byte match. - e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - s += checkAt - if debugMatches { - println("long match (after short)") - } - break - } - - t = candidateS.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - if debugMatches { - println("short match") - } - break - } - - // No match found, move forward in input. - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - - // Index match start+1 (long) and start+2 (short) - index0 := s - l + 1 - // Index match end-2 (long) and end-1 (short) - index1 := s - 2 - - cv0 := load6432(src, index0) - cv1 := load6432(src, index1) - te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} - te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 - e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 - cv0 >>= 8 - cv1 >>= 8 - te0.offset++ - te1.offset++ - te0.val = uint32(cv0) - te1.val = uint32(cv1) - e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 - e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 - - cv = load6432(src, s) - - if !canRepeat { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.table[nextHashS] = entry - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// EncodeNoHist will encode a block with no history and no following blocks. -// Most notable difference is that src will not be copied for history and -// we do not need to check for max match length. -func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 2 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - if e.cur >= e.bufferReset { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = tableEntry{} - } - e.cur = e.maxMatchOff - } - - s := int32(0) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 1. - const stepSize = 1 - - const kSearchStrength = 8 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - var t int32 - for { - - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - const repOff = 1 - repIndex := s - offset1 + repOff - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.table[nextHashS] = entry - - if len(blk.sequences) > 2 { - if load3232(src, repIndex) == uint32(cv>>(repOff*8)) { - // Consider history as well. - var seq seq - //length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - length := 4 + int32(matchLen(src[s+4+repOff:], src[repIndex+4:])) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + repOff - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - } - // Find the offsets of our two matches. - coffsetL := s - (candidateL.offset - e.cur) - coffsetS := s - (candidateS.offset - e.cur) - - // Check if we have a long match. - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d). cur: %d", s, t, e.cur)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - break - } - - // Check if we have a short match. - if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { - // found a regular match - // See if we can find a long match at s+1 - const checkAt = 1 - cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) - candidateL = e.longTable[nextHashL] - coffsetL = s - (candidateL.offset - e.cur) + checkAt - - // We can store it, since we have at least a 4 byte match. - e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - s += checkAt - if debugMatches { - println("long match (after short)") - } - break - } - - t = candidateS.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - if debugMatches { - println("short match") - } - break - } - - // No match found, move forward in input. - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - // Extend the 4-byte match as long as possible. - //l := e.matchlen(s+4, t+4, src) + 4 - l := int32(matchLen(src[s+4:], src[t+4:])) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - - // Write our sequence - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - - // Index match start+1 (long) and start+2 (short) - index0 := s - l + 1 - // Index match end-2 (long) and end-1 (short) - index1 := s - 2 - - cv0 := load6432(src, index0) - cv1 := load6432(src, index1) - te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} - te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 - e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 - cv0 >>= 8 - cv1 >>= 8 - te0.offset++ - te1.offset++ - te0.val = uint32(cv0) - te1.val = uint32(cv1) - e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 - e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 - - cv = load6432(src, s) - - if len(blk.sequences) <= 2 { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashS := hashLen(cv1>>8, dFastShortTableBits, dFastShortLen) - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - //l := 4 + e.matchlen(s+4, o2+4, src) - l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) - - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.table[nextHashS] = entry - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } - - // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < e.bufferReset { - e.cur += int32(len(src)) - } -} - -// Encode will encode the content, with a dictionary if initialized for it. -func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { - const ( - // Input margin is the number of bytes we read (8) - // and the maximum we will read ahead (2) - inputMargin = 8 + 2 - minNonLiteralBlockSize = 16 - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = tableEntry{} - } - e.markAllShardsDirty() - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - for i := range e.longTable[:] { - v := e.longTable[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.longTable[i].offset = v - } - e.markAllShardsDirty() - e.cur = e.maxMatchOff - break - } - - s := e.addBlock(src) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 1. - const stepSize = 1 - - const kSearchStrength = 8 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - var t int32 - // We allow the encoder to optionally turn off repeat offsets across blocks - canRepeat := len(blk.sequences) > 2 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - candidateL := e.longTable[nextHashL] - candidateS := e.table[nextHashS] - - const repOff = 1 - repIndex := s - offset1 + repOff - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.markLongShardDirty(nextHashL) - e.table[nextHashS] = entry - e.markShardDirty(nextHashS) - - if canRepeat { - if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + repOff - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - } - // Find the offsets of our two matches. - coffsetL := s - (candidateL.offset - e.cur) - coffsetS := s - (candidateS.offset - e.cur) - - // Check if we have a long match. - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugMatches { - println("long match") - } - break - } - - // Check if we have a short match. - if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { - // found a regular match - // See if we can find a long match at s+1 - const checkAt = 1 - cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) - candidateL = e.longTable[nextHashL] - coffsetL = s - (candidateL.offset - e.cur) + checkAt - - // We can store it, since we have at least a 4 byte match. - e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} - e.markLongShardDirty(nextHashL) - if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { - // Found a long match, likely at least 8 bytes. - // Reference encoder checks all 8 bytes, we only check 4, - // but the likelihood of both the first 4 bytes and the hash matching should be enough. - t = candidateL.offset - e.cur - s += checkAt - if debugMatches { - println("long match (after short)") - } - break - } - - t = candidateS.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - if debugMatches { - println("short match") - } - break - } - - // No match found, move forward in input. - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - - // Index match start+1 (long) and start+2 (short) - index0 := s - l + 1 - // Index match end-2 (long) and end-1 (short) - index1 := s - 2 - - cv0 := load6432(src, index0) - cv1 := load6432(src, index1) - te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} - te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) - longHash2 := hashLen(cv1, dFastLongTableBits, dFastLongLen) - e.longTable[longHash1] = te0 - e.longTable[longHash2] = te1 - e.markLongShardDirty(longHash1) - e.markLongShardDirty(longHash2) - cv0 >>= 8 - cv1 >>= 8 - te0.offset++ - te1.offset++ - te0.val = uint32(cv0) - te1.val = uint32(cv1) - hashVal1 := hashLen(cv0, dFastShortTableBits, dFastShortLen) - hashVal2 := hashLen(cv1, dFastShortTableBits, dFastShortLen) - e.table[hashVal1] = te0 - e.markShardDirty(hashVal1) - e.table[hashVal2] = te1 - e.markShardDirty(hashVal2) - - cv = load6432(src, s) - - if !canRepeat { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - entry := tableEntry{offset: s + e.cur, val: uint32(cv)} - e.longTable[nextHashL] = entry - e.markLongShardDirty(nextHashL) - e.table[nextHashS] = entry - e.markShardDirty(nextHashS) - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } - // If we encoded more than 64K mark all dirty. - if len(src) > 64<<10 { - e.markAllShardsDirty() - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { - e.fastEncoder.Reset(d, singleBlock) - if d != nil { - panic("doubleFastEncoder: Reset with dict not supported") - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { - allDirty := e.allDirty - e.fastEncoderDict.Reset(d, singleBlock) - if d == nil { - return - } - - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { - if len(e.dictLongTable) != len(e.longTable) { - e.dictLongTable = make([]tableEntry, len(e.longTable)) - } - if len(d.content) >= 8 { - cv := load6432(d.content, 0) - e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ - val: uint32(cv), - offset: e.maxMatchOff, - } - end := int32(len(d.content)) - 8 + e.maxMatchOff - for i := e.maxMatchOff + 1; i < end; i++ { - cv = cv>>8 | (uint64(d.content[i-e.maxMatchOff+7]) << 56) - e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ - val: uint32(cv), - offset: i, - } - } - } - e.lastDictID = d.id - allDirty = true - } - // Reset table to initial state - e.cur = e.maxMatchOff - - dirtyShardCnt := 0 - if !allDirty { - for i := range e.longTableShardDirty { - if e.longTableShardDirty[i] { - dirtyShardCnt++ - } - } - } - - if allDirty || dirtyShardCnt > dLongTableShardCnt/2 { - //copy(e.longTable[:], e.dictLongTable) - e.longTable = *(*[dFastLongTableSize]tableEntry)(e.dictLongTable) - for i := range e.longTableShardDirty { - e.longTableShardDirty[i] = false - } - return - } - for i := range e.longTableShardDirty { - if !e.longTableShardDirty[i] { - continue - } - - // copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) - *(*[dLongTableShardSize]tableEntry)(e.longTable[i*dLongTableShardSize:]) = *(*[dLongTableShardSize]tableEntry)(e.dictLongTable[i*dLongTableShardSize:]) - - e.longTableShardDirty[i] = false - } -} - -func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) { - e.longTableShardDirty[entryNum/dLongTableShardSize] = true -} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go deleted file mode 100644 index f45a3da7d..000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ /dev/null @@ -1,891 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "fmt" -) - -const ( - tableBits = 15 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table - tableShardSize = tableSize / tableShardCnt // Size of an individual shard - tableFastHashLen = 6 - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - maxMatchLength = 131074 -) - -type tableEntry struct { - val uint32 - offset int32 -} - -type fastEncoder struct { - fastBase - table [tableSize]tableEntry -} - -type fastEncoderDict struct { - fastEncoder - dictTable []tableEntry - tableShardDirty [tableShardCnt]bool - allDirty bool -} - -// Encode mimmics functionality in zstd_fast.c -func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { - const ( - inputMargin = 8 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - e.cur = e.maxMatchOff - break - } - - s := e.addBlock(src) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 2. - const stepSize = 2 - - // TEMPLATE - const hashLog = tableBits - // seems global, but would be nice to tweak. - const kSearchStrength = 6 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - // t will contain the match offset when we find one. - // When existing the search loop, we have already checked 4 bytes. - var t int32 - - // We will not use repeat offsets across blocks. - // By not using them for the first 3 matches - canRepeat := len(blk.sequences) > 2 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHash := hashLen(cv, hashLog, tableFastHashLen) - nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) - candidate := e.table[nextHash] - candidate2 := e.table[nextHash2] - repIndex := s - offset1 + 2 - - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} - - if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+6, repIndex+4, src) - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + 2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } - for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + 2 - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - coffset0 := s - (candidate.offset - e.cur) - coffset1 := s - (candidate2.offset - e.cur) + 1 - if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { - // found a regular match - t = candidate.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - break - } - - if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { - // found a regular match - t = candidate2.offset - e.cur - s++ - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - break - } - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - // A 4-byte match has been found. We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence. - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - // Don't use repeat offsets - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - - // Check offset 2 - if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - // Store this, since we have it. - nextHash := hashLen(cv, hashLog, tableFastHashLen) - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - break encodeLoop - } - // Prepare next loop. - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// EncodeNoHist will encode a block with no history and no following blocks. -// Most notable difference is that src will not be copied for history and -// we do not need to check for max match length. -func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { - const ( - inputMargin = 8 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - if debugEncoder { - if len(src) > maxCompressedBlockSize { - panic("src too big") - } - } - - // Protect against e.cur wraparound. - if e.cur >= e.bufferReset { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = e.maxMatchOff - } - - s := int32(0) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 2. - const stepSize = 2 - - // TEMPLATE - const hashLog = tableBits - // seems global, but would be nice to tweak. - const kSearchStrength = 6 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - // t will contain the match offset when we find one. - // When existing the search loop, we have already checked 4 bytes. - var t int32 - - // We will not use repeat offsets across blocks. - // By not using them for the first 3 matches - - for { - nextHash := hashLen(cv, hashLog, tableFastHashLen) - nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) - candidate := e.table[nextHash] - candidate2 := e.table[nextHash2] - repIndex := s - offset1 + 2 - - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} - - if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+6, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + 2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } - for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + 2 - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - coffset0 := s - (candidate.offset - e.cur) - coffset1 := s - (candidate2.offset - e.cur) + 1 - if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { - // found a regular match - t = candidate.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic(fmt.Sprintf("t (%d) < 0, candidate.offset: %d, e.cur: %d, coffset0: %d, e.maxMatchOff: %d", t, candidate.offset, e.cur, coffset0, e.maxMatchOff)) - } - break - } - - if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { - // found a regular match - t = candidate2.offset - e.cur - s++ - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - break - } - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - // A 4-byte match has been found. We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && t < 0 { - panic(fmt.Sprintf("t (%d) < 0 ", t)) - } - // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - - // Write our sequence. - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - // Don't use repeat offsets - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - - // Check offset 2 - if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) { - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - // Store this, since we have it. - nextHash := hashLen(cv, hashLog, tableFastHashLen) - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - break encodeLoop - } - // Prepare next loop. - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } - // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < e.bufferReset { - e.cur += int32(len(src)) - } -} - -// Encode will encode the content, with a dictionary if initialized for it. -func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) { - const ( - inputMargin = 8 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - if e.allDirty || len(src) > 32<<10 { - e.fastEncoder.Encode(blk, src) - e.allDirty = true - return - } - // Protect against e.cur wraparound. - for e.cur >= e.bufferReset-int32(len(e.hist)) { - if len(e.hist) == 0 { - e.table = [tableSize]tableEntry{} - e.cur = e.maxMatchOff - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff - for i := range e.table[:] { - v := e.table[i].offset - if v < minOff { - v = 0 - } else { - v = v - e.cur + e.maxMatchOff - } - e.table[i].offset = v - } - e.cur = e.maxMatchOff - break - } - - s := e.addBlock(src) - blk.size = len(src) - if len(src) < minNonLiteralBlockSize { - blk.extraLits = len(src) - blk.literals = blk.literals[:len(src)] - copy(blk.literals, src) - return - } - - // Override src - src = e.hist - sLimit := int32(len(src)) - inputMargin - // stepSize is the number of bytes to skip on every main loop iteration. - // It should be >= 2. - const stepSize = 2 - - // TEMPLATE - const hashLog = tableBits - // seems global, but would be nice to tweak. - const kSearchStrength = 7 - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := s - cv := load6432(src, s) - - // Relative offsets - offset1 := int32(blk.recentOffsets[0]) - offset2 := int32(blk.recentOffsets[1]) - - addLiterals := func(s *seq, until int32) { - if until == nextEmit { - return - } - blk.literals = append(blk.literals, src[nextEmit:until]...) - s.litLen = uint32(until - nextEmit) - } - if debugEncoder { - println("recent offsets:", blk.recentOffsets) - } - -encodeLoop: - for { - // t will contain the match offset when we find one. - // When existing the search loop, we have already checked 4 bytes. - var t int32 - - // We will not use repeat offsets across blocks. - // By not using them for the first 3 matches - canRepeat := len(blk.sequences) > 2 - - for { - if debugAsserts && canRepeat && offset1 == 0 { - panic("offset0 was 0") - } - - nextHash := hashLen(cv, hashLog, tableFastHashLen) - nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) - candidate := e.table[nextHash] - candidate2 := e.table[nextHash2] - repIndex := s - offset1 + 2 - - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - e.markShardDirty(nextHash) - e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} - e.markShardDirty(nextHash2) - - if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) { - // Consider history as well. - var seq seq - length := 4 + e.matchlen(s+6, repIndex+4, src) - - seq.matchLen = uint32(length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + 2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } - for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 0 - seq.offset = 1 - if debugSequences { - println("repeat sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += length + 2 - nextEmit = s - if s >= sLimit { - if debugEncoder { - println("repeat ended", s, length) - - } - break encodeLoop - } - cv = load6432(src, s) - continue - } - coffset0 := s - (candidate.offset - e.cur) - coffset1 := s - (candidate2.offset - e.cur) + 1 - if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { - // found a regular match - t = candidate.offset - e.cur - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - break - } - - if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { - // found a regular match - t = candidate2.offset - e.cur - s++ - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - if debugAsserts && s-t > e.maxMatchOff { - panic("s - t >e.maxMatchOff") - } - if debugAsserts && t < 0 { - panic("t<0") - } - break - } - s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - } - // A 4-byte match has been found. We'll later see if more than 4 bytes. - offset2 = offset1 - offset1 = s - t - - if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) - } - - if debugAsserts && canRepeat && int(offset1) > len(src) { - panic("invalid offset") - } - - // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - - // Write our sequence. - var seq seq - seq.litLen = uint32(s - nextEmit) - seq.matchLen = uint32(l - zstdMinMatch) - if seq.litLen > 0 { - blk.literals = append(blk.literals, src[nextEmit:s]...) - } - // Don't use repeat offsets - seq.offset = uint32(s-t) + 3 - s += l - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - nextEmit = s - if s >= sLimit { - break encodeLoop - } - cv = load6432(src, s) - - // Check offset 2 - if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - // Store this, since we have it. - nextHash := hashLen(cv, hashLog, tableFastHashLen) - e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} - e.markShardDirty(nextHash) - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - break encodeLoop - } - // Prepare next loop. - cv = load6432(src, s) - } - } - - if int(nextEmit) < len(src) { - blk.literals = append(blk.literals, src[nextEmit:]...) - blk.extraLits = len(src) - int(nextEmit) - } - blk.recentOffsets[0] = uint32(offset1) - blk.recentOffsets[1] = uint32(offset2) - if debugEncoder { - println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *fastEncoder) Reset(d *dict, singleBlock bool) { - e.resetBase(d, singleBlock) - if d != nil { - panic("fastEncoder: Reset with dict") - } -} - -// ResetDict will reset and set a dictionary if not nil -func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { - e.resetBase(d, singleBlock) - if d == nil { - return - } - - // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { - if len(e.dictTable) != len(e.table) { - e.dictTable = make([]tableEntry, len(e.table)) - } - if true { - end := e.maxMatchOff + int32(len(d.content)) - 8 - for i := e.maxMatchOff; i < end; i += 2 { - const hashLog = tableBits - - cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hashLen(cv, hashLog, tableFastHashLen) // 0 -> 6 - nextHash1 := hashLen(cv>>8, hashLog, tableFastHashLen) // 1 -> 7 - e.dictTable[nextHash] = tableEntry{ - val: uint32(cv), - offset: i, - } - e.dictTable[nextHash1] = tableEntry{ - val: uint32(cv >> 8), - offset: i + 1, - } - } - } - e.lastDictID = d.id - e.allDirty = true - } - - e.cur = e.maxMatchOff - dirtyShardCnt := 0 - if !e.allDirty { - for i := range e.tableShardDirty { - if e.tableShardDirty[i] { - dirtyShardCnt++ - } - } - } - - const shardCnt = tableShardCnt - const shardSize = tableShardSize - if e.allDirty || dirtyShardCnt > shardCnt*4/6 { - //copy(e.table[:], e.dictTable) - e.table = *(*[tableSize]tableEntry)(e.dictTable) - for i := range e.tableShardDirty { - e.tableShardDirty[i] = false - } - e.allDirty = false - return - } - for i := range e.tableShardDirty { - if !e.tableShardDirty[i] { - continue - } - - //copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) - *(*[shardSize]tableEntry)(e.table[i*shardSize:]) = *(*[shardSize]tableEntry)(e.dictTable[i*shardSize:]) - e.tableShardDirty[i] = false - } - e.allDirty = false -} - -func (e *fastEncoderDict) markAllShardsDirty() { - e.allDirty = true -} - -func (e *fastEncoderDict) markShardDirty(entryNum uint32) { - e.tableShardDirty[entryNum/tableShardSize] = true -} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go deleted file mode 100644 index 8f8223cd3..000000000 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ /dev/null @@ -1,642 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "crypto/rand" - "errors" - "fmt" - "io" - "math" - rdebug "runtime/debug" - "sync" - - "github.com/klauspost/compress/zstd/internal/xxhash" -) - -// Encoder provides encoding to Zstandard. -// An Encoder can be used for either compressing a stream via the -// io.WriteCloser interface supported by the Encoder or as multiple independent -// tasks via the EncodeAll function. -// Smaller encodes are encouraged to use the EncodeAll function. -// Use NewWriter to create a new instance. -type Encoder struct { - o encoderOptions - encoders chan encoder - state encoderState - init sync.Once -} - -type encoder interface { - Encode(blk *blockEnc, src []byte) - EncodeNoHist(blk *blockEnc, src []byte) - Block() *blockEnc - CRC() *xxhash.Digest - AppendCRC([]byte) []byte - WindowSize(size int64) int32 - UseBlock(*blockEnc) - Reset(d *dict, singleBlock bool) -} - -type encoderState struct { - w io.Writer - filling []byte - current []byte - previous []byte - encoder encoder - writing *blockEnc - err error - writeErr error - nWritten int64 - nInput int64 - frameContentSize int64 - headerWritten bool - eofWritten bool - fullFrameWritten bool - - // This waitgroup indicates an encode is running. - wg sync.WaitGroup - // This waitgroup indicates we have a block encoding/writing. - wWg sync.WaitGroup -} - -// NewWriter will create a new Zstandard encoder. -// If the encoder will be used for encoding blocks a nil writer can be used. -func NewWriter(w io.Writer, opts ...EOption) (*Encoder, error) { - initPredefined() - var e Encoder - e.o.setDefault() - for _, o := range opts { - err := o(&e.o) - if err != nil { - return nil, err - } - } - if w != nil { - e.Reset(w) - } - return &e, nil -} - -func (e *Encoder) initialize() { - if e.o.concurrent == 0 { - e.o.setDefault() - } - e.encoders = make(chan encoder, e.o.concurrent) - for i := 0; i < e.o.concurrent; i++ { - enc := e.o.encoder() - e.encoders <- enc - } -} - -// Reset will re-initialize the writer and new writes will encode to the supplied writer -// as a new, independent stream. -func (e *Encoder) Reset(w io.Writer) { - s := &e.state - s.wg.Wait() - s.wWg.Wait() - if cap(s.filling) == 0 { - s.filling = make([]byte, 0, e.o.blockSize) - } - if e.o.concurrent > 1 { - if cap(s.current) == 0 { - s.current = make([]byte, 0, e.o.blockSize) - } - if cap(s.previous) == 0 { - s.previous = make([]byte, 0, e.o.blockSize) - } - s.current = s.current[:0] - s.previous = s.previous[:0] - if s.writing == nil { - s.writing = &blockEnc{lowMem: e.o.lowMem} - s.writing.init() - } - s.writing.initNewEncode() - } - if s.encoder == nil { - s.encoder = e.o.encoder() - } - s.filling = s.filling[:0] - s.encoder.Reset(e.o.dict, false) - s.headerWritten = false - s.eofWritten = false - s.fullFrameWritten = false - s.w = w - s.err = nil - s.nWritten = 0 - s.nInput = 0 - s.writeErr = nil - s.frameContentSize = 0 -} - -// ResetContentSize will reset and set a content size for the next stream. -// If the bytes written does not match the size given an error will be returned -// when calling Close(). -// This is removed when Reset is called. -// Sizes <= 0 results in no content size set. -func (e *Encoder) ResetContentSize(w io.Writer, size int64) { - e.Reset(w) - if size >= 0 { - e.state.frameContentSize = size - } -} - -// Write data to the encoder. -// Input data will be buffered and as the buffer fills up -// content will be compressed and written to the output. -// When done writing, use Close to flush the remaining output -// and write CRC if requested. -func (e *Encoder) Write(p []byte) (n int, err error) { - s := &e.state - if s.eofWritten { - return 0, ErrEncoderClosed - } - for len(p) > 0 { - if len(p)+len(s.filling) < e.o.blockSize { - if e.o.crc { - _, _ = s.encoder.CRC().Write(p) - } - s.filling = append(s.filling, p...) - return n + len(p), nil - } - add := p - if len(p)+len(s.filling) > e.o.blockSize { - add = add[:e.o.blockSize-len(s.filling)] - } - if e.o.crc { - _, _ = s.encoder.CRC().Write(add) - } - s.filling = append(s.filling, add...) - p = p[len(add):] - n += len(add) - if len(s.filling) < e.o.blockSize { - return n, nil - } - err := e.nextBlock(false) - if err != nil { - return n, err - } - if debugAsserts && len(s.filling) > 0 { - panic(len(s.filling)) - } - } - return n, nil -} - -// nextBlock will synchronize and start compressing input in e.state.filling. -// If an error has occurred during encoding it will be returned. -func (e *Encoder) nextBlock(final bool) error { - s := &e.state - // Wait for current block. - s.wg.Wait() - if s.err != nil { - return s.err - } - if len(s.filling) > e.o.blockSize { - return fmt.Errorf("block > maxStoreBlockSize") - } - if !s.headerWritten { - // If we have a single block encode, do a sync compression. - if final && len(s.filling) == 0 && !e.o.fullZero { - s.headerWritten = true - s.fullFrameWritten = true - s.eofWritten = true - return nil - } - if final && len(s.filling) > 0 { - s.current = e.encodeAll(s.encoder, s.filling, s.current[:0]) - var n2 int - n2, s.err = s.w.Write(s.current) - if s.err != nil { - return s.err - } - s.nWritten += int64(n2) - s.nInput += int64(len(s.filling)) - s.current = s.current[:0] - s.filling = s.filling[:0] - s.headerWritten = true - s.fullFrameWritten = true - s.eofWritten = true - return nil - } - - var tmp [maxHeaderSize]byte - fh := frameHeader{ - ContentSize: uint64(s.frameContentSize), - WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)), - SingleSegment: false, - Checksum: e.o.crc, - DictID: e.o.dict.ID(), - } - - dst := fh.appendTo(tmp[:0]) - s.headerWritten = true - s.wWg.Wait() - var n2 int - n2, s.err = s.w.Write(dst) - if s.err != nil { - return s.err - } - s.nWritten += int64(n2) - } - if s.eofWritten { - // Ensure we only write it once. - final = false - } - - if len(s.filling) == 0 { - // Final block, but no data. - if final { - enc := s.encoder - blk := enc.Block() - blk.reset(nil) - blk.last = true - blk.encodeRaw(nil) - s.wWg.Wait() - _, s.err = s.w.Write(blk.output) - s.nWritten += int64(len(blk.output)) - s.eofWritten = true - } - return s.err - } - - // SYNC: - if e.o.concurrent == 1 { - src := s.filling - s.nInput += int64(len(s.filling)) - if debugEncoder { - println("Adding sync block,", len(src), "bytes, final:", final) - } - enc := s.encoder - blk := enc.Block() - blk.reset(nil) - enc.Encode(blk, src) - blk.last = final - if final { - s.eofWritten = true - } - - s.err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - if s.err != nil { - return s.err - } - _, s.err = s.w.Write(blk.output) - s.nWritten += int64(len(blk.output)) - s.filling = s.filling[:0] - return s.err - } - - // Move blocks forward. - s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current - s.nInput += int64(len(s.current)) - s.wg.Add(1) - if final { - s.eofWritten = true - } - go func(src []byte) { - if debugEncoder { - println("Adding block,", len(src), "bytes, final:", final) - } - defer func() { - if r := recover(); r != nil { - s.err = fmt.Errorf("panic while encoding: %v", r) - rdebug.PrintStack() - } - s.wg.Done() - }() - enc := s.encoder - blk := enc.Block() - enc.Encode(blk, src) - blk.last = final - // Wait for pending writes. - s.wWg.Wait() - if s.writeErr != nil { - s.err = s.writeErr - return - } - // Transfer encoders from previous write block. - blk.swapEncoders(s.writing) - // Transfer recent offsets to next. - enc.UseBlock(s.writing) - s.writing = blk - s.wWg.Add(1) - go func() { - defer func() { - if r := recover(); r != nil { - s.writeErr = fmt.Errorf("panic while encoding/writing: %v", r) - rdebug.PrintStack() - } - s.wWg.Done() - }() - s.writeErr = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - if s.writeErr != nil { - return - } - _, s.writeErr = s.w.Write(blk.output) - s.nWritten += int64(len(blk.output)) - }() - }(s.current) - return nil -} - -// ReadFrom reads data from r until EOF or error. -// The return value n is the number of bytes read. -// Any error except io.EOF encountered during the read is also returned. -// -// The Copy function uses ReaderFrom if available. -func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { - if debugEncoder { - println("Using ReadFrom") - } - - // Flush any current writes. - if len(e.state.filling) > 0 { - if err := e.nextBlock(false); err != nil { - return 0, err - } - } - e.state.filling = e.state.filling[:e.o.blockSize] - src := e.state.filling - for { - n2, err := r.Read(src) - if e.o.crc { - _, _ = e.state.encoder.CRC().Write(src[:n2]) - } - // src is now the unfilled part... - src = src[n2:] - n += int64(n2) - switch err { - case io.EOF: - e.state.filling = e.state.filling[:len(e.state.filling)-len(src)] - if debugEncoder { - println("ReadFrom: got EOF final block:", len(e.state.filling)) - } - return n, nil - case nil: - default: - if debugEncoder { - println("ReadFrom: got error:", err) - } - e.state.err = err - return n, err - } - if len(src) > 0 { - if debugEncoder { - println("ReadFrom: got space left in source:", len(src)) - } - continue - } - err = e.nextBlock(false) - if err != nil { - return n, err - } - e.state.filling = e.state.filling[:e.o.blockSize] - src = e.state.filling - } -} - -// Flush will send the currently written data to output -// and block until everything has been written. -// This should only be used on rare occasions where pushing the currently queued data is critical. -func (e *Encoder) Flush() error { - s := &e.state - if len(s.filling) > 0 { - err := e.nextBlock(false) - if err != nil { - // Ignore Flush after Close. - if errors.Is(s.err, ErrEncoderClosed) { - return nil - } - return err - } - } - s.wg.Wait() - s.wWg.Wait() - if s.err != nil { - // Ignore Flush after Close. - if errors.Is(s.err, ErrEncoderClosed) { - return nil - } - return s.err - } - return s.writeErr -} - -// Close will flush the final output and close the stream. -// The function will block until everything has been written. -// The Encoder can still be re-used after calling this. -func (e *Encoder) Close() error { - s := &e.state - if s.encoder == nil { - return nil - } - err := e.nextBlock(true) - if err != nil { - if errors.Is(s.err, ErrEncoderClosed) { - return nil - } - return err - } - if s.frameContentSize > 0 { - if s.nInput != s.frameContentSize { - return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput) - } - } - if e.state.fullFrameWritten { - return s.err - } - s.wg.Wait() - s.wWg.Wait() - - if s.err != nil { - return s.err - } - if s.writeErr != nil { - return s.writeErr - } - - // Write CRC - if e.o.crc && s.err == nil { - // heap alloc. - var tmp [4]byte - _, s.err = s.w.Write(s.encoder.AppendCRC(tmp[:0])) - s.nWritten += 4 - } - - // Add padding with content from crypto/rand.Reader - if s.err == nil && e.o.pad > 0 { - add := calcSkippableFrame(s.nWritten, int64(e.o.pad)) - frame, err := skippableFrame(s.filling[:0], add, rand.Reader) - if err != nil { - return err - } - _, s.err = s.w.Write(frame) - } - if s.err == nil { - s.err = ErrEncoderClosed - return nil - } - - return s.err -} - -// EncodeAll will encode all input in src and append it to dst. -// This function can be called concurrently, but each call will only run on a single goroutine. -// If empty input is given, nothing is returned, unless WithZeroFrames is specified. -// Encoded blocks can be concatenated and the result will be the combined input stream. -// Data compressed with EncodeAll can be decoded with the Decoder, -// using either a stream or DecodeAll. -func (e *Encoder) EncodeAll(src, dst []byte) []byte { - e.init.Do(e.initialize) - enc := <-e.encoders - defer func() { - e.encoders <- enc - }() - return e.encodeAll(enc, src, dst) -} - -func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte { - if len(src) == 0 { - if e.o.fullZero { - // Add frame header. - fh := frameHeader{ - ContentSize: 0, - WindowSize: MinWindowSize, - SingleSegment: true, - // Adding a checksum would be a waste of space. - Checksum: false, - DictID: 0, - } - dst = fh.appendTo(dst) - - // Write raw block as last one only. - var blk blockHeader - blk.setSize(0) - blk.setType(blockTypeRaw) - blk.setLast(true) - dst = blk.appendTo(dst) - } - return dst - } - - // Use single segments when above minimum window and below window size. - single := len(src) <= e.o.windowSize && len(src) > MinWindowSize - if e.o.single != nil { - single = *e.o.single - } - fh := frameHeader{ - ContentSize: uint64(len(src)), - WindowSize: uint32(enc.WindowSize(int64(len(src)))), - SingleSegment: single, - Checksum: e.o.crc, - DictID: e.o.dict.ID(), - } - - // If less than 1MB, allocate a buffer up front. - if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem { - dst = make([]byte, 0, len(src)) - } - dst = fh.appendTo(dst) - - // If we can do everything in one block, prefer that. - if len(src) <= e.o.blockSize { - enc.Reset(e.o.dict, true) - // Slightly faster with no history and everything in one block. - if e.o.crc { - _, _ = enc.CRC().Write(src) - } - blk := enc.Block() - blk.last = true - if e.o.dict == nil { - enc.EncodeNoHist(blk, src) - } else { - enc.Encode(blk, src) - } - - // If we got the exact same number of literals as input, - // assume the literals cannot be compressed. - oldout := blk.output - // Output directly to dst - blk.output = dst - - err := blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - if err != nil { - panic(err) - } - dst = blk.output - blk.output = oldout - } else { - enc.Reset(e.o.dict, false) - blk := enc.Block() - for len(src) > 0 { - todo := src - if len(todo) > e.o.blockSize { - todo = todo[:e.o.blockSize] - } - src = src[len(todo):] - if e.o.crc { - _, _ = enc.CRC().Write(todo) - } - blk.pushOffsets() - enc.Encode(blk, todo) - if len(src) == 0 { - blk.last = true - } - err := blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy) - if err != nil { - panic(err) - } - dst = append(dst, blk.output...) - blk.reset(nil) - } - } - if e.o.crc { - dst = enc.AppendCRC(dst) - } - // Add padding with content from crypto/rand.Reader - if e.o.pad > 0 { - add := calcSkippableFrame(int64(len(dst)), int64(e.o.pad)) - var err error - dst, err = skippableFrame(dst, add, rand.Reader) - if err != nil { - panic(err) - } - } - return dst -} - -// MaxEncodedSize returns the expected maximum -// size of an encoded block or stream. -func (e *Encoder) MaxEncodedSize(size int) int { - frameHeader := 4 + 2 // magic + frame header & window descriptor - if e.o.dict != nil { - frameHeader += 4 - } - // Frame content size: - if size < 256 { - frameHeader++ - } else if size < 65536+256 { - frameHeader += 2 - } else if size < math.MaxInt32 { - frameHeader += 4 - } else { - frameHeader += 8 - } - // Final crc - if e.o.crc { - frameHeader += 4 - } - - // Max overhead is 3 bytes/block. - // There cannot be 0 blocks. - blocks := (size + e.o.blockSize) / e.o.blockSize - - // Combine, add padding. - maxSz := frameHeader + 3*blocks + size - if e.o.pad > 1 { - maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad)) - } - return maxSz -} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go deleted file mode 100644 index 20671dcb9..000000000 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ /dev/null @@ -1,339 +0,0 @@ -package zstd - -import ( - "errors" - "fmt" - "math" - "math/bits" - "runtime" - "strings" -) - -// EOption is an option for creating a encoder. -type EOption func(*encoderOptions) error - -// options retains accumulated state of multiple options. -type encoderOptions struct { - concurrent int - level EncoderLevel - single *bool - pad int - blockSize int - windowSize int - crc bool - fullZero bool - noEntropy bool - allLitEntropy bool - customWindow bool - customALEntropy bool - customBlockSize bool - lowMem bool - dict *dict -} - -func (o *encoderOptions) setDefault() { - *o = encoderOptions{ - concurrent: runtime.GOMAXPROCS(0), - crc: true, - single: nil, - blockSize: maxCompressedBlockSize, - windowSize: 8 << 20, - level: SpeedDefault, - allLitEntropy: false, - lowMem: false, - } -} - -// encoder returns an encoder with the selected options. -func (o encoderOptions) encoder() encoder { - switch o.level { - case SpeedFastest: - if o.dict != nil { - return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} - } - return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} - - case SpeedDefault: - if o.dict != nil { - return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}} - } - return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} - case SpeedBetterCompression: - if o.dict != nil { - return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} - } - return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} - case SpeedBestCompression: - return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} - } - panic("unknown compression level") -} - -// WithEncoderCRC will add CRC value to output. -// Output will be 4 bytes larger. -func WithEncoderCRC(b bool) EOption { - return func(o *encoderOptions) error { o.crc = b; return nil } -} - -// WithEncoderConcurrency will set the concurrency, -// meaning the maximum number of encoders to run concurrently. -// The value supplied must be at least 1. -// For streams, setting a value of 1 will disable async compression. -// By default this will be set to GOMAXPROCS. -func WithEncoderConcurrency(n int) EOption { - return func(o *encoderOptions) error { - if n <= 0 { - return fmt.Errorf("concurrency must be at least 1") - } - o.concurrent = n - return nil - } -} - -// WithWindowSize will set the maximum allowed back-reference distance. -// The value must be a power of two between MinWindowSize and MaxWindowSize. -// A larger value will enable better compression but allocate more memory and, -// for above-default values, take considerably longer. -// The default value is determined by the compression level and max 8MB. -func WithWindowSize(n int) EOption { - return func(o *encoderOptions) error { - switch { - case n < MinWindowSize: - return fmt.Errorf("window size must be at least %d", MinWindowSize) - case n > MaxWindowSize: - return fmt.Errorf("window size must be at most %d", MaxWindowSize) - case (n & (n - 1)) != 0: - return errors.New("window size must be a power of 2") - } - - o.windowSize = n - o.customWindow = true - if o.blockSize > o.windowSize { - o.blockSize = o.windowSize - o.customBlockSize = true - } - return nil - } -} - -// WithEncoderPadding will add padding to all output so the size will be a multiple of n. -// This can be used to obfuscate the exact output size or make blocks of a certain size. -// The contents will be a skippable frame, so it will be invisible by the decoder. -// n must be > 0 and <= 1GB, 1<<30 bytes. -// The padded area will be filled with data from crypto/rand.Reader. -// If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. -func WithEncoderPadding(n int) EOption { - return func(o *encoderOptions) error { - if n <= 0 { - return fmt.Errorf("padding must be at least 1") - } - // No need to waste our time. - if n == 1 { - n = 0 - } - if n > 1<<30 { - return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ") - } - o.pad = n - return nil - } -} - -// EncoderLevel predefines encoder compression levels. -// Only use the constants made available, since the actual mapping -// of these values are very likely to change and your compression could change -// unpredictably when upgrading the library. -type EncoderLevel int - -const ( - speedNotSet EncoderLevel = iota - - // SpeedFastest will choose the fastest reasonable compression. - // This is roughly equivalent to the fastest Zstandard mode. - SpeedFastest - - // SpeedDefault is the default "pretty fast" compression option. - // This is roughly equivalent to the default Zstandard mode (level 3). - SpeedDefault - - // SpeedBetterCompression will yield better compression than the default. - // Currently it is about zstd level 7-8 with ~ 2x-3x the default CPU usage. - // By using this, notice that CPU usage may go up in the future. - SpeedBetterCompression - - // SpeedBestCompression will choose the best available compression option. - // This will offer the best compression no matter the CPU cost. - SpeedBestCompression - - // speedLast should be kept as the last actual compression option. - // The is not for external usage, but is used to keep track of the valid options. - speedLast -) - -// EncoderLevelFromString will convert a string representation of an encoding level back -// to a compression level. The compare is not case sensitive. -// If the string wasn't recognized, (false, SpeedDefault) will be returned. -func EncoderLevelFromString(s string) (bool, EncoderLevel) { - for l := speedNotSet + 1; l < speedLast; l++ { - if strings.EqualFold(s, l.String()) { - return true, l - } - } - return false, SpeedDefault -} - -// EncoderLevelFromZstd will return an encoder level that closest matches the compression -// ratio of a specific zstd compression level. -// Many input values will provide the same compression level. -func EncoderLevelFromZstd(level int) EncoderLevel { - switch { - case level < 3: - return SpeedFastest - case level >= 3 && level < 6: - return SpeedDefault - case level >= 6 && level < 10: - return SpeedBetterCompression - default: - return SpeedBestCompression - } -} - -// String provides a string representation of the compression level. -func (e EncoderLevel) String() string { - switch e { - case SpeedFastest: - return "fastest" - case SpeedDefault: - return "default" - case SpeedBetterCompression: - return "better" - case SpeedBestCompression: - return "best" - default: - return "invalid" - } -} - -// WithEncoderLevel specifies a predefined compression level. -func WithEncoderLevel(l EncoderLevel) EOption { - return func(o *encoderOptions) error { - switch { - case l <= speedNotSet || l >= speedLast: - return fmt.Errorf("unknown encoder level") - } - o.level = l - if !o.customWindow { - switch o.level { - case SpeedFastest: - o.windowSize = 4 << 20 - if !o.customBlockSize { - o.blockSize = 1 << 16 - } - case SpeedDefault: - o.windowSize = 8 << 20 - case SpeedBetterCompression: - o.windowSize = 8 << 20 - case SpeedBestCompression: - o.windowSize = 8 << 20 - } - } - if !o.customALEntropy { - o.allLitEntropy = l > SpeedDefault - } - - return nil - } -} - -// WithZeroFrames will encode 0 length input as full frames. -// This can be needed for compatibility with zstandard usage, -// but is not needed for this package. -func WithZeroFrames(b bool) EOption { - return func(o *encoderOptions) error { - o.fullZero = b - return nil - } -} - -// WithAllLitEntropyCompression will apply entropy compression if no matches are found. -// Disabling this will skip incompressible data faster, but in cases with no matches but -// skewed character distribution compression is lost. -// Default value depends on the compression level selected. -func WithAllLitEntropyCompression(b bool) EOption { - return func(o *encoderOptions) error { - o.customALEntropy = true - o.allLitEntropy = b - return nil - } -} - -// WithNoEntropyCompression will always skip entropy compression of literals. -// This can be useful if content has matches, but unlikely to benefit from entropy -// compression. Usually the slight speed improvement is not worth enabling this. -func WithNoEntropyCompression(b bool) EOption { - return func(o *encoderOptions) error { - o.noEntropy = b - return nil - } -} - -// WithSingleSegment will set the "single segment" flag when EncodeAll is used. -// If this flag is set, data must be regenerated within a single continuous memory segment. -// In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present. -// As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content. -// In order to preserve the decoder from unreasonable memory requirements, -// a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. -// For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. -// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. -// If this is not specified, block encodes will automatically choose this based on the input size and the window size. -// This setting has no effect on streamed encodes. -func WithSingleSegment(b bool) EOption { - return func(o *encoderOptions) error { - o.single = &b - return nil - } -} - -// WithLowerEncoderMem will trade in some memory cases trade less memory usage for -// slower encoding speed. -// This will not change the window size which is the primary function for reducing -// memory usage. See WithWindowSize. -func WithLowerEncoderMem(b bool) EOption { - return func(o *encoderOptions) error { - o.lowMem = b - return nil - } -} - -// WithEncoderDict allows to register a dictionary that will be used for the encode. -// -// The slice dict must be in the [dictionary format] produced by -// "zstd --train" from the Zstandard reference implementation. -// -// The encoder *may* choose to use no dictionary instead for certain payloads. -// -// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format -func WithEncoderDict(dict []byte) EOption { - return func(o *encoderOptions) error { - d, err := loadDict(dict) - if err != nil { - return err - } - o.dict = d - return nil - } -} - -// WithEncoderDictRaw registers a dictionary that may be used by the encoder. -// -// The slice content may contain arbitrary data. It will be used as an initial -// history. -func WithEncoderDictRaw(id uint32, content []byte) EOption { - return func(o *encoderOptions) error { - if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { - return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) - } - o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}} - return nil - } -} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go deleted file mode 100644 index e47af66e7..000000000 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ /dev/null @@ -1,415 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "encoding/binary" - "encoding/hex" - "errors" - "io" - - "github.com/klauspost/compress/zstd/internal/xxhash" -) - -type frameDec struct { - o decoderOptions - crc *xxhash.Digest - - WindowSize uint64 - - // Frame history passed between blocks - history history - - rawInput byteBuffer - - // Byte buffer that can be reused for small input blocks. - bBuf byteBuf - - FrameContentSize uint64 - - DictionaryID uint32 - HasCheckSum bool - SingleSegment bool -} - -const ( - // MinWindowSize is the minimum Window Size, which is 1 KB. - MinWindowSize = 1 << 10 - - // MaxWindowSize is the maximum encoder window size - // and the default decoder maximum window size. - MaxWindowSize = 1 << 29 -) - -const ( - frameMagic = "\x28\xb5\x2f\xfd" - skippableFrameMagic = "\x2a\x4d\x18" -) - -func newFrameDec(o decoderOptions) *frameDec { - if o.maxWindowSize > o.maxDecodedSize { - o.maxWindowSize = o.maxDecodedSize - } - d := frameDec{ - o: o, - } - return &d -} - -// reset will read the frame header and prepare for block decoding. -// If nothing can be read from the input, io.EOF will be returned. -// Any other error indicated that the stream contained data, but -// there was a problem. -func (d *frameDec) reset(br byteBuffer) error { - d.HasCheckSum = false - d.WindowSize = 0 - var signature [4]byte - for { - var err error - // Check if we can read more... - b, err := br.readSmall(1) - switch err { - case io.EOF, io.ErrUnexpectedEOF: - return io.EOF - case nil: - signature[0] = b[0] - default: - return err - } - // Read the rest, don't allow io.ErrUnexpectedEOF - b, err = br.readSmall(3) - switch err { - case io.EOF: - return io.EOF - case nil: - copy(signature[1:], b) - default: - return err - } - - if string(signature[1:4]) != skippableFrameMagic || signature[0]&0xf0 != 0x50 { - if debugDecoder { - println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString([]byte(skippableFrameMagic))) - } - // Break if not skippable frame. - break - } - // Read size to skip - b, err = br.readSmall(4) - if err != nil { - if debugDecoder { - println("Reading Frame Size", err) - } - return err - } - n := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) - println("Skipping frame with", n, "bytes.") - err = br.skipN(int64(n)) - if err != nil { - if debugDecoder { - println("Reading discarded frame", err) - } - return err - } - } - if string(signature[:]) != frameMagic { - if debugDecoder { - println("Got magic numbers: ", signature, "want:", []byte(frameMagic)) - } - return ErrMagicMismatch - } - - // Read Frame_Header_Descriptor - fhd, err := br.readByte() - if err != nil { - if debugDecoder { - println("Reading Frame_Header_Descriptor", err) - } - return err - } - d.SingleSegment = fhd&(1<<5) != 0 - - if fhd&(1<<3) != 0 { - return errors.New("reserved bit set on frame header") - } - - // Read Window_Descriptor - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor - d.WindowSize = 0 - if !d.SingleSegment { - wd, err := br.readByte() - if err != nil { - if debugDecoder { - println("Reading Window_Descriptor", err) - } - return err - } - if debugDecoder { - printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) - } - windowLog := 10 + (wd >> 3) - windowBase := uint64(1) << windowLog - windowAdd := (windowBase / 8) * uint64(wd&0x7) - d.WindowSize = windowBase + windowAdd - } - - // Read Dictionary_ID - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - d.DictionaryID = 0 - if size := fhd & 3; size != 0 { - if size == 3 { - size = 4 - } - - b, err := br.readSmall(int(size)) - if err != nil { - println("Reading Dictionary_ID", err) - return err - } - var id uint32 - switch len(b) { - case 1: - id = uint32(b[0]) - case 2: - id = uint32(b[0]) | (uint32(b[1]) << 8) - case 4: - id = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) - } - if debugDecoder { - println("Dict size", size, "ID:", id) - } - d.DictionaryID = id - } - - // Read Frame_Content_Size - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size - var fcsSize int - v := fhd >> 6 - switch v { - case 0: - if d.SingleSegment { - fcsSize = 1 - } - default: - fcsSize = 1 << v - } - d.FrameContentSize = fcsUnknown - if fcsSize > 0 { - b, err := br.readSmall(fcsSize) - if err != nil { - println("Reading Frame content", err) - return err - } - switch len(b) { - case 1: - d.FrameContentSize = uint64(b[0]) - case 2: - // When FCS_Field_Size is 2, the offset of 256 is added. - d.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) + 256 - case 4: - d.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) | (uint64(b[2]) << 16) | (uint64(b[3]) << 24) - case 8: - d1 := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) - d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24) - d.FrameContentSize = uint64(d1) | (uint64(d2) << 32) - } - if debugDecoder { - println("Read FCS:", d.FrameContentSize) - } - } - - // Move this to shared. - d.HasCheckSum = fhd&(1<<2) != 0 - if d.HasCheckSum { - if d.crc == nil { - d.crc = xxhash.New() - } - d.crc.Reset() - } - - if d.WindowSize > d.o.maxWindowSize { - if debugDecoder { - printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) - } - return ErrWindowSizeExceeded - } - - if d.WindowSize == 0 && d.SingleSegment { - // We may not need window in this case. - d.WindowSize = d.FrameContentSize - if d.WindowSize < MinWindowSize { - d.WindowSize = MinWindowSize - } - if d.WindowSize > d.o.maxDecodedSize { - if debugDecoder { - printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) - } - return ErrDecoderSizeExceeded - } - } - - // The minimum Window_Size is 1 KB. - if d.WindowSize < MinWindowSize { - if debugDecoder { - println("got window size: ", d.WindowSize) - } - return ErrWindowSizeTooSmall - } - d.history.windowSize = int(d.WindowSize) - if !d.o.lowMem || d.history.windowSize < maxBlockSize { - // Alloc 2x window size if not low-mem, or window size below 2MB. - d.history.allocFrameBuffer = d.history.windowSize * 2 - } else { - if d.o.lowMem { - // Alloc with 1MB extra. - d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize/2 - } else { - // Alloc with 2MB extra. - d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize - } - } - - if debugDecoder { - println("Frame: Dict:", d.DictionaryID, "FrameContentSize:", d.FrameContentSize, "singleseg:", d.SingleSegment, "window:", d.WindowSize, "crc:", d.HasCheckSum) - } - - // history contains input - maybe we do something - d.rawInput = br - return nil -} - -// next will start decoding the next block from stream. -func (d *frameDec) next(block *blockDec) error { - if debugDecoder { - println("decoding new block") - } - err := block.reset(d.rawInput, d.WindowSize) - if err != nil { - println("block error:", err) - // Signal the frame decoder we have a problem. - block.sendErr(err) - return err - } - return nil -} - -// checkCRC will check the checksum, assuming the frame has one. -// Will return ErrCRCMismatch if crc check failed, otherwise nil. -func (d *frameDec) checkCRC() error { - // We can overwrite upper tmp now - buf, err := d.rawInput.readSmall(4) - if err != nil { - println("CRC missing?", err) - return err - } - - want := binary.LittleEndian.Uint32(buf[:4]) - got := uint32(d.crc.Sum64()) - - if got != want { - if debugDecoder { - printf("CRC check failed: got %08x, want %08x\n", got, want) - } - return ErrCRCMismatch - } - if debugDecoder { - printf("CRC ok %08x\n", got) - } - return nil -} - -// consumeCRC skips over the checksum, assuming the frame has one. -func (d *frameDec) consumeCRC() error { - _, err := d.rawInput.readSmall(4) - if err != nil { - println("CRC missing?", err) - } - return err -} - -// runDecoder will run the decoder for the remainder of the frame. -func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { - saved := d.history.b - - // We use the history for output to avoid copying it. - d.history.b = dst - d.history.ignoreBuffer = len(dst) - // Store input length, so we only check new data. - crcStart := len(dst) - d.history.decoders.maxSyncLen = 0 - if d.o.limitToCap { - d.history.decoders.maxSyncLen = uint64(cap(dst) - len(dst)) - } - if d.FrameContentSize != fcsUnknown { - if !d.o.limitToCap || d.FrameContentSize+uint64(len(dst)) < d.history.decoders.maxSyncLen { - d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst)) - } - if d.history.decoders.maxSyncLen > d.o.maxDecodedSize { - if debugDecoder { - println("maxSyncLen:", d.history.decoders.maxSyncLen, "> maxDecodedSize:", d.o.maxDecodedSize) - } - return dst, ErrDecoderSizeExceeded - } - if debugDecoder { - println("maxSyncLen:", d.history.decoders.maxSyncLen) - } - if !d.o.limitToCap && uint64(cap(dst)) < d.history.decoders.maxSyncLen { - // Alloc for output - dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc) - copy(dst2, dst) - dst = dst2 - } - } - var err error - for { - err = dec.reset(d.rawInput, d.WindowSize) - if err != nil { - break - } - if debugDecoder { - println("next block:", dec) - } - err = dec.decodeBuf(&d.history) - if err != nil { - break - } - if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize { - println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize) - err = ErrDecoderSizeExceeded - break - } - if d.o.limitToCap && len(d.history.b) > cap(dst) { - println("runDecoder: cap exceeded", uint64(len(d.history.b)), ">", cap(dst)) - err = ErrDecoderSizeExceeded - break - } - if uint64(len(d.history.b)-crcStart) > d.FrameContentSize { - println("runDecoder: FrameContentSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.FrameContentSize) - err = ErrFrameSizeExceeded - break - } - if dec.Last { - break - } - if debugDecoder { - println("runDecoder: FrameContentSize", uint64(len(d.history.b)-crcStart), "<=", d.FrameContentSize) - } - } - dst = d.history.b - if err == nil { - if d.FrameContentSize != fcsUnknown && uint64(len(d.history.b)-crcStart) != d.FrameContentSize { - err = ErrFrameSizeMismatch - } else if d.HasCheckSum { - if d.o.ignoreChecksum { - err = d.consumeCRC() - } else { - d.crc.Write(dst[crcStart:]) - err = d.checkCRC() - } - } - } - d.history.b = saved - return dst, err -} diff --git a/vendor/github.com/klauspost/compress/zstd/frameenc.go b/vendor/github.com/klauspost/compress/zstd/frameenc.go deleted file mode 100644 index 667ca0679..000000000 --- a/vendor/github.com/klauspost/compress/zstd/frameenc.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "encoding/binary" - "fmt" - "io" - "math" - "math/bits" -) - -type frameHeader struct { - ContentSize uint64 - WindowSize uint32 - SingleSegment bool - Checksum bool - DictID uint32 -} - -const maxHeaderSize = 14 - -func (f frameHeader) appendTo(dst []byte) []byte { - dst = append(dst, frameMagic...) - var fhd uint8 - if f.Checksum { - fhd |= 1 << 2 - } - if f.SingleSegment { - fhd |= 1 << 5 - } - - var dictIDContent []byte - if f.DictID > 0 { - var tmp [4]byte - if f.DictID < 256 { - fhd |= 1 - tmp[0] = uint8(f.DictID) - dictIDContent = tmp[:1] - } else if f.DictID < 1<<16 { - fhd |= 2 - binary.LittleEndian.PutUint16(tmp[:2], uint16(f.DictID)) - dictIDContent = tmp[:2] - } else { - fhd |= 3 - binary.LittleEndian.PutUint32(tmp[:4], f.DictID) - dictIDContent = tmp[:4] - } - } - var fcs uint8 - if f.ContentSize >= 256 { - fcs++ - } - if f.ContentSize >= 65536+256 { - fcs++ - } - if f.ContentSize >= 0xffffffff { - fcs++ - } - - fhd |= fcs << 6 - - dst = append(dst, fhd) - if !f.SingleSegment { - const winLogMin = 10 - windowLog := (bits.Len32(f.WindowSize-1) - winLogMin) << 3 - dst = append(dst, uint8(windowLog)) - } - if f.DictID > 0 { - dst = append(dst, dictIDContent...) - } - switch fcs { - case 0: - if f.SingleSegment { - dst = append(dst, uint8(f.ContentSize)) - } - // Unless SingleSegment is set, framessizes < 256 are not stored. - case 1: - f.ContentSize -= 256 - dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8)) - case 2: - dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8), uint8(f.ContentSize>>16), uint8(f.ContentSize>>24)) - case 3: - dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8), uint8(f.ContentSize>>16), uint8(f.ContentSize>>24), - uint8(f.ContentSize>>32), uint8(f.ContentSize>>40), uint8(f.ContentSize>>48), uint8(f.ContentSize>>56)) - default: - panic("invalid fcs") - } - return dst -} - -const skippableFrameHeader = 4 + 4 - -// calcSkippableFrame will return a total size to be added for written -// to be divisible by multiple. -// The value will always be > skippableFrameHeader. -// The function will panic if written < 0 or wantMultiple <= 0. -func calcSkippableFrame(written, wantMultiple int64) int { - if wantMultiple <= 0 { - panic("wantMultiple <= 0") - } - if written < 0 { - panic("written < 0") - } - leftOver := written % wantMultiple - if leftOver == 0 { - return 0 - } - toAdd := wantMultiple - leftOver - for toAdd < skippableFrameHeader { - toAdd += wantMultiple - } - return int(toAdd) -} - -// skippableFrame will add a skippable frame with a total size of bytes. -// total should be >= skippableFrameHeader and < math.MaxUint32. -func skippableFrame(dst []byte, total int, r io.Reader) ([]byte, error) { - if total == 0 { - return dst, nil - } - if total < skippableFrameHeader { - return dst, fmt.Errorf("requested skippable frame (%d) < 8", total) - } - if int64(total) > math.MaxUint32 { - return dst, fmt.Errorf("requested skippable frame (%d) > max uint32", total) - } - dst = append(dst, 0x50, 0x2a, 0x4d, 0x18) - f := uint32(total - skippableFrameHeader) - dst = append(dst, uint8(f), uint8(f>>8), uint8(f>>16), uint8(f>>24)) - start := len(dst) - dst = append(dst, make([]byte, f)...) - _, err := io.ReadFull(r, dst[start:]) - return dst, err -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go deleted file mode 100644 index 2f8860a72..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "encoding/binary" - "errors" - "fmt" - "io" -) - -const ( - tablelogAbsoluteMax = 9 -) - -const ( - /*!MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ - maxMemoryUsage = tablelogAbsoluteMax + 2 - - maxTableLog = maxMemoryUsage - 2 - maxTablesize = 1 << maxTableLog - maxTableMask = (1 << maxTableLog) - 1 - minTablelog = 5 - maxSymbolValue = 255 -) - -// fseDecoder provides temporary storage for compression and decompression. -type fseDecoder struct { - dt [maxTablesize]decSymbol // Decompression table. - symbolLen uint16 // Length of active part of the symbol table. - actualTableLog uint8 // Selected tablelog. - maxBits uint8 // Maximum number of additional bits - - // used for table creation to avoid allocations. - stateTable [256]uint16 - norm [maxSymbolValue + 1]int16 - preDefined bool -} - -// tableStep returns the next table index. -func tableStep(tableSize uint32) uint32 { - return (tableSize >> 1) + (tableSize >> 3) + 3 -} - -// readNCount will read the symbol distribution so decoding tables can be constructed. -func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { - var ( - charnum uint16 - previous0 bool - ) - if b.remain() < 4 { - return errors.New("input too small") - } - bitStream := b.Uint32NC() - nbBits := uint((bitStream & 0xF) + minTablelog) // extract tableLog - if nbBits > tablelogAbsoluteMax { - println("Invalid tablelog:", nbBits) - return errors.New("tableLog too large") - } - bitStream >>= 4 - bitCount := uint(4) - - s.actualTableLog = uint8(nbBits) - remaining := int32((1 << nbBits) + 1) - threshold := int32(1 << nbBits) - gotTotal := int32(0) - nbBits++ - - for remaining > 1 && charnum <= maxSymbol { - if previous0 { - //println("prev0") - n0 := charnum - for (bitStream & 0xFFFF) == 0xFFFF { - //println("24 x 0") - n0 += 24 - if r := b.remain(); r > 5 { - b.advance(2) - // The check above should make sure we can read 32 bits - bitStream = b.Uint32NC() >> bitCount - } else { - // end of bit stream - bitStream >>= 16 - bitCount += 16 - } - } - //printf("bitstream: %d, 0b%b", bitStream&3, bitStream) - for (bitStream & 3) == 3 { - n0 += 3 - bitStream >>= 2 - bitCount += 2 - } - n0 += uint16(bitStream & 3) - bitCount += 2 - - if n0 > maxSymbolValue { - return errors.New("maxSymbolValue too small") - } - //println("inserting ", n0-charnum, "zeroes from idx", charnum, "ending before", n0) - for charnum < n0 { - s.norm[uint8(charnum)] = 0 - charnum++ - } - - if r := b.remain(); r >= 7 || r-int(bitCount>>3) >= 4 { - b.advance(bitCount >> 3) - bitCount &= 7 - // The check above should make sure we can read 32 bits - bitStream = b.Uint32NC() >> bitCount - } else { - bitStream >>= 2 - } - } - - max := (2*threshold - 1) - remaining - var count int32 - - if int32(bitStream)&(threshold-1) < max { - count = int32(bitStream) & (threshold - 1) - if debugAsserts && nbBits < 1 { - panic("nbBits underflow") - } - bitCount += nbBits - 1 - } else { - count = int32(bitStream) & (2*threshold - 1) - if count >= threshold { - count -= max - } - bitCount += nbBits - } - - // extra accuracy - count-- - if count < 0 { - // -1 means +1 - remaining += count - gotTotal -= count - } else { - remaining -= count - gotTotal += count - } - s.norm[charnum&0xff] = int16(count) - charnum++ - previous0 = count == 0 - for remaining < threshold { - nbBits-- - threshold >>= 1 - } - - if r := b.remain(); r >= 7 || r-int(bitCount>>3) >= 4 { - b.advance(bitCount >> 3) - bitCount &= 7 - // The check above should make sure we can read 32 bits - bitStream = b.Uint32NC() >> (bitCount & 31) - } else { - bitCount -= (uint)(8 * (len(b.b) - 4 - b.off)) - b.off = len(b.b) - 4 - bitStream = b.Uint32() >> (bitCount & 31) - } - } - s.symbolLen = charnum - if s.symbolLen <= 1 { - return fmt.Errorf("symbolLen (%d) too small", s.symbolLen) - } - if s.symbolLen > maxSymbolValue+1 { - return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) - } - if remaining != 1 { - return fmt.Errorf("corruption detected (remaining %d != 1)", remaining) - } - if bitCount > 32 { - return fmt.Errorf("corruption detected (bitCount %d > 32)", bitCount) - } - if gotTotal != 1<<s.actualTableLog { - return fmt.Errorf("corruption detected (total %d != %d)", gotTotal, 1<<s.actualTableLog) - } - b.advance((bitCount + 7) >> 3) - return s.buildDtable() -} - -func (s *fseDecoder) mustReadFrom(r io.Reader) { - fatalErr := func(err error) { - if err != nil { - panic(err) - } - } - // dt [maxTablesize]decSymbol // Decompression table. - // symbolLen uint16 // Length of active part of the symbol table. - // actualTableLog uint8 // Selected tablelog. - // maxBits uint8 // Maximum number of additional bits - // // used for table creation to avoid allocations. - // stateTable [256]uint16 - // norm [maxSymbolValue + 1]int16 - // preDefined bool - fatalErr(binary.Read(r, binary.LittleEndian, &s.dt)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.symbolLen)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.actualTableLog)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.maxBits)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.stateTable)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.norm)) - fatalErr(binary.Read(r, binary.LittleEndian, &s.preDefined)) -} - -// decSymbol contains information about a state entry, -// Including the state offset base, the output symbol and -// the number of bits to read for the low part of the destination state. -// Using a composite uint64 is faster than a struct with separate members. -type decSymbol uint64 - -func newDecSymbol(nbits, addBits uint8, newState uint16, baseline uint32) decSymbol { - return decSymbol(nbits) | (decSymbol(addBits) << 8) | (decSymbol(newState) << 16) | (decSymbol(baseline) << 32) -} - -func (d decSymbol) nbBits() uint8 { - return uint8(d) -} - -func (d decSymbol) addBits() uint8 { - return uint8(d >> 8) -} - -func (d decSymbol) newState() uint16 { - return uint16(d >> 16) -} - -func (d decSymbol) baselineInt() int { - return int(d >> 32) -} - -func (d *decSymbol) setNBits(nBits uint8) { - const mask = 0xffffffffffffff00 - *d = (*d & mask) | decSymbol(nBits) -} - -func (d *decSymbol) setAddBits(addBits uint8) { - const mask = 0xffffffffffff00ff - *d = (*d & mask) | (decSymbol(addBits) << 8) -} - -func (d *decSymbol) setNewState(state uint16) { - const mask = 0xffffffff0000ffff - *d = (*d & mask) | decSymbol(state)<<16 -} - -func (d *decSymbol) setExt(addBits uint8, baseline uint32) { - const mask = 0xffff00ff - *d = (*d & mask) | (decSymbol(addBits) << 8) | (decSymbol(baseline) << 32) -} - -// decSymbolValue returns the transformed decSymbol for the given symbol. -func decSymbolValue(symb uint8, t []baseOffset) (decSymbol, error) { - if int(symb) >= len(t) { - return 0, fmt.Errorf("rle symbol %d >= max %d", symb, len(t)) - } - lu := t[symb] - return newDecSymbol(0, lu.addBits, 0, lu.baseLine), nil -} - -// setRLE will set the decoder til RLE mode. -func (s *fseDecoder) setRLE(symbol decSymbol) { - s.actualTableLog = 0 - s.maxBits = symbol.addBits() - s.dt[0] = symbol -} - -// transform will transform the decoder table into a table usable for -// decoding without having to apply the transformation while decoding. -// The state will contain the base value and the number of bits to read. -func (s *fseDecoder) transform(t []baseOffset) error { - tableSize := uint16(1 << s.actualTableLog) - s.maxBits = 0 - for i, v := range s.dt[:tableSize] { - add := v.addBits() - if int(add) >= len(t) { - return fmt.Errorf("invalid decoding table entry %d, symbol %d >= max (%d)", i, v.addBits(), len(t)) - } - lu := t[add] - if lu.addBits > s.maxBits { - s.maxBits = lu.addBits - } - v.setExt(lu.addBits, lu.baseLine) - s.dt[i] = v - } - return nil -} - -type fseState struct { - dt []decSymbol - state decSymbol -} - -// Initialize and decodeAsync first state and symbol. -func (s *fseState) init(br *bitReader, tableLog uint8, dt []decSymbol) { - s.dt = dt - br.fill() - s.state = dt[br.getBits(tableLog)] -} - -// final returns the current state symbol without decoding the next. -func (s decSymbol) final() (int, uint8) { - return s.baselineInt(), s.addBits() -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go deleted file mode 100644 index d04a829b0..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go +++ /dev/null @@ -1,65 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -package zstd - -import ( - "fmt" -) - -type buildDtableAsmContext struct { - // inputs - stateTable *uint16 - norm *int16 - dt *uint64 - - // outputs --- set by the procedure in the case of error; - // for interpretation please see the error handling part below - errParam1 uint64 - errParam2 uint64 -} - -// buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable. -// Function returns non-zero exit code on error. -// -//go:noescape -func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int - -// please keep in sync with _generate/gen_fse.go -const ( - errorCorruptedNormalizedCounter = 1 - errorNewStateTooBig = 2 - errorNewStateNoBits = 3 -) - -// buildDtable will build the decoding table. -func (s *fseDecoder) buildDtable() error { - ctx := buildDtableAsmContext{ - stateTable: &s.stateTable[0], - norm: &s.norm[0], - dt: (*uint64)(&s.dt[0]), - } - code := buildDtable_asm(s, &ctx) - - if code != 0 { - switch code { - case errorCorruptedNormalizedCounter: - position := ctx.errParam1 - return fmt.Errorf("corrupted input (position=%d, expected 0)", position) - - case errorNewStateTooBig: - newState := decSymbol(ctx.errParam1) - size := ctx.errParam2 - return fmt.Errorf("newState (%d) outside table size (%d)", newState, size) - - case errorNewStateNoBits: - newState := decSymbol(ctx.errParam1) - oldState := decSymbol(ctx.errParam2) - return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, oldState) - - default: - return fmt.Errorf("buildDtable_asm returned unhandled nonzero code = %d", code) - } - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s deleted file mode 100644 index bcde39869..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +++ /dev/null @@ -1,126 +0,0 @@ -// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. - -//go:build !appengine && !noasm && gc && !noasm - -// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int -TEXT ·buildDtable_asm(SB), $0-24 - MOVQ ctx+8(FP), CX - MOVQ s+0(FP), DI - - // Load values - MOVBQZX 4098(DI), DX - XORQ AX, AX - BTSQ DX, AX - MOVQ (CX), BX - MOVQ 16(CX), SI - LEAQ -1(AX), R8 - MOVQ 8(CX), CX - MOVWQZX 4096(DI), DI - - // End load values - // Init, lay down lowprob symbols - XORQ R9, R9 - JMP init_main_loop_condition - -init_main_loop: - MOVWQSX (CX)(R9*2), R10 - CMPW R10, $-1 - JNE do_not_update_high_threshold - MOVB R9, 1(SI)(R8*8) - DECQ R8 - MOVQ $0x0000000000000001, R10 - -do_not_update_high_threshold: - MOVW R10, (BX)(R9*2) - INCQ R9 - -init_main_loop_condition: - CMPQ R9, DI - JL init_main_loop - - // Spread symbols - // Calculate table step - MOVQ AX, R9 - SHRQ $0x01, R9 - MOVQ AX, R10 - SHRQ $0x03, R10 - LEAQ 3(R9)(R10*1), R9 - - // Fill add bits values - LEAQ -1(AX), R10 - XORQ R11, R11 - XORQ R12, R12 - JMP spread_main_loop_condition - -spread_main_loop: - XORQ R13, R13 - MOVWQSX (CX)(R12*2), R14 - JMP spread_inner_loop_condition - -spread_inner_loop: - MOVB R12, 1(SI)(R11*8) - -adjust_position: - ADDQ R9, R11 - ANDQ R10, R11 - CMPQ R11, R8 - JG adjust_position - INCQ R13 - -spread_inner_loop_condition: - CMPQ R13, R14 - JL spread_inner_loop - INCQ R12 - -spread_main_loop_condition: - CMPQ R12, DI - JL spread_main_loop - TESTQ R11, R11 - JZ spread_check_ok - MOVQ ctx+8(FP), AX - MOVQ R11, 24(AX) - MOVQ $+1, ret+16(FP) - RET - -spread_check_ok: - // Build Decoding table - XORQ DI, DI - -build_table_main_table: - MOVBQZX 1(SI)(DI*8), CX - MOVWQZX (BX)(CX*2), R8 - LEAQ 1(R8), R9 - MOVW R9, (BX)(CX*2) - MOVQ R8, R9 - BSRQ R9, R9 - MOVQ DX, CX - SUBQ R9, CX - SHLQ CL, R8 - SUBQ AX, R8 - MOVB CL, (SI)(DI*8) - MOVW R8, 2(SI)(DI*8) - CMPQ R8, AX - JLE build_table_check1_ok - MOVQ ctx+8(FP), CX - MOVQ R8, 24(CX) - MOVQ AX, 32(CX) - MOVQ $+2, ret+16(FP) - RET - -build_table_check1_ok: - TESTB CL, CL - JNZ build_table_check2_ok - CMPW R8, DI - JNE build_table_check2_ok - MOVQ ctx+8(FP), AX - MOVQ R8, 24(AX) - MOVQ DI, 32(AX) - MOVQ $+3, ret+16(FP) - RET - -build_table_check2_ok: - INCQ DI - CMPQ DI, AX - JL build_table_main_table - MOVQ $+0, ret+16(FP) - RET diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go deleted file mode 100644 index 8adfebb02..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ /dev/null @@ -1,73 +0,0 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - -package zstd - -import ( - "errors" - "fmt" -) - -// buildDtable will build the decoding table. -func (s *fseDecoder) buildDtable() error { - tableSize := uint32(1 << s.actualTableLog) - highThreshold := tableSize - 1 - symbolNext := s.stateTable[:256] - - // Init, lay down lowprob symbols - { - for i, v := range s.norm[:s.symbolLen] { - if v == -1 { - s.dt[highThreshold].setAddBits(uint8(i)) - highThreshold-- - v = 1 - } - symbolNext[i] = uint16(v) - } - } - - // Spread symbols - { - tableMask := tableSize - 1 - step := tableStep(tableSize) - position := uint32(0) - for ss, v := range s.norm[:s.symbolLen] { - for i := 0; i < int(v); i++ { - s.dt[position].setAddBits(uint8(ss)) - for { - // lowprob area - position = (position + step) & tableMask - if position <= highThreshold { - break - } - } - } - } - if position != 0 { - // position must reach all cells once, otherwise normalizedCounter is incorrect - return errors.New("corrupted input (position != 0)") - } - } - - // Build Decoding table - { - tableSize := uint16(1 << s.actualTableLog) - for u, v := range s.dt[:tableSize] { - symbol := v.addBits() - nextState := symbolNext[symbol] - symbolNext[symbol] = nextState + 1 - nBits := s.actualTableLog - byte(highBits(uint32(nextState))) - s.dt[u&maxTableMask].setNBits(nBits) - newState := (nextState << nBits) - tableSize - if newState > tableSize { - return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) - } - if newState == uint16(u) && nBits == 0 { - // Seems weird that this is possible with nbits > 0. - return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) - } - s.dt[u&maxTableMask].setNewState(newState) - } - } - return nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go deleted file mode 100644 index ab26326a8..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ /dev/null @@ -1,701 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "errors" - "fmt" - "math" -) - -const ( - // For encoding we only support up to - maxEncTableLog = 8 - maxEncTablesize = 1 << maxTableLog - maxEncTableMask = (1 << maxTableLog) - 1 - minEncTablelog = 5 - maxEncSymbolValue = maxMatchLengthSymbol -) - -// Scratch provides temporary storage for compression and decompression. -type fseEncoder struct { - symbolLen uint16 // Length of active part of the symbol table. - actualTableLog uint8 // Selected tablelog. - ct cTable // Compression tables. - maxCount int // count of the most probable symbol - zeroBits bool // no bits has prob > 50%. - clearCount bool // clear count - useRLE bool // This encoder is for RLE - preDefined bool // This encoder is predefined. - reUsed bool // Set to know when the encoder has been reused. - rleVal uint8 // RLE Symbol - maxBits uint8 // Maximum output bits after transform. - - // TODO: Technically zstd should be fine with 64 bytes. - count [256]uint32 - norm [256]int16 -} - -// cTable contains tables used for compression. -type cTable struct { - tableSymbol []byte - stateTable []uint16 - symbolTT []symbolTransform -} - -// symbolTransform contains the state transform for a symbol. -type symbolTransform struct { - deltaNbBits uint32 - deltaFindState int16 - outBits uint8 -} - -// String prints values as a human readable string. -func (s symbolTransform) String() string { - return fmt.Sprintf("{deltabits: %08x, findstate:%d outbits:%d}", s.deltaNbBits, s.deltaFindState, s.outBits) -} - -// Histogram allows to populate the histogram and skip that step in the compression, -// It otherwise allows to inspect the histogram when compression is done. -// To indicate that you have populated the histogram call HistogramFinished -// with the value of the highest populated symbol, as well as the number of entries -// in the most populated entry. These are accepted at face value. -func (s *fseEncoder) Histogram() *[256]uint32 { - return &s.count -} - -// HistogramFinished can be called to indicate that the histogram has been populated. -// maxSymbol is the index of the highest set symbol of the next data segment. -// maxCount is the number of entries in the most populated entry. -// These are accepted at face value. -func (s *fseEncoder) HistogramFinished(maxSymbol uint8, maxCount int) { - s.maxCount = maxCount - s.symbolLen = uint16(maxSymbol) + 1 - s.clearCount = maxCount != 0 -} - -// allocCtable will allocate tables needed for compression. -// If existing tables a re big enough, they are simply re-used. -func (s *fseEncoder) allocCtable() { - tableSize := 1 << s.actualTableLog - // get tableSymbol that is big enough. - if cap(s.ct.tableSymbol) < tableSize { - s.ct.tableSymbol = make([]byte, tableSize) - } - s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] - - ctSize := tableSize - if cap(s.ct.stateTable) < ctSize { - s.ct.stateTable = make([]uint16, ctSize) - } - s.ct.stateTable = s.ct.stateTable[:ctSize] - - if cap(s.ct.symbolTT) < 256 { - s.ct.symbolTT = make([]symbolTransform, 256) - } - s.ct.symbolTT = s.ct.symbolTT[:256] -} - -// buildCTable will populate the compression table so it is ready to be used. -func (s *fseEncoder) buildCTable() error { - tableSize := uint32(1 << s.actualTableLog) - highThreshold := tableSize - 1 - var cumul [256]int16 - - s.allocCtable() - tableSymbol := s.ct.tableSymbol[:tableSize] - // symbol start positions - { - cumul[0] = 0 - for ui, v := range s.norm[:s.symbolLen-1] { - u := byte(ui) // one less than reference - if v == -1 { - // Low proba symbol - cumul[u+1] = cumul[u] + 1 - tableSymbol[highThreshold] = u - highThreshold-- - } else { - cumul[u+1] = cumul[u] + v - } - } - // Encode last symbol separately to avoid overflowing u - u := int(s.symbolLen - 1) - v := s.norm[s.symbolLen-1] - if v == -1 { - // Low proba symbol - cumul[u+1] = cumul[u] + 1 - tableSymbol[highThreshold] = byte(u) - highThreshold-- - } else { - cumul[u+1] = cumul[u] + v - } - if uint32(cumul[s.symbolLen]) != tableSize { - return fmt.Errorf("internal error: expected cumul[s.symbolLen] (%d) == tableSize (%d)", cumul[s.symbolLen], tableSize) - } - cumul[s.symbolLen] = int16(tableSize) + 1 - } - // Spread symbols - s.zeroBits = false - { - step := tableStep(tableSize) - tableMask := tableSize - 1 - var position uint32 - // if any symbol > largeLimit, we may have 0 bits output. - largeLimit := int16(1 << (s.actualTableLog - 1)) - for ui, v := range s.norm[:s.symbolLen] { - symbol := byte(ui) - if v > largeLimit { - s.zeroBits = true - } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { - tableSymbol[position] = symbol - position = (position + step) & tableMask - for position > highThreshold { - position = (position + step) & tableMask - } /* Low proba area */ - } - } - - // Check if we have gone through all positions - if position != 0 { - return errors.New("position!=0") - } - } - - // Build table - table := s.ct.stateTable - { - tsi := int(tableSize) - for u, v := range tableSymbol { - // TableU16 : sorted by symbol order; gives next state value - table[cumul[v]] = uint16(tsi + u) - cumul[v]++ - } - } - - // Build Symbol Transformation Table - { - total := int16(0) - symbolTT := s.ct.symbolTT[:s.symbolLen] - tableLog := s.actualTableLog - tl := (uint32(tableLog) << 16) - (1 << tableLog) - for i, v := range s.norm[:s.symbolLen] { - switch v { - case 0: - case -1, 1: - symbolTT[i].deltaNbBits = tl - symbolTT[i].deltaFindState = total - 1 - total++ - default: - maxBitsOut := uint32(tableLog) - highBit(uint32(v-1)) - minStatePlus := uint32(v) << maxBitsOut - symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus - symbolTT[i].deltaFindState = total - v - total += v - } - } - if total != int16(tableSize) { - return fmt.Errorf("total mismatch %d (got) != %d (want)", total, tableSize) - } - } - return nil -} - -var rtbTable = [...]uint32{0, 473195, 504333, 520860, 550000, 700000, 750000, 830000} - -func (s *fseEncoder) setRLE(val byte) { - s.allocCtable() - s.actualTableLog = 0 - s.ct.stateTable = s.ct.stateTable[:1] - s.ct.symbolTT[val] = symbolTransform{ - deltaFindState: 0, - deltaNbBits: 0, - } - if debugEncoder { - println("setRLE: val", val, "symbolTT", s.ct.symbolTT[val]) - } - s.rleVal = val - s.useRLE = true -} - -// setBits will set output bits for the transform. -// if nil is provided, the number of bits is equal to the index. -func (s *fseEncoder) setBits(transform []byte) { - if s.reUsed || s.preDefined { - return - } - if s.useRLE { - if transform == nil { - s.ct.symbolTT[s.rleVal].outBits = s.rleVal - s.maxBits = s.rleVal - return - } - s.maxBits = transform[s.rleVal] - s.ct.symbolTT[s.rleVal].outBits = s.maxBits - return - } - if transform == nil { - for i := range s.ct.symbolTT[:s.symbolLen] { - s.ct.symbolTT[i].outBits = uint8(i) - } - s.maxBits = uint8(s.symbolLen - 1) - return - } - s.maxBits = 0 - for i, v := range transform[:s.symbolLen] { - s.ct.symbolTT[i].outBits = v - if v > s.maxBits { - // We could assume bits always going up, but we play safe. - s.maxBits = v - } - } -} - -// normalizeCount will normalize the count of the symbols so -// the total is equal to the table size. -// If successful, compression tables will also be made ready. -func (s *fseEncoder) normalizeCount(length int) error { - if s.reUsed { - return nil - } - s.optimalTableLog(length) - var ( - tableLog = s.actualTableLog - scale = 62 - uint64(tableLog) - step = (1 << 62) / uint64(length) - vStep = uint64(1) << (scale - 20) - stillToDistribute = int16(1 << tableLog) - largest int - largestP int16 - lowThreshold = (uint32)(length >> tableLog) - ) - if s.maxCount == length { - s.useRLE = true - return nil - } - s.useRLE = false - for i, cnt := range s.count[:s.symbolLen] { - // already handled - // if (count[s] == s.length) return 0; /* rle special case */ - - if cnt == 0 { - s.norm[i] = 0 - continue - } - if cnt <= lowThreshold { - s.norm[i] = -1 - stillToDistribute-- - } else { - proba := (int16)((uint64(cnt) * step) >> scale) - if proba < 8 { - restToBeat := vStep * uint64(rtbTable[proba]) - v := uint64(cnt)*step - (uint64(proba) << scale) - if v > restToBeat { - proba++ - } - } - if proba > largestP { - largestP = proba - largest = i - } - s.norm[i] = proba - stillToDistribute -= proba - } - } - - if -stillToDistribute >= (s.norm[largest] >> 1) { - // corner case, need another normalization method - err := s.normalizeCount2(length) - if err != nil { - return err - } - if debugAsserts { - err = s.validateNorm() - if err != nil { - return err - } - } - return s.buildCTable() - } - s.norm[largest] += stillToDistribute - if debugAsserts { - err := s.validateNorm() - if err != nil { - return err - } - } - return s.buildCTable() -} - -// Secondary normalization method. -// To be used when primary method fails. -func (s *fseEncoder) normalizeCount2(length int) error { - const notYetAssigned = -2 - var ( - distributed uint32 - total = uint32(length) - tableLog = s.actualTableLog - lowThreshold = total >> tableLog - lowOne = (total * 3) >> (tableLog + 1) - ) - for i, cnt := range s.count[:s.symbolLen] { - if cnt == 0 { - s.norm[i] = 0 - continue - } - if cnt <= lowThreshold { - s.norm[i] = -1 - distributed++ - total -= cnt - continue - } - if cnt <= lowOne { - s.norm[i] = 1 - distributed++ - total -= cnt - continue - } - s.norm[i] = notYetAssigned - } - toDistribute := (1 << tableLog) - distributed - - if (total / toDistribute) > lowOne { - // risk of rounding to zero - lowOne = (total * 3) / (toDistribute * 2) - for i, cnt := range s.count[:s.symbolLen] { - if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { - s.norm[i] = 1 - distributed++ - total -= cnt - continue - } - } - toDistribute = (1 << tableLog) - distributed - } - if distributed == uint32(s.symbolLen)+1 { - // all values are pretty poor; - // probably incompressible data (should have already been detected); - // find max, then give all remaining points to max - var maxV int - var maxC uint32 - for i, cnt := range s.count[:s.symbolLen] { - if cnt > maxC { - maxV = i - maxC = cnt - } - } - s.norm[maxV] += int16(toDistribute) - return nil - } - - if total == 0 { - // all of the symbols were low enough for the lowOne or lowThreshold - for i := uint32(0); toDistribute > 0; i = (i + 1) % (uint32(s.symbolLen)) { - if s.norm[i] > 0 { - toDistribute-- - s.norm[i]++ - } - } - return nil - } - - var ( - vStepLog = 62 - uint64(tableLog) - mid = uint64((1 << (vStepLog - 1)) - 1) - rStep = (((1 << vStepLog) * uint64(toDistribute)) + mid) / uint64(total) // scale on remaining - tmpTotal = mid - ) - for i, cnt := range s.count[:s.symbolLen] { - if s.norm[i] == notYetAssigned { - var ( - end = tmpTotal + uint64(cnt)*rStep - sStart = uint32(tmpTotal >> vStepLog) - sEnd = uint32(end >> vStepLog) - weight = sEnd - sStart - ) - if weight < 1 { - return errors.New("weight < 1") - } - s.norm[i] = int16(weight) - tmpTotal = end - } - } - return nil -} - -// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog -func (s *fseEncoder) optimalTableLog(length int) { - tableLog := uint8(maxEncTableLog) - minBitsSrc := highBit(uint32(length)) + 1 - minBitsSymbols := highBit(uint32(s.symbolLen-1)) + 2 - minBits := uint8(minBitsSymbols) - if minBitsSrc < minBitsSymbols { - minBits = uint8(minBitsSrc) - } - - maxBitsSrc := uint8(highBit(uint32(length-1))) - 2 - if maxBitsSrc < tableLog { - // Accuracy can be reduced - tableLog = maxBitsSrc - } - if minBits > tableLog { - tableLog = minBits - } - // Need a minimum to safely represent all symbol values - if tableLog < minEncTablelog { - tableLog = minEncTablelog - } - if tableLog > maxEncTableLog { - tableLog = maxEncTableLog - } - s.actualTableLog = tableLog -} - -// validateNorm validates the normalized histogram table. -func (s *fseEncoder) validateNorm() (err error) { - var total int - for _, v := range s.norm[:s.symbolLen] { - if v >= 0 { - total += int(v) - } else { - total -= int(v) - } - } - defer func() { - if err == nil { - return - } - fmt.Printf("selected TableLog: %d, Symbol length: %d\n", s.actualTableLog, s.symbolLen) - for i, v := range s.norm[:s.symbolLen] { - fmt.Printf("%3d: %5d -> %4d \n", i, s.count[i], v) - } - }() - if total != (1 << s.actualTableLog) { - return fmt.Errorf("warning: Total == %d != %d", total, 1<<s.actualTableLog) - } - for i, v := range s.count[s.symbolLen:] { - if v != 0 { - return fmt.Errorf("warning: Found symbol out of range, %d after cut", i) - } - } - return nil -} - -// writeCount will write the normalized histogram count to header. -// This is read back by readNCount. -func (s *fseEncoder) writeCount(out []byte) ([]byte, error) { - if s.useRLE { - return append(out, s.rleVal), nil - } - if s.preDefined || s.reUsed { - // Never write predefined. - return out, nil - } - - var ( - tableLog = s.actualTableLog - tableSize = 1 << tableLog - previous0 bool - charnum uint16 - - // maximum header size plus 2 extra bytes for final output if bitCount == 0. - maxHeaderSize = ((int(s.symbolLen) * int(tableLog)) >> 3) + 3 + 2 - - // Write Table Size - bitStream = uint32(tableLog - minEncTablelog) - bitCount = uint(4) - remaining = int16(tableSize + 1) /* +1 for extra accuracy */ - threshold = int16(tableSize) - nbBits = uint(tableLog + 1) - outP = len(out) - ) - if cap(out) < outP+maxHeaderSize { - out = append(out, make([]byte, maxHeaderSize*3)...) - out = out[:len(out)-maxHeaderSize*3] - } - out = out[:outP+maxHeaderSize] - - // stops at 1 - for remaining > 1 { - if previous0 { - start := charnum - for s.norm[charnum] == 0 { - charnum++ - } - for charnum >= start+24 { - start += 24 - bitStream += uint32(0xFFFF) << bitCount - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - } - for charnum >= start+3 { - start += 3 - bitStream += 3 << bitCount - bitCount += 2 - } - bitStream += uint32(charnum-start) << bitCount - bitCount += 2 - if bitCount > 16 { - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - bitCount -= 16 - } - } - - count := s.norm[charnum] - charnum++ - max := (2*threshold - 1) - remaining - if count < 0 { - remaining += count - } else { - remaining -= count - } - count++ // +1 for extra accuracy - if count >= threshold { - count += max // [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ - } - bitStream += uint32(count) << bitCount - bitCount += nbBits - if count < max { - bitCount-- - } - - previous0 = count == 1 - if remaining < 1 { - return nil, errors.New("internal error: remaining < 1") - } - for remaining < threshold { - nbBits-- - threshold >>= 1 - } - - if bitCount > 16 { - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += 2 - bitStream >>= 16 - bitCount -= 16 - } - } - - if outP+2 > len(out) { - return nil, fmt.Errorf("internal error: %d > %d, maxheader: %d, sl: %d, tl: %d, normcount: %v", outP+2, len(out), maxHeaderSize, s.symbolLen, int(tableLog), s.norm[:s.symbolLen]) - } - out[outP] = byte(bitStream) - out[outP+1] = byte(bitStream >> 8) - outP += int((bitCount + 7) / 8) - - if charnum > s.symbolLen { - return nil, errors.New("internal error: charnum > s.symbolLen") - } - return out[:outP], nil -} - -// Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) -// note 1 : assume symbolValue is valid (<= maxSymbolValue) -// note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits * -func (s *fseEncoder) bitCost(symbolValue uint8, accuracyLog uint32) uint32 { - minNbBits := s.ct.symbolTT[symbolValue].deltaNbBits >> 16 - threshold := (minNbBits + 1) << 16 - if debugAsserts { - if !(s.actualTableLog < 16) { - panic("!s.actualTableLog < 16") - } - // ensure enough room for renormalization double shift - if !(uint8(accuracyLog) < 31-s.actualTableLog) { - panic("!uint8(accuracyLog) < 31-s.actualTableLog") - } - } - tableSize := uint32(1) << s.actualTableLog - deltaFromThreshold := threshold - (s.ct.symbolTT[symbolValue].deltaNbBits + tableSize) - // linear interpolation (very approximate) - normalizedDeltaFromThreshold := (deltaFromThreshold << accuracyLog) >> s.actualTableLog - bitMultiplier := uint32(1) << accuracyLog - if debugAsserts { - if s.ct.symbolTT[symbolValue].deltaNbBits+tableSize > threshold { - panic("s.ct.symbolTT[symbolValue].deltaNbBits+tableSize > threshold") - } - if normalizedDeltaFromThreshold > bitMultiplier { - panic("normalizedDeltaFromThreshold > bitMultiplier") - } - } - return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold -} - -// Returns the cost in bits of encoding the distribution in count using ctable. -// Histogram should only be up to the last non-zero symbol. -// Returns an -1 if ctable cannot represent all the symbols in count. -func (s *fseEncoder) approxSize(hist []uint32) uint32 { - if int(s.symbolLen) < len(hist) { - // More symbols than we have. - return math.MaxUint32 - } - if s.useRLE { - // We will never reuse RLE encoders. - return math.MaxUint32 - } - const kAccuracyLog = 8 - badCost := (uint32(s.actualTableLog) + 1) << kAccuracyLog - var cost uint32 - for i, v := range hist { - if v == 0 { - continue - } - if s.norm[i] == 0 { - return math.MaxUint32 - } - bitCost := s.bitCost(uint8(i), kAccuracyLog) - if bitCost > badCost { - return math.MaxUint32 - } - cost += v * bitCost - } - return cost >> kAccuracyLog -} - -// maxHeaderSize returns the maximum header size in bits. -// This is not exact size, but we want a penalty for new tables anyway. -func (s *fseEncoder) maxHeaderSize() uint32 { - if s.preDefined { - return 0 - } - if s.useRLE { - return 8 - } - return (((uint32(s.symbolLen) * uint32(s.actualTableLog)) >> 3) + 3) * 8 -} - -// cState contains the compression state of a stream. -type cState struct { - bw *bitWriter - stateTable []uint16 - state uint16 -} - -// init will initialize the compression state to the first symbol of the stream. -func (c *cState) init(bw *bitWriter, ct *cTable, first symbolTransform) { - c.bw = bw - c.stateTable = ct.stateTable - if len(c.stateTable) == 1 { - // RLE - c.stateTable[0] = uint16(0) - c.state = 0 - return - } - nbBitsOut := (first.deltaNbBits + (1 << 15)) >> 16 - im := int32((nbBitsOut << 16) - first.deltaNbBits) - lu := (im >> nbBitsOut) + int32(first.deltaFindState) - c.state = c.stateTable[lu] -} - -// flush will write the tablelog to the output and flush the remaining full bytes. -func (c *cState) flush(tableLog uint8) { - c.bw.flush32() - c.bw.addBits16NC(c.state, tableLog) -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go deleted file mode 100644 index 474cb77d2..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "fmt" - "math" - "sync" -) - -var ( - // fsePredef are the predefined fse tables as defined here: - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions - // These values are already transformed. - fsePredef [3]fseDecoder - - // fsePredefEnc are the predefined encoder based on fse tables as defined here: - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions - // These values are already transformed. - fsePredefEnc [3]fseEncoder - - // symbolTableX contain the transformations needed for each type as defined in - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets - symbolTableX [3][]baseOffset - - // maxTableSymbol is the biggest supported symbol for each table type - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets - maxTableSymbol = [3]uint8{tableLiteralLengths: maxLiteralLengthSymbol, tableOffsets: maxOffsetLengthSymbol, tableMatchLengths: maxMatchLengthSymbol} - - // bitTables is the bits table for each table. - bitTables = [3][]byte{tableLiteralLengths: llBitsTable[:], tableOffsets: nil, tableMatchLengths: mlBitsTable[:]} -) - -type tableIndex uint8 - -const ( - // indexes for fsePredef and symbolTableX - tableLiteralLengths tableIndex = 0 - tableOffsets tableIndex = 1 - tableMatchLengths tableIndex = 2 - - maxLiteralLengthSymbol = 35 - maxOffsetLengthSymbol = 30 - maxMatchLengthSymbol = 52 -) - -// baseOffset is used for calculating transformations. -type baseOffset struct { - baseLine uint32 - addBits uint8 -} - -// fillBase will precalculate base offsets with the given bit distributions. -func fillBase(dst []baseOffset, base uint32, bits ...uint8) { - if len(bits) != len(dst) { - panic(fmt.Sprintf("len(dst) (%d) != len(bits) (%d)", len(dst), len(bits))) - } - for i, bit := range bits { - if base > math.MaxInt32 { - panic("invalid decoding table, base overflows int32") - } - - dst[i] = baseOffset{ - baseLine: base, - addBits: bit, - } - base += 1 << bit - } -} - -var predef sync.Once - -func initPredefined() { - predef.Do(func() { - // Literals length codes - tmp := make([]baseOffset, 36) - for i := range tmp[:16] { - tmp[i] = baseOffset{ - baseLine: uint32(i), - addBits: 0, - } - } - fillBase(tmp[16:], 16, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) - symbolTableX[tableLiteralLengths] = tmp - - // Match length codes - tmp = make([]baseOffset, 53) - for i := range tmp[:32] { - tmp[i] = baseOffset{ - // The transformation adds the 3 length. - baseLine: uint32(i) + 3, - addBits: 0, - } - } - fillBase(tmp[32:], 35, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) - symbolTableX[tableMatchLengths] = tmp - - // Offset codes - tmp = make([]baseOffset, maxOffsetBits+1) - tmp[1] = baseOffset{ - baseLine: 1, - addBits: 1, - } - fillBase(tmp[2:], 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30) - symbolTableX[tableOffsets] = tmp - - // Fill predefined tables and transform them. - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions - for i := range fsePredef[:] { - f := &fsePredef[i] - switch tableIndex(i) { - case tableLiteralLengths: - // https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L243 - f.actualTableLog = 6 - copy(f.norm[:], []int16{4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, - -1, -1, -1, -1}) - f.symbolLen = 36 - case tableOffsets: - // https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L281 - f.actualTableLog = 5 - copy(f.norm[:], []int16{ - 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}) - f.symbolLen = 29 - case tableMatchLengths: - //https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L304 - f.actualTableLog = 6 - copy(f.norm[:], []int16{ - 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, - -1, -1, -1, -1, -1}) - f.symbolLen = 53 - } - if err := f.buildDtable(); err != nil { - panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) - } - if err := f.transform(symbolTableX[i]); err != nil { - panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) - } - f.preDefined = true - - // Create encoder as well - enc := &fsePredefEnc[i] - copy(enc.norm[:], f.norm[:]) - enc.symbolLen = f.symbolLen - enc.actualTableLog = f.actualTableLog - if err := enc.buildCTable(); err != nil { - panic(fmt.Errorf("building encoding table %v: %v", tableIndex(i), err)) - } - enc.setBits(bitTables[i]) - enc.preDefined = true - } - }) -} diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go deleted file mode 100644 index 5d73c21eb..000000000 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -const ( - prime3bytes = 506832829 - prime4bytes = 2654435761 - prime5bytes = 889523592379 - prime6bytes = 227718039650203 - prime7bytes = 58295818150454627 - prime8bytes = 0xcf1bbcdcb7a56463 -) - -// hashLen returns a hash of the lowest mls bytes of with length output bits. -// mls must be >=3 and <=8. Any other value will return hash for 4 bytes. -// length should always be < 32. -// Preferably length and mls should be a constant for inlining. -func hashLen(u uint64, length, mls uint8) uint32 { - switch mls { - case 3: - return (uint32(u<<8) * prime3bytes) >> (32 - length) - case 5: - return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) - case 6: - return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) - case 7: - return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) - case 8: - return uint32((u * prime8bytes) >> (64 - length)) - default: - return (uint32(u) * prime4bytes) >> (32 - length) - } -} diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go deleted file mode 100644 index 09164856d..000000000 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "github.com/klauspost/compress/huff0" -) - -// history contains the information transferred between blocks. -type history struct { - // Literal decompression - huffTree *huff0.Scratch - - // Sequence decompression - decoders sequenceDecs - recentOffsets [3]int - - // History buffer... - b []byte - - // ignoreBuffer is meant to ignore a number of bytes - // when checking for matches in history - ignoreBuffer int - - windowSize int - allocFrameBuffer int // needed? - error bool - dict *dict -} - -// reset will reset the history to initial state of a frame. -// The history must already have been initialized to the desired size. -func (h *history) reset() { - h.b = h.b[:0] - h.ignoreBuffer = 0 - h.error = false - h.recentOffsets = [3]int{1, 4, 8} - h.decoders.freeDecoders() - h.decoders = sequenceDecs{br: h.decoders.br} - h.freeHuffDecoder() - h.huffTree = nil - h.dict = nil - //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) -} - -func (h *history) freeHuffDecoder() { - if h.huffTree != nil { - if h.dict == nil || h.dict.litEnc != h.huffTree { - huffDecoderPool.Put(h.huffTree) - h.huffTree = nil - } - } -} - -func (h *history) setDict(dict *dict) { - if dict == nil { - return - } - h.dict = dict - h.decoders.litLengths = dict.llDec - h.decoders.offsets = dict.ofDec - h.decoders.matchLengths = dict.mlDec - h.decoders.dict = dict.content - h.recentOffsets = dict.offsets - h.huffTree = dict.litEnc -} - -// append bytes to history. -// This function will make sure there is space for it, -// if the buffer has been allocated with enough extra space. -func (h *history) append(b []byte) { - if len(b) >= h.windowSize { - // Discard all history by simply overwriting - h.b = h.b[:h.windowSize] - copy(h.b, b[len(b)-h.windowSize:]) - return - } - - // If there is space, append it. - if len(b) < cap(h.b)-len(h.b) { - h.b = append(h.b, b...) - return - } - - // Move data down so we only have window size left. - // We know we have less than window size in b at this point. - discard := len(b) + len(h.b) - h.windowSize - copy(h.b, h.b[discard:]) - h.b = h.b[:h.windowSize] - copy(h.b[h.windowSize-len(b):], b) -} - -// ensureBlock will ensure there is space for at least one block... -func (h *history) ensureBlock() { - if cap(h.b) < h.allocFrameBuffer { - h.b = make([]byte, 0, h.allocFrameBuffer) - return - } - - avail := cap(h.b) - len(h.b) - if avail >= h.windowSize || avail > maxCompressedBlockSize { - return - } - // Move data down so we only have window size left. - // We know we have less than window size in b at this point. - discard := len(h.b) - h.windowSize - copy(h.b, h.b[discard:]) - h.b = h.b[:h.windowSize] -} - -// append bytes to history without ever discarding anything. -func (h *history) appendKeep(b []byte) { - h.b = append(h.b, b...) -} diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt deleted file mode 100644 index 24b53065f..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2016 Caleb Spare - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md deleted file mode 100644 index 777290d44..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# xxhash - -VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package. - -xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a -high-quality hashing algorithm that is much faster than anything in the Go -standard library. - -This package provides a straightforward API: - -``` -func Sum64(b []byte) uint64 -func Sum64String(s string) uint64 -type Digest struct{ ... } - func New() *Digest -``` - -The `Digest` type implements hash.Hash64. Its key methods are: - -``` -func (*Digest) Write([]byte) (int, error) -func (*Digest) WriteString(string) (int, error) -func (*Digest) Sum64() uint64 -``` - -The package is written with optimized pure Go and also contains even faster -assembly implementations for amd64 and arm64. If desired, the `purego` build tag -opts into using the Go code even on those architectures. - -[xxHash]: http://cyan4973.github.io/xxHash/ - -## Compatibility - -This package is in a module and the latest code is in version 2 of the module. -You need a version of Go with at least "minimal module compatibility" to use -github.com/cespare/xxhash/v2: - -* 1.9.7+ for Go 1.9 -* 1.10.3+ for Go 1.10 -* Go 1.11 or later - -I recommend using the latest release of Go. - -## Benchmarks - -Here are some quick benchmarks comparing the pure-Go and assembly -implementations of Sum64. - -| input size | purego | asm | -| ---------- | --------- | --------- | -| 4 B | 1.3 GB/s | 1.2 GB/s | -| 16 B | 2.9 GB/s | 3.5 GB/s | -| 100 B | 6.9 GB/s | 8.1 GB/s | -| 4 KB | 11.7 GB/s | 16.7 GB/s | -| 10 MB | 12.0 GB/s | 17.3 GB/s | - -These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C -CPU using the following commands under Go 1.19.2: - -``` -benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') -benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') -``` - -## Projects using this package - -- [InfluxDB](https://github.com/influxdata/influxdb) -- [Prometheus](https://github.com/prometheus/prometheus) -- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) -- [FreeCache](https://github.com/coocood/freecache) -- [FastCache](https://github.com/VictoriaMetrics/fastcache) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go deleted file mode 100644 index fc40c8200..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ /dev/null @@ -1,230 +0,0 @@ -// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described -// at http://cyan4973.github.io/xxHash/. -// THIS IS VENDORED: Go to github.com/cespare/xxhash for original package. - -package xxhash - -import ( - "encoding/binary" - "errors" - "math/bits" -) - -const ( - prime1 uint64 = 11400714785074694791 - prime2 uint64 = 14029467366897019727 - prime3 uint64 = 1609587929392839161 - prime4 uint64 = 9650029242287828579 - prime5 uint64 = 2870177450012600261 -) - -// Store the primes in an array as well. -// -// The consts are used when possible in Go code to avoid MOVs but we need a -// contiguous array of the assembly code. -var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5} - -// Digest implements hash.Hash64. -type Digest struct { - v1 uint64 - v2 uint64 - v3 uint64 - v4 uint64 - total uint64 - mem [32]byte - n int // how much of mem is used -} - -// New creates a new Digest that computes the 64-bit xxHash algorithm. -func New() *Digest { - var d Digest - d.Reset() - return &d -} - -// Reset clears the Digest's state so that it can be reused. -func (d *Digest) Reset() { - d.v1 = primes[0] + prime2 - d.v2 = prime2 - d.v3 = 0 - d.v4 = -primes[0] - d.total = 0 - d.n = 0 -} - -// Size always returns 8 bytes. -func (d *Digest) Size() int { return 8 } - -// BlockSize always returns 32 bytes. -func (d *Digest) BlockSize() int { return 32 } - -// Write adds more data to d. It always returns len(b), nil. -func (d *Digest) Write(b []byte) (n int, err error) { - n = len(b) - d.total += uint64(n) - - memleft := d.mem[d.n&(len(d.mem)-1):] - - if d.n+n < 32 { - // This new data doesn't even fill the current block. - copy(memleft, b) - d.n += n - return - } - - if d.n > 0 { - // Finish off the partial block. - c := copy(memleft, b) - d.v1 = round(d.v1, u64(d.mem[0:8])) - d.v2 = round(d.v2, u64(d.mem[8:16])) - d.v3 = round(d.v3, u64(d.mem[16:24])) - d.v4 = round(d.v4, u64(d.mem[24:32])) - b = b[c:] - d.n = 0 - } - - if len(b) >= 32 { - // One or more full blocks left. - nw := writeBlocks(d, b) - b = b[nw:] - } - - // Store any remaining partial block. - copy(d.mem[:], b) - d.n = len(b) - - return -} - -// Sum appends the current hash to b and returns the resulting slice. -func (d *Digest) Sum(b []byte) []byte { - s := d.Sum64() - return append( - b, - byte(s>>56), - byte(s>>48), - byte(s>>40), - byte(s>>32), - byte(s>>24), - byte(s>>16), - byte(s>>8), - byte(s), - ) -} - -// Sum64 returns the current hash. -func (d *Digest) Sum64() uint64 { - var h uint64 - - if d.total >= 32 { - v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 - h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) - h = mergeRound(h, v1) - h = mergeRound(h, v2) - h = mergeRound(h, v3) - h = mergeRound(h, v4) - } else { - h = d.v3 + prime5 - } - - h += d.total - - b := d.mem[:d.n&(len(d.mem)-1)] - for ; len(b) >= 8; b = b[8:] { - k1 := round(0, u64(b[:8])) - h ^= k1 - h = rol27(h)*prime1 + prime4 - } - if len(b) >= 4 { - h ^= uint64(u32(b[:4])) * prime1 - h = rol23(h)*prime2 + prime3 - b = b[4:] - } - for ; len(b) > 0; b = b[1:] { - h ^= uint64(b[0]) * prime5 - h = rol11(h) * prime1 - } - - h ^= h >> 33 - h *= prime2 - h ^= h >> 29 - h *= prime3 - h ^= h >> 32 - - return h -} - -const ( - magic = "xxh\x06" - marshaledSize = len(magic) + 8*5 + 32 -) - -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (d *Digest) MarshalBinary() ([]byte, error) { - b := make([]byte, 0, marshaledSize) - b = append(b, magic...) - b = appendUint64(b, d.v1) - b = appendUint64(b, d.v2) - b = appendUint64(b, d.v3) - b = appendUint64(b, d.v4) - b = appendUint64(b, d.total) - b = append(b, d.mem[:d.n]...) - b = b[:len(b)+len(d.mem)-d.n] - return b, nil -} - -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. -func (d *Digest) UnmarshalBinary(b []byte) error { - if len(b) < len(magic) || string(b[:len(magic)]) != magic { - return errors.New("xxhash: invalid hash state identifier") - } - if len(b) != marshaledSize { - return errors.New("xxhash: invalid hash state size") - } - b = b[len(magic):] - b, d.v1 = consumeUint64(b) - b, d.v2 = consumeUint64(b) - b, d.v3 = consumeUint64(b) - b, d.v4 = consumeUint64(b) - b, d.total = consumeUint64(b) - copy(d.mem[:], b) - d.n = int(d.total % uint64(len(d.mem))) - return nil -} - -func appendUint64(b []byte, x uint64) []byte { - var a [8]byte - binary.LittleEndian.PutUint64(a[:], x) - return append(b, a[:]...) -} - -func consumeUint64(b []byte) ([]byte, uint64) { - x := u64(b) - return b[8:], x -} - -func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } -func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } - -func round(acc, input uint64) uint64 { - acc += input * prime2 - acc = rol31(acc) - acc *= prime1 - return acc -} - -func mergeRound(acc, val uint64) uint64 { - val = round(0, val) - acc ^= val - acc = acc*prime1 + prime4 - return acc -} - -func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } -func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } -func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } -func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } -func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } -func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } -func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } -func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s deleted file mode 100644 index ddb63aa91..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ /dev/null @@ -1,210 +0,0 @@ -//go:build !appengine && gc && !purego && !noasm -// +build !appengine -// +build gc -// +build !purego -// +build !noasm - -#include "textflag.h" - -// Registers: -#define h AX -#define d AX -#define p SI // pointer to advance through b -#define n DX -#define end BX // loop end -#define v1 R8 -#define v2 R9 -#define v3 R10 -#define v4 R11 -#define x R12 -#define prime1 R13 -#define prime2 R14 -#define prime4 DI - -#define round(acc, x) \ - IMULQ prime2, x \ - ADDQ x, acc \ - ROLQ $31, acc \ - IMULQ prime1, acc - -// round0 performs the operation x = round(0, x). -#define round0(x) \ - IMULQ prime2, x \ - ROLQ $31, x \ - IMULQ prime1, x - -// mergeRound applies a merge round on the two registers acc and x. -// It assumes that prime1, prime2, and prime4 have been loaded. -#define mergeRound(acc, x) \ - round0(x) \ - XORQ x, acc \ - IMULQ prime1, acc \ - ADDQ prime4, acc - -// blockLoop processes as many 32-byte blocks as possible, -// updating v1, v2, v3, and v4. It assumes that there is at least one block -// to process. -#define blockLoop() \ -loop: \ - MOVQ +0(p), x \ - round(v1, x) \ - MOVQ +8(p), x \ - round(v2, x) \ - MOVQ +16(p), x \ - round(v3, x) \ - MOVQ +24(p), x \ - round(v4, x) \ - ADDQ $32, p \ - CMPQ p, end \ - JLE loop - -// func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 - // Load fixed primes. - MOVQ ·primes+0(SB), prime1 - MOVQ ·primes+8(SB), prime2 - MOVQ ·primes+24(SB), prime4 - - // Load slice. - MOVQ b_base+0(FP), p - MOVQ b_len+8(FP), n - LEAQ (p)(n*1), end - - // The first loop limit will be len(b)-32. - SUBQ $32, end - - // Check whether we have at least one block. - CMPQ n, $32 - JLT noBlocks - - // Set up initial state (v1, v2, v3, v4). - MOVQ prime1, v1 - ADDQ prime2, v1 - MOVQ prime2, v2 - XORQ v3, v3 - XORQ v4, v4 - SUBQ prime1, v4 - - blockLoop() - - MOVQ v1, h - ROLQ $1, h - MOVQ v2, x - ROLQ $7, x - ADDQ x, h - MOVQ v3, x - ROLQ $12, x - ADDQ x, h - MOVQ v4, x - ROLQ $18, x - ADDQ x, h - - mergeRound(h, v1) - mergeRound(h, v2) - mergeRound(h, v3) - mergeRound(h, v4) - - JMP afterBlocks - -noBlocks: - MOVQ ·primes+32(SB), h - -afterBlocks: - ADDQ n, h - - ADDQ $24, end - CMPQ p, end - JG try4 - -loop8: - MOVQ (p), x - ADDQ $8, p - round0(x) - XORQ x, h - ROLQ $27, h - IMULQ prime1, h - ADDQ prime4, h - - CMPQ p, end - JLE loop8 - -try4: - ADDQ $4, end - CMPQ p, end - JG try1 - - MOVL (p), x - ADDQ $4, p - IMULQ prime1, x - XORQ x, h - - ROLQ $23, h - IMULQ prime2, h - ADDQ ·primes+16(SB), h - -try1: - ADDQ $4, end - CMPQ p, end - JGE finalize - -loop1: - MOVBQZX (p), x - ADDQ $1, p - IMULQ ·primes+32(SB), x - XORQ x, h - ROLQ $11, h - IMULQ prime1, h - - CMPQ p, end - JL loop1 - -finalize: - MOVQ h, x - SHRQ $33, x - XORQ x, h - IMULQ prime2, h - MOVQ h, x - SHRQ $29, x - XORQ x, h - IMULQ ·primes+16(SB), h - MOVQ h, x - SHRQ $32, x - XORQ x, h - - MOVQ h, ret+24(FP) - RET - -// func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 - // Load fixed primes needed for round. - MOVQ ·primes+0(SB), prime1 - MOVQ ·primes+8(SB), prime2 - - // Load slice. - MOVQ b_base+8(FP), p - MOVQ b_len+16(FP), n - LEAQ (p)(n*1), end - SUBQ $32, end - - // Load vN from d. - MOVQ s+0(FP), d - MOVQ 0(d), v1 - MOVQ 8(d), v2 - MOVQ 16(d), v3 - MOVQ 24(d), v4 - - // We don't need to check the loop condition here; this function is - // always called with at least one block of data to process. - blockLoop() - - // Copy vN back to d. - MOVQ v1, 0(d) - MOVQ v2, 8(d) - MOVQ v3, 16(d) - MOVQ v4, 24(d) - - // The number of bytes written is p minus the old base pointer. - SUBQ b_base+8(FP), p - MOVQ p, ret+32(FP) - - RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s deleted file mode 100644 index ae7d4d329..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ /dev/null @@ -1,184 +0,0 @@ -//go:build !appengine && gc && !purego && !noasm -// +build !appengine -// +build gc -// +build !purego -// +build !noasm - -#include "textflag.h" - -// Registers: -#define digest R1 -#define h R2 // return value -#define p R3 // input pointer -#define n R4 // input length -#define nblocks R5 // n / 32 -#define prime1 R7 -#define prime2 R8 -#define prime3 R9 -#define prime4 R10 -#define prime5 R11 -#define v1 R12 -#define v2 R13 -#define v3 R14 -#define v4 R15 -#define x1 R20 -#define x2 R21 -#define x3 R22 -#define x4 R23 - -#define round(acc, x) \ - MADD prime2, acc, x, acc \ - ROR $64-31, acc \ - MUL prime1, acc - -// round0 performs the operation x = round(0, x). -#define round0(x) \ - MUL prime2, x \ - ROR $64-31, x \ - MUL prime1, x - -#define mergeRound(acc, x) \ - round0(x) \ - EOR x, acc \ - MADD acc, prime4, prime1, acc - -// blockLoop processes as many 32-byte blocks as possible, -// updating v1, v2, v3, and v4. It assumes that n >= 32. -#define blockLoop() \ - LSR $5, n, nblocks \ - PCALIGN $16 \ - loop: \ - LDP.P 16(p), (x1, x2) \ - LDP.P 16(p), (x3, x4) \ - round(v1, x1) \ - round(v2, x2) \ - round(v3, x3) \ - round(v4, x4) \ - SUB $1, nblocks \ - CBNZ nblocks, loop - -// func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 - LDP b_base+0(FP), (p, n) - - LDP ·primes+0(SB), (prime1, prime2) - LDP ·primes+16(SB), (prime3, prime4) - MOVD ·primes+32(SB), prime5 - - CMP $32, n - CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 } - BLT afterLoop - - ADD prime1, prime2, v1 - MOVD prime2, v2 - MOVD $0, v3 - NEG prime1, v4 - - blockLoop() - - ROR $64-1, v1, x1 - ROR $64-7, v2, x2 - ADD x1, x2 - ROR $64-12, v3, x3 - ROR $64-18, v4, x4 - ADD x3, x4 - ADD x2, x4, h - - mergeRound(h, v1) - mergeRound(h, v2) - mergeRound(h, v3) - mergeRound(h, v4) - -afterLoop: - ADD n, h - - TBZ $4, n, try8 - LDP.P 16(p), (x1, x2) - - round0(x1) - - // NOTE: here and below, sequencing the EOR after the ROR (using a - // rotated register) is worth a small but measurable speedup for small - // inputs. - ROR $64-27, h - EOR x1 @> 64-27, h, h - MADD h, prime4, prime1, h - - round0(x2) - ROR $64-27, h - EOR x2 @> 64-27, h, h - MADD h, prime4, prime1, h - -try8: - TBZ $3, n, try4 - MOVD.P 8(p), x1 - - round0(x1) - ROR $64-27, h - EOR x1 @> 64-27, h, h - MADD h, prime4, prime1, h - -try4: - TBZ $2, n, try2 - MOVWU.P 4(p), x2 - - MUL prime1, x2 - ROR $64-23, h - EOR x2 @> 64-23, h, h - MADD h, prime3, prime2, h - -try2: - TBZ $1, n, try1 - MOVHU.P 2(p), x3 - AND $255, x3, x1 - LSR $8, x3, x2 - - MUL prime5, x1 - ROR $64-11, h - EOR x1 @> 64-11, h, h - MUL prime1, h - - MUL prime5, x2 - ROR $64-11, h - EOR x2 @> 64-11, h, h - MUL prime1, h - -try1: - TBZ $0, n, finalize - MOVBU (p), x4 - - MUL prime5, x4 - ROR $64-11, h - EOR x4 @> 64-11, h, h - MUL prime1, h - -finalize: - EOR h >> 33, h - MUL prime2, h - EOR h >> 29, h - MUL prime3, h - EOR h >> 32, h - - MOVD h, ret+24(FP) - RET - -// func writeBlocks(s *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 - LDP ·primes+0(SB), (prime1, prime2) - - // Load state. Assume v[1-4] are stored contiguously. - MOVD s+0(FP), digest - LDP 0(digest), (v1, v2) - LDP 16(digest), (v3, v4) - - LDP b_base+8(FP), (p, n) - - blockLoop() - - // Store updated state. - STP (v1, v2), 0(digest) - STP (v3, v4), 16(digest) - - BIC $31, n - MOVD n, ret+32(FP) - RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go deleted file mode 100644 index d4221edf4..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build (amd64 || arm64) && !appengine && gc && !purego && !noasm -// +build amd64 arm64 -// +build !appengine -// +build gc -// +build !purego -// +build !noasm - -package xxhash - -// Sum64 computes the 64-bit xxHash digest of b. -// -//go:noescape -func Sum64(b []byte) uint64 - -//go:noescape -func writeBlocks(s *Digest, b []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go deleted file mode 100644 index 0be16cefc..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm -// +build !amd64,!arm64 appengine !gc purego noasm - -package xxhash - -// Sum64 computes the 64-bit xxHash digest of b. -func Sum64(b []byte) uint64 { - // A simpler version would be - // d := New() - // d.Write(b) - // return d.Sum64() - // but this is faster, particularly for small inputs. - - n := len(b) - var h uint64 - - if n >= 32 { - v1 := primes[0] + prime2 - v2 := prime2 - v3 := uint64(0) - v4 := -primes[0] - for len(b) >= 32 { - v1 = round(v1, u64(b[0:8:len(b)])) - v2 = round(v2, u64(b[8:16:len(b)])) - v3 = round(v3, u64(b[16:24:len(b)])) - v4 = round(v4, u64(b[24:32:len(b)])) - b = b[32:len(b):len(b)] - } - h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) - h = mergeRound(h, v1) - h = mergeRound(h, v2) - h = mergeRound(h, v3) - h = mergeRound(h, v4) - } else { - h = prime5 - } - - h += uint64(n) - - for ; len(b) >= 8; b = b[8:] { - k1 := round(0, u64(b[:8])) - h ^= k1 - h = rol27(h)*prime1 + prime4 - } - if len(b) >= 4 { - h ^= uint64(u32(b[:4])) * prime1 - h = rol23(h)*prime2 + prime3 - b = b[4:] - } - for ; len(b) > 0; b = b[1:] { - h ^= uint64(b[0]) * prime5 - h = rol11(h) * prime1 - } - - h ^= h >> 33 - h *= prime2 - h ^= h >> 29 - h *= prime3 - h ^= h >> 32 - - return h -} - -func writeBlocks(d *Digest, b []byte) int { - v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 - n := len(b) - for len(b) >= 32 { - v1 = round(v1, u64(b[0:8:len(b)])) - v2 = round(v2, u64(b[8:16:len(b)])) - v3 = round(v3, u64(b[16:24:len(b)])) - v4 = round(v4, u64(b[24:32:len(b)])) - b = b[32:len(b):len(b)] - } - d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4 - return n - len(b) -} diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go deleted file mode 100644 index 6f3b0cb10..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go +++ /dev/null @@ -1,11 +0,0 @@ -package xxhash - -// Sum64String computes the 64-bit xxHash digest of s. -func Sum64String(s string) uint64 { - return Sum64([]byte(s)) -} - -// WriteString adds more data to d. It always returns len(s), nil. -func (d *Digest) WriteString(s string) (n int, err error) { - return d.Write([]byte(s)) -} diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go deleted file mode 100644 index f41932b7a..000000000 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package zstd - -// matchLen returns how many bytes match in a and b -// -// It assumes that: -// -// len(a) <= len(b) and len(a) > 0 -// -//go:noescape -func matchLen(a []byte, b []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s deleted file mode 100644 index 0782b86e3..000000000 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +++ /dev/null @@ -1,66 +0,0 @@ -// Copied from S2 implementation. - -//go:build !appengine && !noasm && gc && !noasm - -#include "textflag.h" - -// func matchLen(a []byte, b []byte) int -TEXT ·matchLen(SB), NOSPLIT, $0-56 - MOVQ a_base+0(FP), AX - MOVQ b_base+24(FP), CX - MOVQ a_len+8(FP), DX - - // matchLen - XORL SI, SI - CMPL DX, $0x08 - JB matchlen_match4_standalone - -matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - JZ matchlen_loop_standalone - -#ifdef GOAMD64_v3 - TZCNTQ BX, BX -#else - BSFQ BX, BX -#endif - SHRL $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end - -matchlen_loop_standalone: - LEAL -8(DX), DX - LEAL 8(SI), SI - CMPL DX, $0x08 - JAE matchlen_loopback_standalone - -matchlen_match4_standalone: - CMPL DX, $0x04 - JB matchlen_match2_standalone - MOVL (AX)(SI*1), BX - CMPL (CX)(SI*1), BX - JNE matchlen_match2_standalone - LEAL -4(DX), DX - LEAL 4(SI), SI - -matchlen_match2_standalone: - CMPL DX, $0x02 - JB matchlen_match1_standalone - MOVW (AX)(SI*1), BX - CMPW (CX)(SI*1), BX - JNE matchlen_match1_standalone - LEAL -2(DX), DX - LEAL 2(SI), SI - -matchlen_match1_standalone: - CMPL DX, $0x01 - JB gen_match_len_end - MOVB (AX)(SI*1), BL - CMPB (CX)(SI*1), BL - JNE gen_match_len_end - INCL SI - -gen_match_len_end: - MOVQ SI, ret+48(FP) - RET diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go deleted file mode 100644 index 57b9c31c0..000000000 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go +++ /dev/null @@ -1,33 +0,0 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package zstd - -import ( - "encoding/binary" - "math/bits" -) - -// matchLen returns the maximum common prefix length of a and b. -// a must be the shortest of the two. -func matchLen(a, b []byte) (n int) { - for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] { - diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b) - if diff != 0 { - return n + bits.TrailingZeros64(diff)>>3 - } - n += 8 - } - - for i := range a { - if a[i] != b[i] { - break - } - n++ - } - return n - -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go deleted file mode 100644 index d7fe6d82d..000000000 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ /dev/null @@ -1,503 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "errors" - "fmt" - "io" -) - -type seq struct { - litLen uint32 - matchLen uint32 - offset uint32 - - // Codes are stored here for the encoder - // so they only have to be looked up once. - llCode, mlCode, ofCode uint8 -} - -type seqVals struct { - ll, ml, mo int -} - -func (s seq) String() string { - if s.offset <= 3 { - if s.offset == 0 { - return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset: INVALID (0)") - } - return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset:", s.offset, " (repeat)") - } - return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset:", s.offset-3, " (new)") -} - -type seqCompMode uint8 - -const ( - compModePredefined seqCompMode = iota - compModeRLE - compModeFSE - compModeRepeat -) - -type sequenceDec struct { - // decoder keeps track of the current state and updates it from the bitstream. - fse *fseDecoder - state fseState - repeat bool -} - -// init the state of the decoder with input from stream. -func (s *sequenceDec) init(br *bitReader) error { - if s.fse == nil { - return errors.New("sequence decoder not defined") - } - s.state.init(br, s.fse.actualTableLog, s.fse.dt[:1<<s.fse.actualTableLog]) - return nil -} - -// sequenceDecs contains all 3 sequence decoders and their state. -type sequenceDecs struct { - litLengths sequenceDec - offsets sequenceDec - matchLengths sequenceDec - prevOffset [3]int - dict []byte - literals []byte - out []byte - nSeqs int - br *bitReader - seqSize int - windowSize int - maxBits uint8 - maxSyncLen uint64 -} - -// initialize all 3 decoders from the stream input. -func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) error { - if err := s.litLengths.init(br); err != nil { - return errors.New("litLengths:" + err.Error()) - } - if err := s.offsets.init(br); err != nil { - return errors.New("offsets:" + err.Error()) - } - if err := s.matchLengths.init(br); err != nil { - return errors.New("matchLengths:" + err.Error()) - } - s.br = br - s.prevOffset = hist.recentOffsets - s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits - s.windowSize = hist.windowSize - s.out = out - s.dict = nil - if hist.dict != nil { - s.dict = hist.dict.content - } - return nil -} - -func (s *sequenceDecs) freeDecoders() { - if f := s.litLengths.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - s.litLengths.fse = nil - } - if f := s.offsets.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - s.offsets.fse = nil - } - if f := s.matchLengths.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - s.matchLengths.fse = nil - } -} - -// execute will execute the decoded sequence with the provided history. -// The sequence must be evaluated before being sent. -func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error { - if len(s.dict) == 0 { - return s.executeSimple(seqs, hist) - } - - // Ensure we have enough output size... - if len(s.out)+s.seqSize > cap(s.out) { - addBytes := s.seqSize + len(s.out) - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] - } - - if debugDecoder { - printf("Execute %d seqs with hist %d, dict %d, literals: %d into %d bytes\n", len(seqs), len(hist), len(s.dict), len(s.literals), s.seqSize) - } - - var t = len(s.out) - out := s.out[:t+s.seqSize] - - for _, seq := range seqs { - // Add literals - copy(out[t:], s.literals[:seq.ll]) - t += seq.ll - s.literals = s.literals[seq.ll:] - - // Copy from dictionary... - if seq.mo > t+len(hist) || seq.mo > s.windowSize { - if len(s.dict) == 0 { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist)) - } - - // we may be in dictionary. - dictO := len(s.dict) - (seq.mo - (t + len(hist))) - if dictO < 0 || dictO >= len(s.dict) { - return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", seq.mo, t+len(hist)+len(s.dict)) - } - end := dictO + seq.ml - if end > len(s.dict) { - n := len(s.dict) - dictO - copy(out[t:], s.dict[dictO:]) - t += n - seq.ml -= n - } else { - copy(out[t:], s.dict[dictO:end]) - t += end - dictO - continue - } - } - - // Copy from history. - if v := seq.mo - t; v > 0 { - // v is the start position in history from end. - start := len(hist) - v - if seq.ml > v { - // Some goes into current block. - // Copy remainder of history - copy(out[t:], hist[start:]) - t += v - seq.ml -= v - } else { - copy(out[t:], hist[start:start+seq.ml]) - t += seq.ml - continue - } - } - // We must be in current buffer now - if seq.ml > 0 { - start := t - seq.mo - if seq.ml <= t-start { - // No overlap - copy(out[t:], out[start:start+seq.ml]) - t += seq.ml - continue - } else { - // Overlapping copy - // Extend destination slice and copy one byte at the time. - src := out[start : start+seq.ml] - dst := out[t:] - dst = dst[:len(src)] - t += len(src) - // Destination is the space we just added. - for i := range src { - dst[i] = src[i] - } - } - } - } - - // Add final literals - copy(out[t:], s.literals) - if debugDecoder { - t += len(s.literals) - if t != len(out) { - panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) - } - } - s.out = out - - return nil -} - -// decode sequences from the stream with the provided history. -func (s *sequenceDecs) decodeSync(hist []byte) error { - supported, err := s.decodeSyncSimple(hist) - if supported { - return err - } - - br := s.br - seqs := s.nSeqs - startSize := len(s.out) - // Grab full sizes tables, to avoid bounds checks. - llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] - llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state - out := s.out - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } - - if debugDecoder { - println("decodeSync: decoding", seqs, "sequences", br.remain(), "bits remain on stream") - } - for i := seqs - 1; i >= 0; i-- { - if br.overread() { - printf("reading sequence %d, exceeded available data. Overread by %d\n", seqs-i, -br.remain()) - return io.ErrUnexpectedEOF - } - var ll, mo, ml int - if len(br.in) > 4+((maxOffsetBits+16+16)>>3) { - // inlined function: - // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) - - // Final will not read from stream. - var llB, mlB, moB uint8 - ll, llB = llState.final() - ml, mlB = mlState.final() - mo, moB = ofState.final() - - // extra bits are stored in reverse order. - br.fillFast() - mo += br.getBits(moB) - if s.maxBits > 32 { - br.fillFast() - } - ml += br.getBits(mlB) - ll += br.getBits(llB) - - if moB > 1 { - s.prevOffset[2] = s.prevOffset[1] - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = mo - } else { - // mo = s.adjustOffset(mo, ll, moB) - // Inlined for rather big speedup - if ll == 0 { - // There is an exception though, when current sequence's literals_length = 0. - // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, - // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. - mo++ - } - - if mo == 0 { - mo = s.prevOffset[0] - } else { - var temp int - if mo == 3 { - temp = s.prevOffset[0] - 1 - } else { - temp = s.prevOffset[mo] - } - - if temp == 0 { - // 0 is not valid; input is corrupted; force offset to 1 - println("WARNING: temp was 0") - temp = 1 - } - - if mo != 1 { - s.prevOffset[2] = s.prevOffset[1] - } - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = temp - mo = temp - } - } - br.fillFast() - } else { - ll, mo, ml = s.next(br, llState, mlState, ofState) - br.fill() - } - - if debugSequences { - println("Seq", seqs-i-1, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml) - } - - if ll > len(s.literals) { - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals)) - } - size := ll + ml + len(out) - if size-startSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - if size > cap(out) { - // Not enough size, which can happen under high volume block streaming conditions - // but could be if destination slice is too small for sync operations. - // over-allocating here can create a large amount of GC pressure so we try to keep - // it as contained as possible - used := len(out) - startSize - addBytes := 256 + ll + ml + used>>2 - // Clamp to max block size. - if used+addBytes > maxBlockSize { - addBytes = maxBlockSize - used - } - out = append(out, make([]byte, addBytes)...) - out = out[:len(out)-addBytes] - } - if ml > maxMatchLen { - return fmt.Errorf("match len (%d) bigger than max allowed length", ml) - } - - // Add literals - out = append(out, s.literals[:ll]...) - s.literals = s.literals[ll:] - - if mo == 0 && ml > 0 { - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - } - - if mo > len(out)+len(hist) || mo > s.windowSize { - if len(s.dict) == 0 { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) - } - - // we may be in dictionary. - dictO := len(s.dict) - (mo - (len(out) + len(hist))) - if dictO < 0 || dictO >= len(s.dict) { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) - } - end := dictO + ml - if end > len(s.dict) { - out = append(out, s.dict[dictO:]...) - ml -= len(s.dict) - dictO - } else { - out = append(out, s.dict[dictO:end]...) - mo = 0 - ml = 0 - } - } - - // Copy from history. - // TODO: Blocks without history could be made to ignore this completely. - if v := mo - len(out); v > 0 { - // v is the start position in history from end. - start := len(hist) - v - if ml > v { - // Some goes into current block. - // Copy remainder of history - out = append(out, hist[start:]...) - ml -= v - } else { - out = append(out, hist[start:start+ml]...) - ml = 0 - } - } - // We must be in current buffer now - if ml > 0 { - start := len(out) - mo - if ml <= len(out)-start { - // No overlap - out = append(out, out[start:start+ml]...) - } else { - // Overlapping copy - // Extend destination slice and copy one byte at the time. - out = out[:len(out)+ml] - src := out[start : start+ml] - // Destination is the space we just added. - dst := out[len(out)-ml:] - dst = dst[:len(src)] - for i := range src { - dst[i] = src[i] - } - } - } - if i == 0 { - // This is the last sequence, so we shouldn't update state. - break - } - - // Manually inlined, ~ 5-20% faster - // Update all 3 states at once. Approx 20% faster. - nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits() - if nBits == 0 { - llState = llTable[llState.newState()&maxTableMask] - mlState = mlTable[mlState.newState()&maxTableMask] - ofState = ofTable[ofState.newState()&maxTableMask] - } else { - bits := br.get32BitsFast(nBits) - - lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) - llState = llTable[(llState.newState()+lowBits)&maxTableMask] - - lowBits = uint16(bits >> (ofState.nbBits() & 31)) - lowBits &= bitMask[mlState.nbBits()&15] - mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask] - - lowBits = uint16(bits) & bitMask[ofState.nbBits()&15] - ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask] - } - } - - if size := len(s.literals) + len(out) - startSize; size > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - - // Add final literals - s.out = append(out, s.literals...) - return br.close() -} - -var bitMask [16]uint16 - -func init() { - for i := range bitMask[:] { - bitMask[i] = uint16((1 << uint(i)) - 1) - } -} - -func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { - // Final will not read from stream. - ll, llB := llState.final() - ml, mlB := mlState.final() - mo, moB := ofState.final() - - // extra bits are stored in reverse order. - br.fill() - mo += br.getBits(moB) - if s.maxBits > 32 { - br.fill() - } - // matchlength+literal length, max 32 bits - ml += br.getBits(mlB) - ll += br.getBits(llB) - mo = s.adjustOffset(mo, ll, moB) - return -} - -func (s *sequenceDecs) adjustOffset(offset, litLen int, offsetB uint8) int { - if offsetB > 1 { - s.prevOffset[2] = s.prevOffset[1] - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = offset - return offset - } - - if litLen == 0 { - // There is an exception though, when current sequence's literals_length = 0. - // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, - // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. - offset++ - } - - if offset == 0 { - return s.prevOffset[0] - } - var temp int - if offset == 3 { - temp = s.prevOffset[0] - 1 - } else { - temp = s.prevOffset[offset] - } - - if temp == 0 { - // 0 is not valid; input is corrupted; force offset to 1 - println("temp was 0") - temp = 1 - } - - if offset != 1 { - s.prevOffset[2] = s.prevOffset[1] - } - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = temp - return temp -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go deleted file mode 100644 index c59f17e07..000000000 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ /dev/null @@ -1,394 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -package zstd - -import ( - "fmt" - "io" - - "github.com/klauspost/compress/internal/cpuinfo" -) - -type decodeSyncAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - litRemain int - out []byte - outPosition int - literals []byte - litPosition int - history []byte - windowSize int - ll int // set on error (not for all errors, please refer to _generate/gen.go) - ml int // set on error (not for all errors, please refer to _generate/gen.go) - mo int // set on error (not for all errors, please refer to _generate/gen.go) -} - -// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. -// -// Please refer to seqdec_generic.go for the reference implementation. -// -//go:noescape -func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. -// -//go:noescape -func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. -// -//go:noescape -func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. -// -//go:noescape -func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// decode sequences from the stream with the provided history but without a dictionary. -func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { - if len(s.dict) > 0 { - return false, nil - } - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { - return false, nil - } - - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true - /* - useSafe := false - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { - useSafe = true - } - if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { - useSafe = true - } - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - useSafe = true - } - */ - - br := s.br - - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } - - ctx := decodeSyncAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - iteration: s.nSeqs - 1, - litRemain: len(s.literals), - out: s.out, - outPosition: len(s.out), - literals: s.literals, - windowSize: s.windowSize, - history: hist, - } - - s.seqSize = 0 - startSize := len(s.out) - - var errCode int - if cpuinfo.HasBMI2() { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_bmi2(s, br, &ctx) - } - } else { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_amd64(s, br, &ctx) - } - } - switch errCode { - case noError: - break - - case errorMatchLenOfsMismatch: - return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml) - - case errorMatchLenTooBig: - return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml) - - case errorMatchOffTooBig: - return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", - ctx.mo, ctx.outPosition+len(hist)-startSize) - - case errorNotEnoughLiterals: - return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", - ctx.ll, ctx.litRemain+ctx.ll) - - case errorOverread: - return true, io.ErrUnexpectedEOF - - case errorNotEnoughSpace: - size := ctx.outPosition + ctx.ll + ctx.ml - if debugDecoder { - println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize) - } - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - - default: - return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) - } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - return true, err - } - - s.literals = s.literals[ctx.litPosition:] - t := ctx.outPosition - s.out = s.out[:t] - - // Add final literals - s.out = append(s.out, s.literals...) - if debugDecoder { - t += len(s.literals) - if t != len(s.out) { - panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t)) - } - } - - return true, nil -} - -// -------------------------------------------------------------------------------- - -type decodeAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - seqs []seqVals - litRemain int -} - -const noError = 0 - -// error reported when mo == 0 && ml > 0 -const errorMatchLenOfsMismatch = 1 - -// error reported when ml > maxMatchLen -const errorMatchLenTooBig = 2 - -// error reported when mo > available history or mo > s.windowSize -const errorMatchOffTooBig = 3 - -// error reported when the sum of literal lengths exeeceds the literal buffer size -const errorNotEnoughLiterals = 4 - -// error reported when capacity of `out` is too small -const errorNotEnoughSpace = 5 - -// error reported when bits are overread. -const errorOverread = 6 - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. -// -// Please refer to seqdec_generic.go for the reference implementation. -// -//go:noescape -func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. -// -// Please refer to seqdec_generic.go for the reference implementation. -// -//go:noescape -func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. -// -//go:noescape -func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. -// -//go:noescape -func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// decode sequences from the stream without the provided history. -func (s *sequenceDecs) decode(seqs []seqVals) error { - br := s.br - - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } - - ctx := decodeAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - seqs: seqs, - iteration: len(seqs) - 1, - litRemain: len(s.literals), - } - - if debugDecoder { - println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream") - } - - s.seqSize = 0 - lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 - var errCode int - if cpuinfo.HasBMI2() { - if lte56bits { - errCode = sequenceDecs_decode_56_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_bmi2(s, br, &ctx) - } - } else { - if lte56bits { - errCode = sequenceDecs_decode_56_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_amd64(s, br, &ctx) - } - } - if errCode != 0 { - i := len(seqs) - ctx.iteration - 1 - switch errCode { - case errorMatchLenOfsMismatch: - ml := ctx.seqs[i].ml - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - - case errorMatchLenTooBig: - ml := ctx.seqs[i].ml - return fmt.Errorf("match len (%d) bigger than max allowed length", ml) - - case errorNotEnoughLiterals: - ll := ctx.seqs[i].ll - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) - case errorOverread: - return io.ErrUnexpectedEOF - } - - return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) - } - - if ctx.litRemain < 0 { - return fmt.Errorf("literal count is too big: total available %d, total requested %d", - len(s.literals), len(s.literals)-ctx.litRemain) - } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - if debugDecoder { - println("decode: ", br.remain(), "bits remain on stream. code:", errCode) - } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - } - return err -} - -// -------------------------------------------------------------------------------- - -type executeAsmContext struct { - seqs []seqVals - seqIndex int - out []byte - history []byte - literals []byte - outPosition int - litPosition int - windowSize int -} - -// sequenceDecs_executeSimple_amd64 implements the main loop of sequenceDecs.executeSimple in x86 asm. -// -// Returns false if a match offset is too big. -// -// Please refer to seqdec_generic.go for the reference implementation. -// -//go:noescape -func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool - -// Same as above, but with safe memcopies -// -//go:noescape -func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool - -// executeSimple handles cases when dictionary is not used. -func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { - // Ensure we have enough output size... - if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { - addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] - } - - if debugDecoder { - printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) - } - - var t = len(s.out) - out := s.out[:t+s.seqSize] - - ctx := executeAsmContext{ - seqs: seqs, - seqIndex: 0, - out: out, - history: hist, - outPosition: t, - litPosition: 0, - literals: s.literals, - windowSize: s.windowSize, - } - var ok bool - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - ok = sequenceDecs_executeSimple_safe_amd64(&ctx) - } else { - ok = sequenceDecs_executeSimple_amd64(&ctx) - } - if !ok { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", - seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) - } - s.literals = s.literals[ctx.litPosition:] - t = ctx.outPosition - - // Add final literals - copy(out[t:], s.literals) - if debugDecoder { - t += len(s.literals) - if t != len(out) { - panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) - } - } - s.out = out - - return nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s deleted file mode 100644 index f5591fa1e..000000000 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ /dev/null @@ -1,4151 +0,0 @@ -// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. - -//go:build !appengine && !noasm && gc && !noasm - -// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// Requires: CMOV -TEXT ·sequenceDecs_decode_amd64(SB), $8-32 - MOVQ br+8(FP), CX - MOVQ 24(CX), DX - MOVBQZX 32(CX), BX - MOVQ (CX), AX - MOVQ 8(CX), SI - ADDQ SI, AX - MOVQ AX, (SP) - MOVQ ctx+16(FP), AX - MOVQ 72(AX), DI - MOVQ 80(AX), R8 - MOVQ 88(AX), R9 - MOVQ 104(AX), R10 - MOVQ s+0(FP), AX - MOVQ 144(AX), R11 - MOVQ 152(AX), R12 - MOVQ 160(AX), R13 - -sequenceDecs_decode_amd64_main_loop: - MOVQ (SP), R14 - - // Fill bitreader to have enough for the offset and match length. - CMPQ SI, $0x08 - JL sequenceDecs_decode_amd64_fill_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R14 - MOVQ (R14), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decode_amd64_fill_end - -sequenceDecs_decode_amd64_fill_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decode_amd64_fill_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decode_amd64_fill_end - SHLQ $0x08, DX - SUBQ $0x01, R14 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R14), AX - ORQ AX, DX - JMP sequenceDecs_decode_amd64_fill_byte_by_byte - -sequenceDecs_decode_amd64_fill_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decode_amd64_fill_end: - // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_of_update_zero: - MOVQ AX, 16(R10) - - // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ml_update_zero: - MOVQ AX, 8(R10) - - // Fill bitreader to have enough for the remaining - CMPQ SI, $0x08 - JL sequenceDecs_decode_amd64_fill_2_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R14 - MOVQ (R14), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decode_amd64_fill_2_end - -sequenceDecs_decode_amd64_fill_2_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decode_amd64_fill_2_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decode_amd64_fill_2_end - SHLQ $0x08, DX - SUBQ $0x01, R14 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R14), AX - ORQ AX, DX - JMP sequenceDecs_decode_amd64_fill_2_byte_by_byte - -sequenceDecs_decode_amd64_fill_2_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decode_amd64_fill_2_end: - // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ll_update_zero: - MOVQ AX, (R10) - - // Fill bitreader for state updates - MOVQ R14, (SP) - MOVQ R9, AX - SHRQ $0x08, AX - MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_amd64_skip_update - - // Update Literal Length State - MOVBQZX DI, R14 - SHRL $0x10, DI - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, DI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(DI*8), DI - - // Update Match Length State - MOVBQZX R8, R14 - SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, R8 - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Offset State - MOVBQZX R9, R14 - SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, R9 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R9*8), R9 - -sequenceDecs_decode_amd64_skip_update: - // Adjust offset - MOVQ 16(R10), CX - CMPQ AX, $0x01 - JBE sequenceDecs_decode_amd64_adjust_offsetB_1_or_0 - MOVQ R12, R13 - MOVQ R11, R12 - MOVQ CX, R11 - JMP sequenceDecs_decode_amd64_after_adjust - -sequenceDecs_decode_amd64_adjust_offsetB_1_or_0: - CMPQ (R10), $0x00000000 - JNE sequenceDecs_decode_amd64_adjust_offset_maybezero - INCQ CX - JMP sequenceDecs_decode_amd64_adjust_offset_nonzero - -sequenceDecs_decode_amd64_adjust_offset_maybezero: - TESTQ CX, CX - JNZ sequenceDecs_decode_amd64_adjust_offset_nonzero - MOVQ R11, CX - JMP sequenceDecs_decode_amd64_after_adjust - -sequenceDecs_decode_amd64_adjust_offset_nonzero: - CMPQ CX, $0x01 - JB sequenceDecs_decode_amd64_adjust_zero - JEQ sequenceDecs_decode_amd64_adjust_one - CMPQ CX, $0x02 - JA sequenceDecs_decode_amd64_adjust_three - JMP sequenceDecs_decode_amd64_adjust_two - -sequenceDecs_decode_amd64_adjust_zero: - MOVQ R11, AX - JMP sequenceDecs_decode_amd64_adjust_test_temp_valid - -sequenceDecs_decode_amd64_adjust_one: - MOVQ R12, AX - JMP sequenceDecs_decode_amd64_adjust_test_temp_valid - -sequenceDecs_decode_amd64_adjust_two: - MOVQ R13, AX - JMP sequenceDecs_decode_amd64_adjust_test_temp_valid - -sequenceDecs_decode_amd64_adjust_three: - LEAQ -1(R11), AX - -sequenceDecs_decode_amd64_adjust_test_temp_valid: - TESTQ AX, AX - JNZ sequenceDecs_decode_amd64_adjust_temp_valid - MOVQ $0x00000001, AX - -sequenceDecs_decode_amd64_adjust_temp_valid: - CMPQ CX, $0x01 - CMOVQNE R12, R13 - MOVQ R11, R12 - MOVQ AX, R11 - MOVQ AX, CX - -sequenceDecs_decode_amd64_after_adjust: - MOVQ CX, 16(R10) - - // Check values - MOVQ 8(R10), AX - MOVQ (R10), R14 - LEAQ (AX)(R14*1), R15 - MOVQ s+0(FP), BP - ADDQ R15, 256(BP) - MOVQ ctx+16(FP), R15 - SUBQ R14, 128(R15) - JS error_not_enough_literals - CMPQ AX, $0x00020002 - JA sequenceDecs_decode_amd64_error_match_len_too_big - TESTQ CX, CX - JNZ sequenceDecs_decode_amd64_match_len_ofs_ok - TESTQ AX, AX - JNZ sequenceDecs_decode_amd64_error_match_len_ofs_mismatch - -sequenceDecs_decode_amd64_match_len_ofs_ok: - ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) - JNS sequenceDecs_decode_amd64_main_loop - MOVQ s+0(FP), AX - MOVQ R11, 144(AX) - MOVQ R12, 152(AX) - MOVQ R13, 160(AX) - MOVQ br+8(FP), AX - MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decode_amd64_error_match_len_too_big: - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - -// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// Requires: CMOV -TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 - MOVQ br+8(FP), CX - MOVQ 24(CX), DX - MOVBQZX 32(CX), BX - MOVQ (CX), AX - MOVQ 8(CX), SI - ADDQ SI, AX - MOVQ AX, (SP) - MOVQ ctx+16(FP), AX - MOVQ 72(AX), DI - MOVQ 80(AX), R8 - MOVQ 88(AX), R9 - MOVQ 104(AX), R10 - MOVQ s+0(FP), AX - MOVQ 144(AX), R11 - MOVQ 152(AX), R12 - MOVQ 160(AX), R13 - -sequenceDecs_decode_56_amd64_main_loop: - MOVQ (SP), R14 - - // Fill bitreader to have enough for the offset and match length. - CMPQ SI, $0x08 - JL sequenceDecs_decode_56_amd64_fill_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R14 - MOVQ (R14), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decode_56_amd64_fill_end - -sequenceDecs_decode_56_amd64_fill_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decode_56_amd64_fill_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decode_56_amd64_fill_end - SHLQ $0x08, DX - SUBQ $0x01, R14 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R14), AX - ORQ AX, DX - JMP sequenceDecs_decode_56_amd64_fill_byte_by_byte - -sequenceDecs_decode_56_amd64_fill_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decode_56_amd64_fill_end: - // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_of_update_zero: - MOVQ AX, 16(R10) - - // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ml_update_zero: - MOVQ AX, 8(R10) - - // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ll_update_zero: - MOVQ AX, (R10) - - // Fill bitreader for state updates - MOVQ R14, (SP) - MOVQ R9, AX - SHRQ $0x08, AX - MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_56_amd64_skip_update - - // Update Literal Length State - MOVBQZX DI, R14 - SHRL $0x10, DI - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, DI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(DI*8), DI - - // Update Match Length State - MOVBQZX R8, R14 - SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, R8 - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Offset State - MOVBQZX R9, R14 - SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX - MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 - ADDQ R15, R9 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R9*8), R9 - -sequenceDecs_decode_56_amd64_skip_update: - // Adjust offset - MOVQ 16(R10), CX - CMPQ AX, $0x01 - JBE sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0 - MOVQ R12, R13 - MOVQ R11, R12 - MOVQ CX, R11 - JMP sequenceDecs_decode_56_amd64_after_adjust - -sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0: - CMPQ (R10), $0x00000000 - JNE sequenceDecs_decode_56_amd64_adjust_offset_maybezero - INCQ CX - JMP sequenceDecs_decode_56_amd64_adjust_offset_nonzero - -sequenceDecs_decode_56_amd64_adjust_offset_maybezero: - TESTQ CX, CX - JNZ sequenceDecs_decode_56_amd64_adjust_offset_nonzero - MOVQ R11, CX - JMP sequenceDecs_decode_56_amd64_after_adjust - -sequenceDecs_decode_56_amd64_adjust_offset_nonzero: - CMPQ CX, $0x01 - JB sequenceDecs_decode_56_amd64_adjust_zero - JEQ sequenceDecs_decode_56_amd64_adjust_one - CMPQ CX, $0x02 - JA sequenceDecs_decode_56_amd64_adjust_three - JMP sequenceDecs_decode_56_amd64_adjust_two - -sequenceDecs_decode_56_amd64_adjust_zero: - MOVQ R11, AX - JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid - -sequenceDecs_decode_56_amd64_adjust_one: - MOVQ R12, AX - JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid - -sequenceDecs_decode_56_amd64_adjust_two: - MOVQ R13, AX - JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid - -sequenceDecs_decode_56_amd64_adjust_three: - LEAQ -1(R11), AX - -sequenceDecs_decode_56_amd64_adjust_test_temp_valid: - TESTQ AX, AX - JNZ sequenceDecs_decode_56_amd64_adjust_temp_valid - MOVQ $0x00000001, AX - -sequenceDecs_decode_56_amd64_adjust_temp_valid: - CMPQ CX, $0x01 - CMOVQNE R12, R13 - MOVQ R11, R12 - MOVQ AX, R11 - MOVQ AX, CX - -sequenceDecs_decode_56_amd64_after_adjust: - MOVQ CX, 16(R10) - - // Check values - MOVQ 8(R10), AX - MOVQ (R10), R14 - LEAQ (AX)(R14*1), R15 - MOVQ s+0(FP), BP - ADDQ R15, 256(BP) - MOVQ ctx+16(FP), R15 - SUBQ R14, 128(R15) - JS error_not_enough_literals - CMPQ AX, $0x00020002 - JA sequenceDecs_decode_56_amd64_error_match_len_too_big - TESTQ CX, CX - JNZ sequenceDecs_decode_56_amd64_match_len_ofs_ok - TESTQ AX, AX - JNZ sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch - -sequenceDecs_decode_56_amd64_match_len_ofs_ok: - ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) - JNS sequenceDecs_decode_56_amd64_main_loop - MOVQ s+0(FP), AX - MOVQ R11, 144(AX) - MOVQ R12, 152(AX) - MOVQ R13, 160(AX) - MOVQ br+8(FP), AX - MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decode_56_amd64_error_match_len_too_big: - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - -// func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 - MOVQ br+8(FP), BX - MOVQ 24(BX), AX - MOVBQZX 32(BX), DX - MOVQ (BX), CX - MOVQ 8(BX), BX - ADDQ BX, CX - MOVQ CX, (SP) - MOVQ ctx+16(FP), CX - MOVQ 72(CX), SI - MOVQ 80(CX), DI - MOVQ 88(CX), R8 - MOVQ 104(CX), R9 - MOVQ s+0(FP), CX - MOVQ 144(CX), R10 - MOVQ 152(CX), R11 - MOVQ 160(CX), R12 - -sequenceDecs_decode_bmi2_main_loop: - MOVQ (SP), R13 - - // Fill bitreader to have enough for the offset and match length. - CMPQ BX, $0x08 - JL sequenceDecs_decode_bmi2_fill_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R13 - MOVQ (R13), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decode_bmi2_fill_end - -sequenceDecs_decode_bmi2_fill_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decode_bmi2_fill_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decode_bmi2_fill_end - SHLQ $0x08, AX - SUBQ $0x01, R13 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R13), CX - ORQ CX, AX - JMP sequenceDecs_decode_bmi2_fill_byte_by_byte - -sequenceDecs_decode_bmi2_fill_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decode_bmi2_fill_end: - // Update offset - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ R8, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, 16(R9) - - // Update match length - MOVQ $0x00000808, CX - BEXTRQ CX, DI, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ DI, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, 8(R9) - - // Fill bitreader to have enough for the remaining - CMPQ BX, $0x08 - JL sequenceDecs_decode_bmi2_fill_2_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R13 - MOVQ (R13), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decode_bmi2_fill_2_end - -sequenceDecs_decode_bmi2_fill_2_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decode_bmi2_fill_2_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decode_bmi2_fill_2_end - SHLQ $0x08, AX - SUBQ $0x01, R13 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R13), CX - ORQ CX, AX - JMP sequenceDecs_decode_bmi2_fill_2_byte_by_byte - -sequenceDecs_decode_bmi2_fill_2_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decode_bmi2_fill_2_end: - // Update literal length - MOVQ $0x00000808, CX - BEXTRQ CX, SI, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ SI, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, (R9) - - // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_bmi2_skip_update - LEAQ (SI)(DI*1), R14 - ADDQ R8, R14 - MOVBQZX R14, R14 - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - - // Update Offset State - BZHIQ R8, R15, CX - SHRXQ R8, R15, R15 - SHRL $0x10, R8 - ADDQ CX, R8 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Match Length State - BZHIQ DI, R15, CX - SHRXQ DI, R15, R15 - SHRL $0x10, DI - ADDQ CX, DI - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(DI*8), DI - - // Update Literal Length State - BZHIQ SI, R15, CX - SHRL $0x10, SI - ADDQ CX, SI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI - -sequenceDecs_decode_bmi2_skip_update: - // Adjust offset - MOVQ 16(R9), CX - CMPQ R13, $0x01 - JBE sequenceDecs_decode_bmi2_adjust_offsetB_1_or_0 - MOVQ R11, R12 - MOVQ R10, R11 - MOVQ CX, R10 - JMP sequenceDecs_decode_bmi2_after_adjust - -sequenceDecs_decode_bmi2_adjust_offsetB_1_or_0: - CMPQ (R9), $0x00000000 - JNE sequenceDecs_decode_bmi2_adjust_offset_maybezero - INCQ CX - JMP sequenceDecs_decode_bmi2_adjust_offset_nonzero - -sequenceDecs_decode_bmi2_adjust_offset_maybezero: - TESTQ CX, CX - JNZ sequenceDecs_decode_bmi2_adjust_offset_nonzero - MOVQ R10, CX - JMP sequenceDecs_decode_bmi2_after_adjust - -sequenceDecs_decode_bmi2_adjust_offset_nonzero: - CMPQ CX, $0x01 - JB sequenceDecs_decode_bmi2_adjust_zero - JEQ sequenceDecs_decode_bmi2_adjust_one - CMPQ CX, $0x02 - JA sequenceDecs_decode_bmi2_adjust_three - JMP sequenceDecs_decode_bmi2_adjust_two - -sequenceDecs_decode_bmi2_adjust_zero: - MOVQ R10, R13 - JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_bmi2_adjust_one: - MOVQ R11, R13 - JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_bmi2_adjust_two: - MOVQ R12, R13 - JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_bmi2_adjust_three: - LEAQ -1(R10), R13 - -sequenceDecs_decode_bmi2_adjust_test_temp_valid: - TESTQ R13, R13 - JNZ sequenceDecs_decode_bmi2_adjust_temp_valid - MOVQ $0x00000001, R13 - -sequenceDecs_decode_bmi2_adjust_temp_valid: - CMPQ CX, $0x01 - CMOVQNE R11, R12 - MOVQ R10, R11 - MOVQ R13, R10 - MOVQ R13, CX - -sequenceDecs_decode_bmi2_after_adjust: - MOVQ CX, 16(R9) - - // Check values - MOVQ 8(R9), R13 - MOVQ (R9), R14 - LEAQ (R13)(R14*1), R15 - MOVQ s+0(FP), BP - ADDQ R15, 256(BP) - MOVQ ctx+16(FP), R15 - SUBQ R14, 128(R15) - JS error_not_enough_literals - CMPQ R13, $0x00020002 - JA sequenceDecs_decode_bmi2_error_match_len_too_big - TESTQ CX, CX - JNZ sequenceDecs_decode_bmi2_match_len_ofs_ok - TESTQ R13, R13 - JNZ sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch - -sequenceDecs_decode_bmi2_match_len_ofs_ok: - ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) - JNS sequenceDecs_decode_bmi2_main_loop - MOVQ s+0(FP), CX - MOVQ R10, 144(CX) - MOVQ R11, 152(CX) - MOVQ R12, 160(CX) - MOVQ br+8(FP), CX - MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decode_bmi2_error_match_len_too_big: - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - -// func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 - MOVQ br+8(FP), BX - MOVQ 24(BX), AX - MOVBQZX 32(BX), DX - MOVQ (BX), CX - MOVQ 8(BX), BX - ADDQ BX, CX - MOVQ CX, (SP) - MOVQ ctx+16(FP), CX - MOVQ 72(CX), SI - MOVQ 80(CX), DI - MOVQ 88(CX), R8 - MOVQ 104(CX), R9 - MOVQ s+0(FP), CX - MOVQ 144(CX), R10 - MOVQ 152(CX), R11 - MOVQ 160(CX), R12 - -sequenceDecs_decode_56_bmi2_main_loop: - MOVQ (SP), R13 - - // Fill bitreader to have enough for the offset and match length. - CMPQ BX, $0x08 - JL sequenceDecs_decode_56_bmi2_fill_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R13 - MOVQ (R13), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decode_56_bmi2_fill_end - -sequenceDecs_decode_56_bmi2_fill_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decode_56_bmi2_fill_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decode_56_bmi2_fill_end - SHLQ $0x08, AX - SUBQ $0x01, R13 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R13), CX - ORQ CX, AX - JMP sequenceDecs_decode_56_bmi2_fill_byte_by_byte - -sequenceDecs_decode_56_bmi2_fill_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decode_56_bmi2_fill_end: - // Update offset - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ R8, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, 16(R9) - - // Update match length - MOVQ $0x00000808, CX - BEXTRQ CX, DI, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ DI, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, 8(R9) - - // Update literal length - MOVQ $0x00000808, CX - BEXTRQ CX, SI, R14 - MOVQ AX, R15 - LEAQ (DX)(R14*1), CX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - MOVQ CX, DX - MOVQ SI, CX - SHRQ $0x20, CX - ADDQ R15, CX - MOVQ CX, (R9) - - // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_56_bmi2_skip_update - LEAQ (SI)(DI*1), R14 - ADDQ R8, R14 - MOVBQZX R14, R14 - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - - // Update Offset State - BZHIQ R8, R15, CX - SHRXQ R8, R15, R15 - SHRL $0x10, R8 - ADDQ CX, R8 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Match Length State - BZHIQ DI, R15, CX - SHRXQ DI, R15, R15 - SHRL $0x10, DI - ADDQ CX, DI - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(DI*8), DI - - // Update Literal Length State - BZHIQ SI, R15, CX - SHRL $0x10, SI - ADDQ CX, SI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI - -sequenceDecs_decode_56_bmi2_skip_update: - // Adjust offset - MOVQ 16(R9), CX - CMPQ R13, $0x01 - JBE sequenceDecs_decode_56_bmi2_adjust_offsetB_1_or_0 - MOVQ R11, R12 - MOVQ R10, R11 - MOVQ CX, R10 - JMP sequenceDecs_decode_56_bmi2_after_adjust - -sequenceDecs_decode_56_bmi2_adjust_offsetB_1_or_0: - CMPQ (R9), $0x00000000 - JNE sequenceDecs_decode_56_bmi2_adjust_offset_maybezero - INCQ CX - JMP sequenceDecs_decode_56_bmi2_adjust_offset_nonzero - -sequenceDecs_decode_56_bmi2_adjust_offset_maybezero: - TESTQ CX, CX - JNZ sequenceDecs_decode_56_bmi2_adjust_offset_nonzero - MOVQ R10, CX - JMP sequenceDecs_decode_56_bmi2_after_adjust - -sequenceDecs_decode_56_bmi2_adjust_offset_nonzero: - CMPQ CX, $0x01 - JB sequenceDecs_decode_56_bmi2_adjust_zero - JEQ sequenceDecs_decode_56_bmi2_adjust_one - CMPQ CX, $0x02 - JA sequenceDecs_decode_56_bmi2_adjust_three - JMP sequenceDecs_decode_56_bmi2_adjust_two - -sequenceDecs_decode_56_bmi2_adjust_zero: - MOVQ R10, R13 - JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_56_bmi2_adjust_one: - MOVQ R11, R13 - JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_56_bmi2_adjust_two: - MOVQ R12, R13 - JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid - -sequenceDecs_decode_56_bmi2_adjust_three: - LEAQ -1(R10), R13 - -sequenceDecs_decode_56_bmi2_adjust_test_temp_valid: - TESTQ R13, R13 - JNZ sequenceDecs_decode_56_bmi2_adjust_temp_valid - MOVQ $0x00000001, R13 - -sequenceDecs_decode_56_bmi2_adjust_temp_valid: - CMPQ CX, $0x01 - CMOVQNE R11, R12 - MOVQ R10, R11 - MOVQ R13, R10 - MOVQ R13, CX - -sequenceDecs_decode_56_bmi2_after_adjust: - MOVQ CX, 16(R9) - - // Check values - MOVQ 8(R9), R13 - MOVQ (R9), R14 - LEAQ (R13)(R14*1), R15 - MOVQ s+0(FP), BP - ADDQ R15, 256(BP) - MOVQ ctx+16(FP), R15 - SUBQ R14, 128(R15) - JS error_not_enough_literals - CMPQ R13, $0x00020002 - JA sequenceDecs_decode_56_bmi2_error_match_len_too_big - TESTQ CX, CX - JNZ sequenceDecs_decode_56_bmi2_match_len_ofs_ok - TESTQ R13, R13 - JNZ sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch - -sequenceDecs_decode_56_bmi2_match_len_ofs_ok: - ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) - JNS sequenceDecs_decode_56_bmi2_main_loop - MOVQ s+0(FP), CX - MOVQ R10, 144(CX) - MOVQ R11, 152(CX) - MOVQ R12, 160(CX) - MOVQ br+8(FP), CX - MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decode_56_bmi2_error_match_len_too_big: - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - -// func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9 - MOVQ ctx+0(FP), R10 - MOVQ 8(R10), CX - TESTQ CX, CX - JZ empty_seqs - MOVQ (R10), AX - MOVQ 24(R10), DX - MOVQ 32(R10), BX - MOVQ 80(R10), SI - MOVQ 104(R10), DI - MOVQ 120(R10), R8 - MOVQ 56(R10), R9 - MOVQ 64(R10), R10 - ADDQ R10, R9 - - // seqsBase += 24 * seqIndex - LEAQ (DX)(DX*2), R11 - SHLQ $0x03, R11 - ADDQ R11, AX - - // outBase += outPosition - ADDQ DI, BX - -main_loop: - MOVQ (AX), R11 - MOVQ 16(AX), R12 - MOVQ 8(AX), R13 - - // Copy literals - TESTQ R11, R11 - JZ check_offset - XORQ R14, R14 - -copy_1: - MOVUPS (SI)(R14*1), X0 - MOVUPS X0, (BX)(R14*1) - ADDQ $0x10, R14 - CMPQ R14, R11 - JB copy_1 - ADDQ R11, SI - ADDQ R11, BX - ADDQ R11, DI - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - LEAQ (DI)(R10*1), R11 - CMPQ R12, R11 - JG error_match_off_too_big - CMPQ R12, R8 - JG error_match_off_too_big - - // Copy match from history - MOVQ R12, R11 - SUBQ DI, R11 - JLS copy_match - MOVQ R9, R14 - SUBQ R11, R14 - CMPQ R13, R11 - JG copy_all_from_history - MOVQ R13, R11 - SUBQ $0x10, R11 - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (BX) - ADDQ $0x10, R14 - ADDQ $0x10, BX - SUBQ $0x10, R11 - JAE copy_4_loop - LEAQ 16(R14)(R11*1), R14 - LEAQ 16(BX)(R11*1), BX - MOVUPS -16(R14), X0 - MOVUPS X0, -16(BX) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), R11 - MOVL -4(R14)(R13*1), R12 - MOVL R11, (BX) - MOVL R12, -4(BX)(R13*1) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), R11 - MOVQ -8(R14)(R13*1), R12 - MOVQ R11, (BX) - MOVQ R12, -8(BX)(R13*1) - ADDQ R13, R14 - ADDQ R13, BX - -copy_4_end: - ADDQ R13, DI - ADDQ $0x18, AX - INCQ DX - CMPQ DX, CX - JB main_loop - JMP loop_finished - -copy_all_from_history: - MOVQ R11, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (BX) - ADDQ $0x10, R14 - ADDQ $0x10, BX - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(BX)(R15*1), BX - MOVUPS -16(R14), X0 - MOVUPS X0, -16(BX) - JMP copy_5_end - -copy_5_small: - CMPQ R11, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ R11, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(R11*1), BP - MOVB R15, (BX) - MOVB BP, -1(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(R11*1), BP - MOVL R15, (BX) - MOVL BP, -4(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(R11*1), BP - MOVQ R15, (BX) - MOVQ BP, -8(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - -copy_5_end: - ADDQ R11, DI - SUBQ R11, R13 - - // Copy match from the current buffer -copy_match: - MOVQ BX, R11 - SUBQ R12, R11 - - // ml <= mo - CMPQ R13, R12 - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 - ADDQ R13, BX - -copy_2: - MOVUPS (R11), X0 - MOVUPS X0, (R12) - ADDQ $0x10, R11 - ADDQ $0x10, R12 - SUBQ $0x10, R13 - JHI copy_2 - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, DI - -copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - ADDQ $0x18, AX - INCQ DX - CMPQ DX, CX - JB main_loop - -loop_finished: - // Return value - MOVB $0x01, ret+8(FP) - - // Update the context - MOVQ ctx+0(FP), AX - MOVQ DX, 24(AX) - MOVQ DI, 104(AX) - SUBQ 80(AX), SI - MOVQ SI, 112(AX) - RET - -error_match_off_too_big: - // Return value - MOVB $0x00, ret+8(FP) - - // Update the context - MOVQ ctx+0(FP), AX - MOVQ DX, 24(AX) - MOVQ DI, 104(AX) - SUBQ 80(AX), SI - MOVQ SI, 112(AX) - RET - -empty_seqs: - // Return value - MOVB $0x01, ret+8(FP) - RET - -// func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $8-9 - MOVQ ctx+0(FP), R10 - MOVQ 8(R10), CX - TESTQ CX, CX - JZ empty_seqs - MOVQ (R10), AX - MOVQ 24(R10), DX - MOVQ 32(R10), BX - MOVQ 80(R10), SI - MOVQ 104(R10), DI - MOVQ 120(R10), R8 - MOVQ 56(R10), R9 - MOVQ 64(R10), R10 - ADDQ R10, R9 - - // seqsBase += 24 * seqIndex - LEAQ (DX)(DX*2), R11 - SHLQ $0x03, R11 - ADDQ R11, AX - - // outBase += outPosition - ADDQ DI, BX - -main_loop: - MOVQ (AX), R11 - MOVQ 16(AX), R12 - MOVQ 8(AX), R13 - - // Copy literals - TESTQ R11, R11 - JZ check_offset - MOVQ R11, R14 - SUBQ $0x10, R14 - JB copy_1_small - -copy_1_loop: - MOVUPS (SI), X0 - MOVUPS X0, (BX) - ADDQ $0x10, SI - ADDQ $0x10, BX - SUBQ $0x10, R14 - JAE copy_1_loop - LEAQ 16(SI)(R14*1), SI - LEAQ 16(BX)(R14*1), BX - MOVUPS -16(SI), X0 - MOVUPS X0, -16(BX) - JMP copy_1_end - -copy_1_small: - CMPQ R11, $0x03 - JE copy_1_move_3 - JB copy_1_move_1or2 - CMPQ R11, $0x08 - JB copy_1_move_4through7 - JMP copy_1_move_8through16 - -copy_1_move_1or2: - MOVB (SI), R14 - MOVB -1(SI)(R11*1), R15 - MOVB R14, (BX) - MOVB R15, -1(BX)(R11*1) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end - -copy_1_move_3: - MOVW (SI), R14 - MOVB 2(SI), R15 - MOVW R14, (BX) - MOVB R15, 2(BX) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end - -copy_1_move_4through7: - MOVL (SI), R14 - MOVL -4(SI)(R11*1), R15 - MOVL R14, (BX) - MOVL R15, -4(BX)(R11*1) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end - -copy_1_move_8through16: - MOVQ (SI), R14 - MOVQ -8(SI)(R11*1), R15 - MOVQ R14, (BX) - MOVQ R15, -8(BX)(R11*1) - ADDQ R11, SI - ADDQ R11, BX - -copy_1_end: - ADDQ R11, DI - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - LEAQ (DI)(R10*1), R11 - CMPQ R12, R11 - JG error_match_off_too_big - CMPQ R12, R8 - JG error_match_off_too_big - - // Copy match from history - MOVQ R12, R11 - SUBQ DI, R11 - JLS copy_match - MOVQ R9, R14 - SUBQ R11, R14 - CMPQ R13, R11 - JG copy_all_from_history - MOVQ R13, R11 - SUBQ $0x10, R11 - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (BX) - ADDQ $0x10, R14 - ADDQ $0x10, BX - SUBQ $0x10, R11 - JAE copy_4_loop - LEAQ 16(R14)(R11*1), R14 - LEAQ 16(BX)(R11*1), BX - MOVUPS -16(R14), X0 - MOVUPS X0, -16(BX) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), R11 - MOVL -4(R14)(R13*1), R12 - MOVL R11, (BX) - MOVL R12, -4(BX)(R13*1) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), R11 - MOVQ -8(R14)(R13*1), R12 - MOVQ R11, (BX) - MOVQ R12, -8(BX)(R13*1) - ADDQ R13, R14 - ADDQ R13, BX - -copy_4_end: - ADDQ R13, DI - ADDQ $0x18, AX - INCQ DX - CMPQ DX, CX - JB main_loop - JMP loop_finished - -copy_all_from_history: - MOVQ R11, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (BX) - ADDQ $0x10, R14 - ADDQ $0x10, BX - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(BX)(R15*1), BX - MOVUPS -16(R14), X0 - MOVUPS X0, -16(BX) - JMP copy_5_end - -copy_5_small: - CMPQ R11, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ R11, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(R11*1), BP - MOVB R15, (BX) - MOVB BP, -1(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(R11*1), BP - MOVL R15, (BX) - MOVL BP, -4(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(R11*1), BP - MOVQ R15, (BX) - MOVQ BP, -8(BX)(R11*1) - ADDQ R11, R14 - ADDQ R11, BX - -copy_5_end: - ADDQ R11, DI - SUBQ R11, R13 - - // Copy match from the current buffer -copy_match: - MOVQ BX, R11 - SUBQ R12, R11 - - // ml <= mo - CMPQ R13, R12 - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, DI - MOVQ R13, R12 - SUBQ $0x10, R12 - JB copy_2_small - -copy_2_loop: - MOVUPS (R11), X0 - MOVUPS X0, (BX) - ADDQ $0x10, R11 - ADDQ $0x10, BX - SUBQ $0x10, R12 - JAE copy_2_loop - LEAQ 16(R11)(R12*1), R11 - LEAQ 16(BX)(R12*1), BX - MOVUPS -16(R11), X0 - MOVUPS X0, -16(BX) - JMP copy_2_end - -copy_2_small: - CMPQ R13, $0x03 - JE copy_2_move_3 - JB copy_2_move_1or2 - CMPQ R13, $0x08 - JB copy_2_move_4through7 - JMP copy_2_move_8through16 - -copy_2_move_1or2: - MOVB (R11), R12 - MOVB -1(R11)(R13*1), R14 - MOVB R12, (BX) - MOVB R14, -1(BX)(R13*1) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end - -copy_2_move_3: - MOVW (R11), R12 - MOVB 2(R11), R14 - MOVW R12, (BX) - MOVB R14, 2(BX) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end - -copy_2_move_4through7: - MOVL (R11), R12 - MOVL -4(R11)(R13*1), R14 - MOVL R12, (BX) - MOVL R14, -4(BX)(R13*1) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end - -copy_2_move_8through16: - MOVQ (R11), R12 - MOVQ -8(R11)(R13*1), R14 - MOVQ R12, (BX) - MOVQ R14, -8(BX)(R13*1) - ADDQ R13, R11 - ADDQ R13, BX - -copy_2_end: - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, DI - -copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - ADDQ $0x18, AX - INCQ DX - CMPQ DX, CX - JB main_loop - -loop_finished: - // Return value - MOVB $0x01, ret+8(FP) - - // Update the context - MOVQ ctx+0(FP), AX - MOVQ DX, 24(AX) - MOVQ DI, 104(AX) - SUBQ 80(AX), SI - MOVQ SI, 112(AX) - RET - -error_match_off_too_big: - // Return value - MOVB $0x00, ret+8(FP) - - // Update the context - MOVQ ctx+0(FP), AX - MOVQ DX, 24(AX) - MOVQ DI, 104(AX) - SUBQ 80(AX), SI - MOVQ SI, 112(AX) - RET - -empty_seqs: - // Return value - MOVB $0x01, ret+8(FP) - RET - -// func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 - MOVQ br+8(FP), CX - MOVQ 24(CX), DX - MOVBQZX 32(CX), BX - MOVQ (CX), AX - MOVQ 8(CX), SI - ADDQ SI, AX - MOVQ AX, (SP) - MOVQ ctx+16(FP), AX - MOVQ 72(AX), DI - MOVQ 80(AX), R8 - MOVQ 88(AX), R9 - XORQ CX, CX - MOVQ CX, 8(SP) - MOVQ CX, 16(SP) - MOVQ CX, 24(SP) - MOVQ 112(AX), R10 - MOVQ 128(AX), CX - MOVQ CX, 32(SP) - MOVQ 144(AX), R11 - MOVQ 136(AX), R12 - MOVQ 200(AX), CX - MOVQ CX, 56(SP) - MOVQ 176(AX), CX - MOVQ CX, 48(SP) - MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) - - // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) - - // outBase += outPosition - ADDQ R12, R10 - -sequenceDecs_decodeSync_amd64_main_loop: - MOVQ (SP), R13 - - // Fill bitreader to have enough for the offset and match length. - CMPQ SI, $0x08 - JL sequenceDecs_decodeSync_amd64_fill_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R13 - MOVQ (R13), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decodeSync_amd64_fill_end - -sequenceDecs_decodeSync_amd64_fill_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_amd64_fill_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decodeSync_amd64_fill_end - SHLQ $0x08, DX - SUBQ $0x01, R13 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R13), AX - ORQ AX, DX - JMP sequenceDecs_decodeSync_amd64_fill_byte_by_byte - -sequenceDecs_decodeSync_amd64_fill_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_amd64_fill_end: - // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVQ AX, 8(SP) - - // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVQ AX, 16(SP) - - // Fill bitreader to have enough for the remaining - CMPQ SI, $0x08 - JL sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R13 - MOVQ (R13), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decodeSync_amd64_fill_2_end - -sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_amd64_fill_2_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decodeSync_amd64_fill_2_end - SHLQ $0x08, DX - SUBQ $0x01, R13 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R13), AX - ORQ AX, DX - JMP sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte - -sequenceDecs_decodeSync_amd64_fill_2_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_amd64_fill_2_end: - // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVQ AX, 24(SP) - - // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ R9, AX - SHRQ $0x08, AX - MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_amd64_skip_update - - // Update Literal Length State - MOVBQZX DI, R13 - SHRL $0x10, DI - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, DI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(DI*8), DI - - // Update Match Length State - MOVBQZX R8, R13 - SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, R8 - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Offset State - MOVBQZX R9, R13 - SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, R9 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R9*8), R9 - -sequenceDecs_decodeSync_amd64_skip_update: - // Adjust offset - MOVQ s+0(FP), CX - MOVQ 8(SP), R13 - CMPQ AX, $0x01 - JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 - MOVUPS 144(CX), X0 - MOVQ R13, 144(CX) - MOVUPS X0, 152(CX) - JMP sequenceDecs_decodeSync_amd64_after_adjust - -sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 - JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero - INCQ R13 - JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero - -sequenceDecs_decodeSync_amd64_adjust_offset_maybezero: - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_amd64_adjust_offset_nonzero - MOVQ 144(CX), R13 - JMP sequenceDecs_decodeSync_amd64_after_adjust - -sequenceDecs_decodeSync_amd64_adjust_offset_nonzero: - MOVQ R13, AX - XORQ R14, R14 - MOVQ $-1, R15 - CMPQ R13, $0x03 - CMOVQEQ R14, AX - CMOVQEQ R15, R14 - ADDQ 144(CX)(AX*8), R14 - JNZ sequenceDecs_decodeSync_amd64_adjust_temp_valid - MOVQ $0x00000001, R14 - -sequenceDecs_decodeSync_amd64_adjust_temp_valid: - CMPQ R13, $0x01 - JZ sequenceDecs_decodeSync_amd64_adjust_skip - MOVQ 152(CX), AX - MOVQ AX, 160(CX) - -sequenceDecs_decodeSync_amd64_adjust_skip: - MOVQ 144(CX), AX - MOVQ AX, 152(CX) - MOVQ R14, 144(CX) - MOVQ R14, R13 - -sequenceDecs_decodeSync_amd64_after_adjust: - MOVQ R13, 8(SP) - - // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) - MOVQ ctx+16(FP), R14 - SUBQ CX, 104(R14) - JS error_not_enough_literals - CMPQ AX, $0x00020002 - JA sequenceDecs_decodeSync_amd64_error_match_len_too_big - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_amd64_match_len_ofs_ok - TESTQ AX, AX - JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch - -sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 - ADDQ R10, R14 - CMPQ R14, 32(SP) - JA error_not_enough_space - - // Copy literals - TESTQ AX, AX - JZ check_offset - XORQ R14, R14 - -copy_1: - MOVUPS (R11)(R14*1), X0 - MOVUPS X0, (R10)(R14*1) - ADDQ $0x10, R14 - CMPQ R14, AX - JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX - JG error_match_off_too_big - CMPQ CX, 56(SP) - JG error_match_off_too_big - - // Copy match from history - MOVQ CX, AX - SUBQ R12, AX - JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX - JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R10) - ADDQ $0x10, R14 - ADDQ $0x10, R10 - SUBQ $0x10, AX - JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R10) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 - -copy_4_end: - ADDQ R13, R12 - JMP handle_loop - JMP loop_finished - -copy_all_from_history: - MOVQ AX, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R10) - ADDQ $0x10, R14 - ADDQ $0x10, R10 - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(R10)(R15*1), R10 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R10) - JMP copy_5_end - -copy_5_small: - CMPQ AX, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ AX, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP - MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP - MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP - MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - -copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 - - // Copy match from the current buffer -copy_match: - MOVQ R10, AX - SUBQ CX, AX - - // ml <= mo - CMPQ R13, CX - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R10, CX - ADDQ R13, R10 - -copy_2: - MOVUPS (AX), X0 - MOVUPS X0, (CX) - ADDQ $0x10, AX - ADDQ $0x10, CX - SUBQ $0x10, R13 - JHI copy_2 - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, R12 - -copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) - JNS sequenceDecs_decodeSync_amd64_main_loop - -loop_finished: - MOVQ br+8(FP), AX - MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) - - // Update the context - MOVQ ctx+16(FP), AX - MOVQ R12, 136(AX) - MOVQ 144(AX), CX - SUBQ CX, R11 - MOVQ R11, 168(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX - MOVQ ctx+16(FP), CX - MOVQ AX, 216(CX) - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decodeSync_amd64_error_match_len_too_big: - MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error -error_match_off_too_big: - MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX - MOVQ CX, 224(AX) - MOVQ R12, 136(AX) - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - - // Return with not enough output space error -error_not_enough_space: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ R12, 136(AX) - MOVQ $0x00000005, ret+24(FP) - RET - -// func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 - MOVQ br+8(FP), BX - MOVQ 24(BX), AX - MOVBQZX 32(BX), DX - MOVQ (BX), CX - MOVQ 8(BX), BX - ADDQ BX, CX - MOVQ CX, (SP) - MOVQ ctx+16(FP), CX - MOVQ 72(CX), SI - MOVQ 80(CX), DI - MOVQ 88(CX), R8 - XORQ R9, R9 - MOVQ R9, 8(SP) - MOVQ R9, 16(SP) - MOVQ R9, 24(SP) - MOVQ 112(CX), R9 - MOVQ 128(CX), R10 - MOVQ R10, 32(SP) - MOVQ 144(CX), R10 - MOVQ 136(CX), R11 - MOVQ 200(CX), R12 - MOVQ R12, 56(SP) - MOVQ 176(CX), R12 - MOVQ R12, 48(SP) - MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) - - // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) - - // outBase += outPosition - ADDQ R11, R9 - -sequenceDecs_decodeSync_bmi2_main_loop: - MOVQ (SP), R12 - - // Fill bitreader to have enough for the offset and match length. - CMPQ BX, $0x08 - JL sequenceDecs_decodeSync_bmi2_fill_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R12 - MOVQ (R12), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decodeSync_bmi2_fill_end - -sequenceDecs_decodeSync_bmi2_fill_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_bmi2_fill_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decodeSync_bmi2_fill_end - SHLQ $0x08, AX - SUBQ $0x01, R12 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R12), CX - ORQ CX, AX - JMP sequenceDecs_decodeSync_bmi2_fill_byte_by_byte - -sequenceDecs_decodeSync_bmi2_fill_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_bmi2_fill_end: - // Update offset - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ R8, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 8(SP) - - // Update match length - MOVQ $0x00000808, CX - BEXTRQ CX, DI, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ DI, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 16(SP) - - // Fill bitreader to have enough for the remaining - CMPQ BX, $0x08 - JL sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R12 - MOVQ (R12), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decodeSync_bmi2_fill_2_end - -sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_bmi2_fill_2_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decodeSync_bmi2_fill_2_end - SHLQ $0x08, AX - SUBQ $0x01, R12 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R12), CX - ORQ CX, AX - JMP sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte - -sequenceDecs_decodeSync_bmi2_fill_2_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_bmi2_fill_2_end: - // Update literal length - MOVQ $0x00000808, CX - BEXTRQ CX, SI, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ SI, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 24(SP) - - // Fill bitreader for state updates - MOVQ R12, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_bmi2_skip_update - LEAQ (SI)(DI*1), R13 - ADDQ R8, R13 - MOVBQZX R13, R13 - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - - // Update Offset State - BZHIQ R8, R14, CX - SHRXQ R8, R14, R14 - SHRL $0x10, R8 - ADDQ CX, R8 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Match Length State - BZHIQ DI, R14, CX - SHRXQ DI, R14, R14 - SHRL $0x10, DI - ADDQ CX, DI - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(DI*8), DI - - // Update Literal Length State - BZHIQ SI, R14, CX - SHRL $0x10, SI - ADDQ CX, SI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI - -sequenceDecs_decodeSync_bmi2_skip_update: - // Adjust offset - MOVQ s+0(FP), CX - MOVQ 8(SP), R13 - CMPQ R12, $0x01 - JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 - MOVUPS 144(CX), X0 - MOVQ R13, 144(CX) - MOVUPS X0, 152(CX) - JMP sequenceDecs_decodeSync_bmi2_after_adjust - -sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 - JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero - INCQ R13 - JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero - -sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero: - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero - MOVQ 144(CX), R13 - JMP sequenceDecs_decodeSync_bmi2_after_adjust - -sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero: - MOVQ R13, R12 - XORQ R14, R14 - MOVQ $-1, R15 - CMPQ R13, $0x03 - CMOVQEQ R14, R12 - CMOVQEQ R15, R14 - ADDQ 144(CX)(R12*8), R14 - JNZ sequenceDecs_decodeSync_bmi2_adjust_temp_valid - MOVQ $0x00000001, R14 - -sequenceDecs_decodeSync_bmi2_adjust_temp_valid: - CMPQ R13, $0x01 - JZ sequenceDecs_decodeSync_bmi2_adjust_skip - MOVQ 152(CX), R12 - MOVQ R12, 160(CX) - -sequenceDecs_decodeSync_bmi2_adjust_skip: - MOVQ 144(CX), R12 - MOVQ R12, 152(CX) - MOVQ R14, 144(CX) - MOVQ R14, R13 - -sequenceDecs_decodeSync_bmi2_after_adjust: - MOVQ R13, 8(SP) - - // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) - MOVQ ctx+16(FP), R14 - SUBQ R12, 104(R14) - JS error_not_enough_literals - CMPQ CX, $0x00020002 - JA sequenceDecs_decodeSync_bmi2_error_match_len_too_big - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok - TESTQ CX, CX - JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch - -sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 - ADDQ R9, R14 - CMPQ R14, 32(SP) - JA error_not_enough_space - - // Copy literals - TESTQ CX, CX - JZ check_offset - XORQ R14, R14 - -copy_1: - MOVUPS (R10)(R14*1), X0 - MOVUPS X0, (R9)(R14*1) - ADDQ $0x10, R14 - CMPQ R14, CX - JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX - JG error_match_off_too_big - CMPQ R12, 56(SP) - JG error_match_off_too_big - - // Copy match from history - MOVQ R12, CX - SUBQ R11, CX - JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX - JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R9) - ADDQ $0x10, R14 - ADDQ $0x10, R9 - SUBQ $0x10, CX - JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R9) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 - -copy_4_end: - ADDQ R13, R11 - JMP handle_loop - JMP loop_finished - -copy_all_from_history: - MOVQ CX, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R9) - ADDQ $0x10, R14 - ADDQ $0x10, R9 - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(R9)(R15*1), R9 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R9) - JMP copy_5_end - -copy_5_small: - CMPQ CX, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ CX, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP - MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP - MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP - MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - -copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 - - // Copy match from the current buffer -copy_match: - MOVQ R9, CX - SUBQ R12, CX - - // ml <= mo - CMPQ R13, R12 - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R9, R12 - ADDQ R13, R9 - -copy_2: - MOVUPS (CX), X0 - MOVUPS X0, (R12) - ADDQ $0x10, CX - ADDQ $0x10, R12 - SUBQ $0x10, R13 - JHI copy_2 - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, R11 - -copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) - JNS sequenceDecs_decodeSync_bmi2_main_loop - -loop_finished: - MOVQ br+8(FP), CX - MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) - - // Update the context - MOVQ ctx+16(FP), AX - MOVQ R11, 136(AX) - MOVQ 144(AX), CX - SUBQ CX, R10 - MOVQ R10, 168(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX - MOVQ ctx+16(FP), CX - MOVQ AX, 216(CX) - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decodeSync_bmi2_error_match_len_too_big: - MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error -error_match_off_too_big: - MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX - MOVQ CX, 224(AX) - MOVQ R11, 136(AX) - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - - // Return with not enough output space error -error_not_enough_space: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ R11, 136(AX) - MOVQ $0x00000005, ret+24(FP) - RET - -// func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 - MOVQ br+8(FP), CX - MOVQ 24(CX), DX - MOVBQZX 32(CX), BX - MOVQ (CX), AX - MOVQ 8(CX), SI - ADDQ SI, AX - MOVQ AX, (SP) - MOVQ ctx+16(FP), AX - MOVQ 72(AX), DI - MOVQ 80(AX), R8 - MOVQ 88(AX), R9 - XORQ CX, CX - MOVQ CX, 8(SP) - MOVQ CX, 16(SP) - MOVQ CX, 24(SP) - MOVQ 112(AX), R10 - MOVQ 128(AX), CX - MOVQ CX, 32(SP) - MOVQ 144(AX), R11 - MOVQ 136(AX), R12 - MOVQ 200(AX), CX - MOVQ CX, 56(SP) - MOVQ 176(AX), CX - MOVQ CX, 48(SP) - MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) - - // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) - - // outBase += outPosition - ADDQ R12, R10 - -sequenceDecs_decodeSync_safe_amd64_main_loop: - MOVQ (SP), R13 - - // Fill bitreader to have enough for the offset and match length. - CMPQ SI, $0x08 - JL sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R13 - MOVQ (R13), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decodeSync_safe_amd64_fill_end - -sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_safe_amd64_fill_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decodeSync_safe_amd64_fill_end - SHLQ $0x08, DX - SUBQ $0x01, R13 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R13), AX - ORQ AX, DX - JMP sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte - -sequenceDecs_decodeSync_safe_amd64_fill_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_safe_amd64_fill_end: - // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVQ AX, 8(SP) - - // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVQ AX, 16(SP) - - // Fill bitreader to have enough for the remaining - CMPQ SI, $0x08 - JL sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte - MOVQ BX, AX - SHRQ $0x03, AX - SUBQ AX, R13 - MOVQ (R13), DX - SUBQ AX, SI - ANDQ $0x07, BX - JMP sequenceDecs_decodeSync_safe_amd64_fill_2_end - -sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte: - CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread - CMPQ BX, $0x07 - JLE sequenceDecs_decodeSync_safe_amd64_fill_2_end - SHLQ $0x08, DX - SUBQ $0x01, R13 - SUBQ $0x01, SI - SUBQ $0x08, BX - MOVBQZX (R13), AX - ORQ AX, DX - JMP sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte - -sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: - CMPQ BX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_safe_amd64_fill_2_end: - // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVQ AX, 24(SP) - - // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ R9, AX - SHRQ $0x08, AX - MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_safe_amd64_skip_update - - // Update Literal Length State - MOVBQZX DI, R13 - SHRL $0x10, DI - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, DI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(DI*8), DI - - // Update Match Length State - MOVBQZX R8, R13 - SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, R8 - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Offset State - MOVBQZX R9, R13 - SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX - MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 - ADDQ R14, R9 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R9*8), R9 - -sequenceDecs_decodeSync_safe_amd64_skip_update: - // Adjust offset - MOVQ s+0(FP), CX - MOVQ 8(SP), R13 - CMPQ AX, $0x01 - JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 - MOVUPS 144(CX), X0 - MOVQ R13, 144(CX) - MOVUPS X0, 152(CX) - JMP sequenceDecs_decodeSync_safe_amd64_after_adjust - -sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 - JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero - INCQ R13 - JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero - -sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero: - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero - MOVQ 144(CX), R13 - JMP sequenceDecs_decodeSync_safe_amd64_after_adjust - -sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero: - MOVQ R13, AX - XORQ R14, R14 - MOVQ $-1, R15 - CMPQ R13, $0x03 - CMOVQEQ R14, AX - CMOVQEQ R15, R14 - ADDQ 144(CX)(AX*8), R14 - JNZ sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid - MOVQ $0x00000001, R14 - -sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid: - CMPQ R13, $0x01 - JZ sequenceDecs_decodeSync_safe_amd64_adjust_skip - MOVQ 152(CX), AX - MOVQ AX, 160(CX) - -sequenceDecs_decodeSync_safe_amd64_adjust_skip: - MOVQ 144(CX), AX - MOVQ AX, 152(CX) - MOVQ R14, 144(CX) - MOVQ R14, R13 - -sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVQ R13, 8(SP) - - // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) - MOVQ ctx+16(FP), R14 - SUBQ CX, 104(R14) - JS error_not_enough_literals - CMPQ AX, $0x00020002 - JA sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok - TESTQ AX, AX - JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch - -sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 - ADDQ R10, R14 - CMPQ R14, 32(SP) - JA error_not_enough_space - - // Copy literals - TESTQ AX, AX - JZ check_offset - MOVQ AX, R14 - SUBQ $0x10, R14 - JB copy_1_small - -copy_1_loop: - MOVUPS (R11), X0 - MOVUPS X0, (R10) - ADDQ $0x10, R11 - ADDQ $0x10, R10 - SUBQ $0x10, R14 - JAE copy_1_loop - LEAQ 16(R11)(R14*1), R11 - LEAQ 16(R10)(R14*1), R10 - MOVUPS -16(R11), X0 - MOVUPS X0, -16(R10) - JMP copy_1_end - -copy_1_small: - CMPQ AX, $0x03 - JE copy_1_move_3 - JB copy_1_move_1or2 - CMPQ AX, $0x08 - JB copy_1_move_4through7 - JMP copy_1_move_8through16 - -copy_1_move_1or2: - MOVB (R11), R14 - MOVB -1(R11)(AX*1), R15 - MOVB R14, (R10) - MOVB R15, -1(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end - -copy_1_move_3: - MOVW (R11), R14 - MOVB 2(R11), R15 - MOVW R14, (R10) - MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end - -copy_1_move_4through7: - MOVL (R11), R14 - MOVL -4(R11)(AX*1), R15 - MOVL R14, (R10) - MOVL R15, -4(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end - -copy_1_move_8through16: - MOVQ (R11), R14 - MOVQ -8(R11)(AX*1), R15 - MOVQ R14, (R10) - MOVQ R15, -8(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 - -copy_1_end: - ADDQ AX, R12 - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX - JG error_match_off_too_big - CMPQ CX, 56(SP) - JG error_match_off_too_big - - // Copy match from history - MOVQ CX, AX - SUBQ R12, AX - JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX - JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R10) - ADDQ $0x10, R14 - ADDQ $0x10, R10 - SUBQ $0x10, AX - JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R10) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 - -copy_4_end: - ADDQ R13, R12 - JMP handle_loop - JMP loop_finished - -copy_all_from_history: - MOVQ AX, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R10) - ADDQ $0x10, R14 - ADDQ $0x10, R10 - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(R10)(R15*1), R10 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R10) - JMP copy_5_end - -copy_5_small: - CMPQ AX, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ AX, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP - MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP - MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP - MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 - -copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 - - // Copy match from the current buffer -copy_match: - MOVQ R10, AX - SUBQ CX, AX - - // ml <= mo - CMPQ R13, CX - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R13, CX - SUBQ $0x10, CX - JB copy_2_small - -copy_2_loop: - MOVUPS (AX), X0 - MOVUPS X0, (R10) - ADDQ $0x10, AX - ADDQ $0x10, R10 - SUBQ $0x10, CX - JAE copy_2_loop - LEAQ 16(AX)(CX*1), AX - LEAQ 16(R10)(CX*1), R10 - MOVUPS -16(AX), X0 - MOVUPS X0, -16(R10) - JMP copy_2_end - -copy_2_small: - CMPQ R13, $0x03 - JE copy_2_move_3 - JB copy_2_move_1or2 - CMPQ R13, $0x08 - JB copy_2_move_4through7 - JMP copy_2_move_8through16 - -copy_2_move_1or2: - MOVB (AX), CL - MOVB -1(AX)(R13*1), R14 - MOVB CL, (R10) - MOVB R14, -1(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end - -copy_2_move_3: - MOVW (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) - MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end - -copy_2_move_4through7: - MOVL (AX), CX - MOVL -4(AX)(R13*1), R14 - MOVL CX, (R10) - MOVL R14, -4(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end - -copy_2_move_8through16: - MOVQ (AX), CX - MOVQ -8(AX)(R13*1), R14 - MOVQ CX, (R10) - MOVQ R14, -8(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 - -copy_2_end: - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, R12 - -copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) - JNS sequenceDecs_decodeSync_safe_amd64_main_loop - -loop_finished: - MOVQ br+8(FP), AX - MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) - - // Update the context - MOVQ ctx+16(FP), AX - MOVQ R12, 136(AX) - MOVQ 144(AX), CX - SUBQ CX, R11 - MOVQ R11, 168(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX - MOVQ ctx+16(FP), CX - MOVQ AX, 216(CX) - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: - MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error -error_match_off_too_big: - MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX - MOVQ CX, 224(AX) - MOVQ R12, 136(AX) - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - - // Return with not enough output space error -error_not_enough_space: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ R12, 136(AX) - MOVQ $0x00000005, ret+24(FP) - RET - -// func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 - MOVQ br+8(FP), BX - MOVQ 24(BX), AX - MOVBQZX 32(BX), DX - MOVQ (BX), CX - MOVQ 8(BX), BX - ADDQ BX, CX - MOVQ CX, (SP) - MOVQ ctx+16(FP), CX - MOVQ 72(CX), SI - MOVQ 80(CX), DI - MOVQ 88(CX), R8 - XORQ R9, R9 - MOVQ R9, 8(SP) - MOVQ R9, 16(SP) - MOVQ R9, 24(SP) - MOVQ 112(CX), R9 - MOVQ 128(CX), R10 - MOVQ R10, 32(SP) - MOVQ 144(CX), R10 - MOVQ 136(CX), R11 - MOVQ 200(CX), R12 - MOVQ R12, 56(SP) - MOVQ 176(CX), R12 - MOVQ R12, 48(SP) - MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) - - // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) - - // outBase += outPosition - ADDQ R11, R9 - -sequenceDecs_decodeSync_safe_bmi2_main_loop: - MOVQ (SP), R12 - - // Fill bitreader to have enough for the offset and match length. - CMPQ BX, $0x08 - JL sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R12 - MOVQ (R12), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decodeSync_safe_bmi2_fill_end - -sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_end - SHLQ $0x08, AX - SUBQ $0x01, R12 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R12), CX - ORQ CX, AX - JMP sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte - -sequenceDecs_decodeSync_safe_bmi2_fill_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_safe_bmi2_fill_end: - // Update offset - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ R8, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 8(SP) - - // Update match length - MOVQ $0x00000808, CX - BEXTRQ CX, DI, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ DI, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 16(SP) - - // Fill bitreader to have enough for the remaining - CMPQ BX, $0x08 - JL sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte - MOVQ DX, CX - SHRQ $0x03, CX - SUBQ CX, R12 - MOVQ (R12), AX - SUBQ CX, BX - ANDQ $0x07, DX - JMP sequenceDecs_decodeSync_safe_bmi2_fill_2_end - -sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte: - CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_check_overread - CMPQ DX, $0x07 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_end - SHLQ $0x08, AX - SUBQ $0x01, R12 - SUBQ $0x01, BX - SUBQ $0x08, DX - MOVBQZX (R12), CX - ORQ CX, AX - JMP sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte - -sequenceDecs_decodeSync_safe_bmi2_fill_2_check_overread: - CMPQ DX, $0x40 - JA error_overread - -sequenceDecs_decodeSync_safe_bmi2_fill_2_end: - // Update literal length - MOVQ $0x00000808, CX - BEXTRQ CX, SI, R13 - MOVQ AX, R14 - LEAQ (DX)(R13*1), CX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - MOVQ CX, DX - MOVQ SI, CX - SHRQ $0x20, CX - ADDQ R14, CX - MOVQ CX, 24(SP) - - // Fill bitreader for state updates - MOVQ R12, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_safe_bmi2_skip_update - LEAQ (SI)(DI*1), R13 - ADDQ R8, R13 - MOVBQZX R13, R13 - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - - // Update Offset State - BZHIQ R8, R14, CX - SHRXQ R8, R14, R14 - SHRL $0x10, R8 - ADDQ CX, R8 - - // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 - - // Update Match Length State - BZHIQ DI, R14, CX - SHRXQ DI, R14, R14 - SHRL $0x10, DI - ADDQ CX, DI - - // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX - MOVQ (CX)(DI*8), DI - - // Update Literal Length State - BZHIQ SI, R14, CX - SHRL $0x10, SI - ADDQ CX, SI - - // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI - -sequenceDecs_decodeSync_safe_bmi2_skip_update: - // Adjust offset - MOVQ s+0(FP), CX - MOVQ 8(SP), R13 - CMPQ R12, $0x01 - JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 - MOVUPS 144(CX), X0 - MOVQ R13, 144(CX) - MOVUPS X0, 152(CX) - JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust - -sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 - JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero - INCQ R13 - JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero - -sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero: - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero - MOVQ 144(CX), R13 - JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust - -sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero: - MOVQ R13, R12 - XORQ R14, R14 - MOVQ $-1, R15 - CMPQ R13, $0x03 - CMOVQEQ R14, R12 - CMOVQEQ R15, R14 - ADDQ 144(CX)(R12*8), R14 - JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_temp_valid - MOVQ $0x00000001, R14 - -sequenceDecs_decodeSync_safe_bmi2_adjust_temp_valid: - CMPQ R13, $0x01 - JZ sequenceDecs_decodeSync_safe_bmi2_adjust_skip - MOVQ 152(CX), R12 - MOVQ R12, 160(CX) - -sequenceDecs_decodeSync_safe_bmi2_adjust_skip: - MOVQ 144(CX), R12 - MOVQ R12, 152(CX) - MOVQ R14, 144(CX) - MOVQ R14, R13 - -sequenceDecs_decodeSync_safe_bmi2_after_adjust: - MOVQ R13, 8(SP) - - // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) - MOVQ ctx+16(FP), R14 - SUBQ R12, 104(R14) - JS error_not_enough_literals - CMPQ CX, $0x00020002 - JA sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big - TESTQ R13, R13 - JNZ sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok - TESTQ CX, CX - JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch - -sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 - ADDQ R9, R14 - CMPQ R14, 32(SP) - JA error_not_enough_space - - // Copy literals - TESTQ CX, CX - JZ check_offset - MOVQ CX, R14 - SUBQ $0x10, R14 - JB copy_1_small - -copy_1_loop: - MOVUPS (R10), X0 - MOVUPS X0, (R9) - ADDQ $0x10, R10 - ADDQ $0x10, R9 - SUBQ $0x10, R14 - JAE copy_1_loop - LEAQ 16(R10)(R14*1), R10 - LEAQ 16(R9)(R14*1), R9 - MOVUPS -16(R10), X0 - MOVUPS X0, -16(R9) - JMP copy_1_end - -copy_1_small: - CMPQ CX, $0x03 - JE copy_1_move_3 - JB copy_1_move_1or2 - CMPQ CX, $0x08 - JB copy_1_move_4through7 - JMP copy_1_move_8through16 - -copy_1_move_1or2: - MOVB (R10), R14 - MOVB -1(R10)(CX*1), R15 - MOVB R14, (R9) - MOVB R15, -1(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end - -copy_1_move_3: - MOVW (R10), R14 - MOVB 2(R10), R15 - MOVW R14, (R9) - MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end - -copy_1_move_4through7: - MOVL (R10), R14 - MOVL -4(R10)(CX*1), R15 - MOVL R14, (R9) - MOVL R15, -4(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end - -copy_1_move_8through16: - MOVQ (R10), R14 - MOVQ -8(R10)(CX*1), R15 - MOVQ R14, (R9) - MOVQ R15, -8(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 - -copy_1_end: - ADDQ CX, R11 - - // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX - JG error_match_off_too_big - CMPQ R12, 56(SP) - JG error_match_off_too_big - - // Copy match from history - MOVQ R12, CX - SUBQ R11, CX - JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX - JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX - JB copy_4_small - -copy_4_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R9) - ADDQ $0x10, R14 - ADDQ $0x10, R9 - SUBQ $0x10, CX - JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R9) - JMP copy_4_end - -copy_4_small: - CMPQ R13, $0x03 - JE copy_4_move_3 - CMPQ R13, $0x08 - JB copy_4_move_4through7 - JMP copy_4_move_8through16 - -copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end - -copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end - -copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 - -copy_4_end: - ADDQ R13, R11 - JMP handle_loop - JMP loop_finished - -copy_all_from_history: - MOVQ CX, R15 - SUBQ $0x10, R15 - JB copy_5_small - -copy_5_loop: - MOVUPS (R14), X0 - MOVUPS X0, (R9) - ADDQ $0x10, R14 - ADDQ $0x10, R9 - SUBQ $0x10, R15 - JAE copy_5_loop - LEAQ 16(R14)(R15*1), R14 - LEAQ 16(R9)(R15*1), R9 - MOVUPS -16(R14), X0 - MOVUPS X0, -16(R9) - JMP copy_5_end - -copy_5_small: - CMPQ CX, $0x03 - JE copy_5_move_3 - JB copy_5_move_1or2 - CMPQ CX, $0x08 - JB copy_5_move_4through7 - JMP copy_5_move_8through16 - -copy_5_move_1or2: - MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP - MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_4through7: - MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP - MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end - -copy_5_move_8through16: - MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP - MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 - -copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 - - // Copy match from the current buffer -copy_match: - MOVQ R9, CX - SUBQ R12, CX - - // ml <= mo - CMPQ R13, R12 - JA copy_overlapping_match - - // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R13, R12 - SUBQ $0x10, R12 - JB copy_2_small - -copy_2_loop: - MOVUPS (CX), X0 - MOVUPS X0, (R9) - ADDQ $0x10, CX - ADDQ $0x10, R9 - SUBQ $0x10, R12 - JAE copy_2_loop - LEAQ 16(CX)(R12*1), CX - LEAQ 16(R9)(R12*1), R9 - MOVUPS -16(CX), X0 - MOVUPS X0, -16(R9) - JMP copy_2_end - -copy_2_small: - CMPQ R13, $0x03 - JE copy_2_move_3 - JB copy_2_move_1or2 - CMPQ R13, $0x08 - JB copy_2_move_4through7 - JMP copy_2_move_8through16 - -copy_2_move_1or2: - MOVB (CX), R12 - MOVB -1(CX)(R13*1), R14 - MOVB R12, (R9) - MOVB R14, -1(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end - -copy_2_move_3: - MOVW (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) - MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end - -copy_2_move_4through7: - MOVL (CX), R12 - MOVL -4(CX)(R13*1), R14 - MOVL R12, (R9) - MOVL R14, -4(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end - -copy_2_move_8through16: - MOVQ (CX), R12 - MOVQ -8(CX)(R13*1), R14 - MOVQ R12, (R9) - MOVQ R14, -8(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 - -copy_2_end: - JMP handle_loop - - // Copy overlapping match -copy_overlapping_match: - ADDQ R13, R11 - -copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 - -handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) - JNS sequenceDecs_decodeSync_safe_bmi2_main_loop - -loop_finished: - MOVQ br+8(FP), CX - MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) - - // Update the context - MOVQ ctx+16(FP), AX - MOVQ R11, 136(AX) - MOVQ 144(AX), CX - SUBQ CX, R10 - MOVQ R10, 168(AX) - - // Return success - MOVQ $0x00000000, ret+24(FP) - RET - - // Return with match length error -sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX - MOVQ ctx+16(FP), CX - MOVQ AX, 216(CX) - MOVQ $0x00000001, ret+24(FP) - RET - - // Return with match too long error -sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: - MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ $0x00000002, ret+24(FP) - RET - - // Return with match offset too long error -error_match_off_too_big: - MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX - MOVQ CX, 224(AX) - MOVQ R11, 136(AX) - MOVQ $0x00000003, ret+24(FP) - RET - - // Return with not enough literals error -error_not_enough_literals: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ $0x00000004, ret+24(FP) - RET - - // Return with overread error -error_overread: - MOVQ $0x00000006, ret+24(FP) - RET - - // Return with not enough output space error -error_not_enough_space: - MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX - MOVQ CX, 208(AX) - MOVQ 16(SP), CX - MOVQ CX, 216(AX) - MOVQ R11, 136(AX) - MOVQ $0x00000005, ret+24(FP) - RET diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go deleted file mode 100644 index 2fb35b788..000000000 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ /dev/null @@ -1,237 +0,0 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - -package zstd - -import ( - "fmt" - "io" -) - -// decode sequences from the stream with the provided history but without dictionary. -func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { - return false, nil -} - -// decode sequences from the stream without the provided history. -func (s *sequenceDecs) decode(seqs []seqVals) error { - br := s.br - - // Grab full sizes tables, to avoid bounds checks. - llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] - llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state - s.seqSize = 0 - litRemain := len(s.literals) - - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } - for i := range seqs { - var ll, mo, ml int - if len(br.in) > 4+((maxOffsetBits+16+16)>>3) { - // inlined function: - // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) - - // Final will not read from stream. - var llB, mlB, moB uint8 - ll, llB = llState.final() - ml, mlB = mlState.final() - mo, moB = ofState.final() - - // extra bits are stored in reverse order. - br.fillFast() - mo += br.getBits(moB) - if s.maxBits > 32 { - br.fillFast() - } - ml += br.getBits(mlB) - ll += br.getBits(llB) - - if moB > 1 { - s.prevOffset[2] = s.prevOffset[1] - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = mo - } else { - // mo = s.adjustOffset(mo, ll, moB) - // Inlined for rather big speedup - if ll == 0 { - // There is an exception though, when current sequence's literals_length = 0. - // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, - // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. - mo++ - } - - if mo == 0 { - mo = s.prevOffset[0] - } else { - var temp int - if mo == 3 { - temp = s.prevOffset[0] - 1 - } else { - temp = s.prevOffset[mo] - } - - if temp == 0 { - // 0 is not valid; input is corrupted; force offset to 1 - println("WARNING: temp was 0") - temp = 1 - } - - if mo != 1 { - s.prevOffset[2] = s.prevOffset[1] - } - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = temp - mo = temp - } - } - br.fillFast() - } else { - if br.overread() { - if debugDecoder { - printf("reading sequence %d, exceeded available data\n", i) - } - return io.ErrUnexpectedEOF - } - ll, mo, ml = s.next(br, llState, mlState, ofState) - br.fill() - } - - if debugSequences { - println("Seq", i, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml) - } - // Evaluate. - // We might be doing this async, so do it early. - if mo == 0 && ml > 0 { - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - } - if ml > maxMatchLen { - return fmt.Errorf("match len (%d) bigger than max allowed length", ml) - } - s.seqSize += ll + ml - if s.seqSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - litRemain -= ll - if litRemain < 0 { - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, litRemain+ll) - } - seqs[i] = seqVals{ - ll: ll, - ml: ml, - mo: mo, - } - if i == len(seqs)-1 { - // This is the last sequence, so we shouldn't update state. - break - } - - // Manually inlined, ~ 5-20% faster - // Update all 3 states at once. Approx 20% faster. - nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits() - if nBits == 0 { - llState = llTable[llState.newState()&maxTableMask] - mlState = mlTable[mlState.newState()&maxTableMask] - ofState = ofTable[ofState.newState()&maxTableMask] - } else { - bits := br.get32BitsFast(nBits) - lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) - llState = llTable[(llState.newState()+lowBits)&maxTableMask] - - lowBits = uint16(bits >> (ofState.nbBits() & 31)) - lowBits &= bitMask[mlState.nbBits()&15] - mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask] - - lowBits = uint16(bits) & bitMask[ofState.nbBits()&15] - ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask] - } - } - s.seqSize += litRemain - if s.seqSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - } - return err -} - -// executeSimple handles cases when a dictionary is not used. -func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { - // Ensure we have enough output size... - if len(s.out)+s.seqSize > cap(s.out) { - addBytes := s.seqSize + len(s.out) - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] - } - - if debugDecoder { - printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) - } - - var t = len(s.out) - out := s.out[:t+s.seqSize] - - for _, seq := range seqs { - // Add literals - copy(out[t:], s.literals[:seq.ll]) - t += seq.ll - s.literals = s.literals[seq.ll:] - - // Malformed input - if seq.mo > t+len(hist) || seq.mo > s.windowSize { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist)) - } - - // Copy from history. - if v := seq.mo - t; v > 0 { - // v is the start position in history from end. - start := len(hist) - v - if seq.ml > v { - // Some goes into the current block. - // Copy remainder of history - copy(out[t:], hist[start:]) - t += v - seq.ml -= v - } else { - copy(out[t:], hist[start:start+seq.ml]) - t += seq.ml - continue - } - } - - // We must be in the current buffer now - if seq.ml > 0 { - start := t - seq.mo - if seq.ml <= t-start { - // No overlap - copy(out[t:], out[start:start+seq.ml]) - t += seq.ml - } else { - // Overlapping copy - // Extend destination slice and copy one byte at the time. - src := out[start : start+seq.ml] - dst := out[t:] - dst = dst[:len(src)] - t += len(src) - // Destination is the space we just added. - for i := range src { - dst[i] = src[i] - } - } - } - } - // Add final literals - copy(out[t:], s.literals) - if debugDecoder { - t += len(s.literals) - if t != len(out) { - panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) - } - } - s.out = out - - return nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqenc.go b/vendor/github.com/klauspost/compress/zstd/seqenc.go deleted file mode 100644 index 8014174a7..000000000 --- a/vendor/github.com/klauspost/compress/zstd/seqenc.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import "math/bits" - -type seqCoders struct { - llEnc, ofEnc, mlEnc *fseEncoder - llPrev, ofPrev, mlPrev *fseEncoder -} - -// swap coders with another (block). -func (s *seqCoders) swap(other *seqCoders) { - *s, *other = *other, *s -} - -// setPrev will update the previous encoders to the actually used ones -// and make sure a fresh one is in the main slot. -func (s *seqCoders) setPrev(ll, ml, of *fseEncoder) { - compareSwap := func(used *fseEncoder, current, prev **fseEncoder) { - // We used the new one, more current to history and reuse the previous history - if *current == used { - *prev, *current = *current, *prev - c := *current - p := *prev - c.reUsed = false - p.reUsed = true - return - } - if used == *prev { - return - } - // Ensure we cannot reuse by accident - prevEnc := *prev - prevEnc.symbolLen = 0 - } - compareSwap(ll, &s.llEnc, &s.llPrev) - compareSwap(ml, &s.mlEnc, &s.mlPrev) - compareSwap(of, &s.ofEnc, &s.ofPrev) -} - -func highBit(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} - -var llCodeTable = [64]byte{0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24} - -// Up to 6 bits -const maxLLCode = 35 - -// llBitsTable translates from ll code to number of bits. -var llBitsTable = [maxLLCode + 1]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 16} - -// llCode returns the code that represents the literal length requested. -func llCode(litLength uint32) uint8 { - const llDeltaCode = 19 - if litLength <= 63 { - // Compiler insists on bounds check (Go 1.12) - return llCodeTable[litLength&63] - } - return uint8(highBit(litLength)) + llDeltaCode -} - -var mlCodeTable = [128]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42} - -// Up to 6 bits -const maxMLCode = 52 - -// mlBitsTable translates from ml code to number of bits. -var mlBitsTable = [maxMLCode + 1]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 16} - -// note : mlBase = matchLength - MINMATCH; -// because it's the format it's stored in seqStore->sequences -func mlCode(mlBase uint32) uint8 { - const mlDeltaCode = 36 - if mlBase <= 127 { - // Compiler insists on bounds check (Go 1.12) - return mlCodeTable[mlBase&127] - } - return uint8(highBit(mlBase)) + mlDeltaCode -} - -func ofCode(offset uint32) uint8 { - // A valid offset will always be > 0. - return uint8(bits.Len32(offset) - 1) -} diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go deleted file mode 100644 index ec13594e8..000000000 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -import ( - "encoding/binary" - "errors" - "hash/crc32" - "io" - - "github.com/klauspost/compress/huff0" - snappy "github.com/klauspost/compress/internal/snapref" -) - -const ( - snappyTagLiteral = 0x00 - snappyTagCopy1 = 0x01 - snappyTagCopy2 = 0x02 - snappyTagCopy4 = 0x03 -) - -const ( - snappyChecksumSize = 4 - snappyMagicBody = "sNaPpY" - - // snappyMaxBlockSize is the maximum size of the input to encodeBlock. It is not - // part of the wire format per se, but some parts of the encoder assume - // that an offset fits into a uint16. - // - // Also, for the framing format (Writer type instead of Encode function), - // https://github.com/google/snappy/blob/master/framing_format.txt says - // that "the uncompressed data in a chunk must be no longer than 65536 - // bytes". - snappyMaxBlockSize = 65536 - - // snappyMaxEncodedLenOfMaxBlockSize equals MaxEncodedLen(snappyMaxBlockSize), but is - // hard coded to be a const instead of a variable, so that obufLen can also - // be a const. Their equivalence is confirmed by - // TestMaxEncodedLenOfMaxBlockSize. - snappyMaxEncodedLenOfMaxBlockSize = 76490 -) - -const ( - chunkTypeCompressedData = 0x00 - chunkTypeUncompressedData = 0x01 - chunkTypePadding = 0xfe - chunkTypeStreamIdentifier = 0xff -) - -var ( - // ErrSnappyCorrupt reports that the input is invalid. - ErrSnappyCorrupt = errors.New("snappy: corrupt input") - // ErrSnappyTooLarge reports that the uncompressed length is too large. - ErrSnappyTooLarge = errors.New("snappy: decoded block is too large") - // ErrSnappyUnsupported reports that the input isn't supported. - ErrSnappyUnsupported = errors.New("snappy: unsupported input") - - errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") -) - -// SnappyConverter can read SnappyConverter-compressed streams and convert them to zstd. -// Conversion is done by converting the stream directly from Snappy without intermediate -// full decoding. -// Therefore the compression ratio is much less than what can be done by a full decompression -// and compression, and a faulty Snappy stream may lead to a faulty Zstandard stream without -// any errors being generated. -// No CRC value is being generated and not all CRC values of the Snappy stream are checked. -// However, it provides really fast recompression of Snappy streams. -// The converter can be reused to avoid allocations, even after errors. -type SnappyConverter struct { - r io.Reader - err error - buf []byte - block *blockEnc -} - -// Convert the Snappy stream supplied in 'in' and write the zStandard stream to 'w'. -// If any error is detected on the Snappy stream it is returned. -// The number of bytes written is returned. -func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { - initPredefined() - r.err = nil - r.r = in - if r.block == nil { - r.block = &blockEnc{} - r.block.init() - } - r.block.initNewEncode() - if len(r.buf) != snappyMaxEncodedLenOfMaxBlockSize+snappyChecksumSize { - r.buf = make([]byte, snappyMaxEncodedLenOfMaxBlockSize+snappyChecksumSize) - } - r.block.litEnc.Reuse = huff0.ReusePolicyNone - var written int64 - var readHeader bool - { - header := frameHeader{WindowSize: snappyMaxBlockSize}.appendTo(r.buf[:0]) - - var n int - n, r.err = w.Write(header) - if r.err != nil { - return written, r.err - } - written += int64(n) - } - - for { - if !r.readFull(r.buf[:4], true) { - // Add empty last block - r.block.reset(nil) - r.block.last = true - err := r.block.encodeLits(r.block.literals, false) - if err != nil { - return written, err - } - n, err := w.Write(r.block.output) - if err != nil { - return written, err - } - written += int64(n) - - return written, r.err - } - chunkType := r.buf[0] - if !readHeader { - if chunkType != chunkTypeStreamIdentifier { - println("chunkType != chunkTypeStreamIdentifier", chunkType) - r.err = ErrSnappyCorrupt - return written, r.err - } - readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - if chunkLen > len(r.buf) { - println("chunkLen > len(r.buf)", chunkType) - r.err = ErrSnappyUnsupported - return written, r.err - } - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < snappyChecksumSize { - println("chunkLen < snappyChecksumSize", chunkLen, snappyChecksumSize) - r.err = ErrSnappyCorrupt - return written, r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf, false) { - return written, r.err - } - //checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[snappyChecksumSize:] - - n, hdr, err := snappyDecodedLen(buf) - if err != nil { - r.err = err - return written, r.err - } - buf = buf[hdr:] - if n > snappyMaxBlockSize { - println("n > snappyMaxBlockSize", n, snappyMaxBlockSize) - r.err = ErrSnappyCorrupt - return written, r.err - } - r.block.reset(nil) - r.block.pushOffsets() - if err := decodeSnappy(r.block, buf); err != nil { - r.err = err - return written, r.err - } - if r.block.size+r.block.extraLits != n { - printf("invalid size, want %d, got %d\n", n, r.block.size+r.block.extraLits) - r.err = ErrSnappyCorrupt - return written, r.err - } - err = r.block.encode(nil, false, false) - switch err { - case errIncompressible: - r.block.popOffsets() - r.block.reset(nil) - r.block.literals, err = snappy.Decode(r.block.literals[:n], r.buf[snappyChecksumSize:chunkLen]) - if err != nil { - return written, err - } - err = r.block.encodeLits(r.block.literals, false) - if err != nil { - return written, err - } - case nil: - default: - return written, err - } - - n, r.err = w.Write(r.block.output) - if r.err != nil { - return written, err - } - written += int64(n) - continue - case chunkTypeUncompressedData: - if debugEncoder { - println("Uncompressed, chunklen", chunkLen) - } - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < snappyChecksumSize { - println("chunkLen < snappyChecksumSize", chunkLen, snappyChecksumSize) - r.err = ErrSnappyCorrupt - return written, r.err - } - r.block.reset(nil) - buf := r.buf[:snappyChecksumSize] - if !r.readFull(buf, false) { - return written, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n := chunkLen - snappyChecksumSize - if n > snappyMaxBlockSize { - println("n > snappyMaxBlockSize", n, snappyMaxBlockSize) - r.err = ErrSnappyCorrupt - return written, r.err - } - r.block.literals = r.block.literals[:n] - if !r.readFull(r.block.literals, false) { - return written, r.err - } - if snappyCRC(r.block.literals) != checksum { - println("literals crc mismatch") - r.err = ErrSnappyCorrupt - return written, r.err - } - err := r.block.encodeLits(r.block.literals, false) - if err != nil { - return written, err - } - n, r.err = w.Write(r.block.output) - if r.err != nil { - return written, err - } - written += int64(n) - continue - - case chunkTypeStreamIdentifier: - if debugEncoder { - println("stream id", chunkLen, len(snappyMagicBody)) - } - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(snappyMagicBody) { - println("chunkLen != len(snappyMagicBody)", chunkLen, len(snappyMagicBody)) - r.err = ErrSnappyCorrupt - return written, r.err - } - if !r.readFull(r.buf[:len(snappyMagicBody)], false) { - return written, r.err - } - for i := 0; i < len(snappyMagicBody); i++ { - if r.buf[i] != snappyMagicBody[i] { - println("r.buf[i] != snappyMagicBody[i]", r.buf[i], snappyMagicBody[i], i) - r.err = ErrSnappyCorrupt - return written, r.err - } - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - println("chunkType <= 0x7f") - r.err = ErrSnappyUnsupported - return written, r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if !r.readFull(r.buf[:chunkLen], false) { - return written, r.err - } - } -} - -// decodeSnappy writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read. -func decodeSnappy(blk *blockEnc, src []byte) error { - //decodeRef(make([]byte, snappyMaxBlockSize), src) - var s, length int - lits := blk.extraLits - var offset uint32 - for s < len(src) { - switch src[s] & 0x03 { - case snappyTagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, src) - return ErrSnappyCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, src) - return ErrSnappyCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, src) - return ErrSnappyCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, src) - return ErrSnappyCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - if x > snappyMaxBlockSize { - println("x > snappyMaxBlockSize", x, snappyMaxBlockSize) - return ErrSnappyCorrupt - } - length = int(x) + 1 - if length <= 0 { - println("length <= 0 ", length) - - return errUnsupportedLiteralLength - } - //if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s { - // return ErrSnappyCorrupt - //} - - blk.literals = append(blk.literals, src[s:s+length]...) - //println(length, "litLen") - lits += length - s += length - continue - - case snappyTagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, len(src)) - return ErrSnappyCorrupt - } - length = 4 + int(src[s-2])>>2&0x7 - offset = uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]) - - case snappyTagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, len(src)) - return ErrSnappyCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = uint32(src[s-2]) | uint32(src[s-1])<<8 - - case snappyTagCopy4: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - println("uint(s) > uint(len(src)", s, len(src)) - return ErrSnappyCorrupt - } - length = 1 + int(src[s-5])>>2 - offset = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - - if offset <= 0 || blk.size+lits < int(offset) /*|| length > len(blk)-d */ { - println("offset <= 0 || blk.size+lits < int(offset)", offset, blk.size+lits, int(offset), blk.size, lits) - - return ErrSnappyCorrupt - } - - // Check if offset is one of the recent offsets. - // Adjusts the output offset accordingly. - // Gives a tiny bit of compression, typically around 1%. - if false { - offset = blk.matchOffset(offset, uint32(lits)) - } else { - offset += 3 - } - - blk.sequences = append(blk.sequences, seq{ - litLen: uint32(lits), - offset: offset, - matchLen: uint32(length) - zstdMinMatch, - }) - blk.size += length + lits - lits = 0 - } - blk.extraLits = lits - return nil -} - -func (r *SnappyConverter) readFull(p []byte, allowEOF bool) (ok bool) { - if _, r.err = io.ReadFull(r.r, p); r.err != nil { - if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrSnappyCorrupt - } - return false - } - return true -} - -var crcTable = crc32.MakeTable(crc32.Castagnoli) - -// crc implements the checksum specified in section 3 of -// https://github.com/google/snappy/blob/master/framing_format.txt -func snappyCRC(b []byte) uint32 { - c := crc32.Update(0, crcTable, b) - return c>>15 | c<<17 + 0xa282ead8 -} - -// snappyDecodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func snappyDecodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { - return 0, 0, ErrSnappyCorrupt - } - - const wordSize = 32 << (^uint(0) >> 32 & 1) - if wordSize == 32 && v > 0x7fffffff { - return 0, 0, ErrSnappyTooLarge - } - return int(v), n, nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go deleted file mode 100644 index 29c15c8c4..000000000 --- a/vendor/github.com/klauspost/compress/zstd/zip.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package zstd - -import ( - "errors" - "io" - "sync" -) - -// ZipMethodWinZip is the method for Zstandard compressed data inside Zip files for WinZip. -// See https://www.winzip.com/win/en/comp_info.html -const ZipMethodWinZip = 93 - -// ZipMethodPKWare is the original method number used by PKWARE to indicate Zstandard compression. -// Deprecated: This has been deprecated by PKWARE, use ZipMethodWinZip instead for compression. -// See https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT -const ZipMethodPKWare = 20 - -// zipReaderPool is the default reader pool. -var zipReaderPool = sync.Pool{New: func() interface{} { - z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1)) - if err != nil { - panic(err) - } - return z -}} - -// newZipReader creates a pooled zip decompressor. -func newZipReader(opts ...DOption) func(r io.Reader) io.ReadCloser { - pool := &zipReaderPool - if len(opts) > 0 { - opts = append([]DOption{WithDecoderLowmem(true), WithDecoderMaxWindow(128 << 20)}, opts...) - // Force concurrency 1 - opts = append(opts, WithDecoderConcurrency(1)) - // Create our own pool - pool = &sync.Pool{} - } - return func(r io.Reader) io.ReadCloser { - dec, ok := pool.Get().(*Decoder) - if ok { - dec.Reset(r) - } else { - d, err := NewReader(r, opts...) - if err != nil { - panic(err) - } - dec = d - } - return &pooledZipReader{dec: dec, pool: pool} - } -} - -type pooledZipReader struct { - mu sync.Mutex // guards Close and Read - pool *sync.Pool - dec *Decoder -} - -func (r *pooledZipReader) Read(p []byte) (n int, err error) { - r.mu.Lock() - defer r.mu.Unlock() - if r.dec == nil { - return 0, errors.New("read after close or EOF") - } - dec, err := r.dec.Read(p) - if err == io.EOF { - r.dec.Reset(nil) - r.pool.Put(r.dec) - r.dec = nil - } - return dec, err -} - -func (r *pooledZipReader) Close() error { - r.mu.Lock() - defer r.mu.Unlock() - var err error - if r.dec != nil { - err = r.dec.Reset(nil) - r.pool.Put(r.dec) - r.dec = nil - } - return err -} - -type pooledZipWriter struct { - mu sync.Mutex // guards Close and Read - enc *Encoder - pool *sync.Pool -} - -func (w *pooledZipWriter) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() - if w.enc == nil { - return 0, errors.New("Write after Close") - } - return w.enc.Write(p) -} - -func (w *pooledZipWriter) Close() error { - w.mu.Lock() - defer w.mu.Unlock() - var err error - if w.enc != nil { - err = w.enc.Close() - w.pool.Put(w.enc) - w.enc = nil - } - return err -} - -// ZipCompressor returns a compressor that can be registered with zip libraries. -// The provided encoder options will be used on all encodes. -func ZipCompressor(opts ...EOption) func(w io.Writer) (io.WriteCloser, error) { - var pool sync.Pool - return func(w io.Writer) (io.WriteCloser, error) { - enc, ok := pool.Get().(*Encoder) - if ok { - enc.Reset(w) - } else { - var err error - enc, err = NewWriter(w, opts...) - if err != nil { - return nil, err - } - } - return &pooledZipWriter{enc: enc, pool: &pool}, nil - } -} - -// ZipDecompressor returns a decompressor that can be registered with zip libraries. -// See ZipCompressor for example. -// Options can be specified. WithDecoderConcurrency(1) is forced, -// and by default a 128MB maximum decompression window is specified. -// The window size can be overridden if required. -func ZipDecompressor(opts ...DOption) func(r io.Reader) io.ReadCloser { - return newZipReader(opts...) -} diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go deleted file mode 100644 index 066bef2a4..000000000 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ /dev/null @@ -1,125 +0,0 @@ -// Package zstd provides decompression of zstandard files. -// -// For advanced usage and examples, go to the README: https://github.com/klauspost/compress/tree/master/zstd#zstd -package zstd - -import ( - "bytes" - "encoding/binary" - "errors" - "log" - "math" -) - -// enable debug printing -const debug = false - -// enable encoding debug printing -const debugEncoder = debug - -// enable decoding debug printing -const debugDecoder = debug - -// Enable extra assertions. -const debugAsserts = debug || false - -// print sequence details -const debugSequences = false - -// print detailed matching information -const debugMatches = false - -// force encoder to use predefined tables. -const forcePreDef = false - -// zstdMinMatch is the minimum zstd match length. -const zstdMinMatch = 3 - -// fcsUnknown is used for unknown frame content size. -const fcsUnknown = math.MaxUint64 - -var ( - // ErrReservedBlockType is returned when a reserved block type is found. - // Typically this indicates wrong or corrupted input. - ErrReservedBlockType = errors.New("invalid input: reserved block type encountered") - - // ErrCompressedSizeTooBig is returned when a block is bigger than allowed. - // Typically this indicates wrong or corrupted input. - ErrCompressedSizeTooBig = errors.New("invalid input: compressed size too big") - - // ErrBlockTooSmall is returned when a block is too small to be decoded. - // Typically returned on invalid input. - ErrBlockTooSmall = errors.New("block too small") - - // ErrUnexpectedBlockSize is returned when a block has unexpected size. - // Typically returned on invalid input. - ErrUnexpectedBlockSize = errors.New("unexpected block size") - - // ErrMagicMismatch is returned when a "magic" number isn't what is expected. - // Typically this indicates wrong or corrupted input. - ErrMagicMismatch = errors.New("invalid input: magic number mismatch") - - // ErrWindowSizeExceeded is returned when a reference exceeds the valid window size. - // Typically this indicates wrong or corrupted input. - ErrWindowSizeExceeded = errors.New("window size exceeded") - - // ErrWindowSizeTooSmall is returned when no window size is specified. - // Typically this indicates wrong or corrupted input. - ErrWindowSizeTooSmall = errors.New("invalid input: window size was too small") - - // ErrDecoderSizeExceeded is returned if decompressed size exceeds the configured limit. - ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit") - - // ErrUnknownDictionary is returned if the dictionary ID is unknown. - ErrUnknownDictionary = errors.New("unknown dictionary") - - // ErrFrameSizeExceeded is returned if the stated frame size is exceeded. - // This is only returned if SingleSegment is specified on the frame. - ErrFrameSizeExceeded = errors.New("frame size exceeded") - - // ErrFrameSizeMismatch is returned if the stated frame size does not match the expected size. - // This is only returned if SingleSegment is specified on the frame. - ErrFrameSizeMismatch = errors.New("frame size does not match size on stream") - - // ErrCRCMismatch is returned if CRC mismatches. - ErrCRCMismatch = errors.New("CRC check failed") - - // ErrDecoderClosed will be returned if the Decoder was used after - // Close has been called. - ErrDecoderClosed = errors.New("decoder used after Close") - - // ErrEncoderClosed will be returned if the Encoder was used after - // Close has been called. - ErrEncoderClosed = errors.New("encoder used after Close") - - // ErrDecoderNilInput is returned when a nil Reader was provided - // and an operation other than Reset/DecodeAll/Close was attempted. - ErrDecoderNilInput = errors.New("nil input provided as reader") -) - -func println(a ...interface{}) { - if debug || debugDecoder || debugEncoder { - log.Println(a...) - } -} - -func printf(format string, a ...interface{}) { - if debug || debugDecoder || debugEncoder { - log.Printf(format, a...) - } -} - -func load3232(b []byte, i int32) uint32 { - return binary.LittleEndian.Uint32(b[:len(b):len(b)][i:]) -} - -func load6432(b []byte, i int32) uint64 { - return binary.LittleEndian.Uint64(b[:len(b):len(b)][i:]) -} - -type byter interface { - Bytes() []byte - Len() int -} - -var _ byter = &bytes.Buffer{} diff --git a/vendor/github.com/klauspost/cpuid/v2/.gitignore b/vendor/github.com/klauspost/cpuid/v2/.gitignore deleted file mode 100644 index daf913b1b..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/vendor/github.com/klauspost/cpuid/v2/.goreleaser.yml b/vendor/github.com/klauspost/cpuid/v2/.goreleaser.yml deleted file mode 100644 index 944cc0007..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/.goreleaser.yml +++ /dev/null @@ -1,74 +0,0 @@ -# This is an example goreleaser.yaml file with some sane defaults. -# Make sure to check the documentation at http://goreleaser.com - -builds: - - - id: "cpuid" - binary: cpuid - main: ./cmd/cpuid/main.go - env: - - CGO_ENABLED=0 - flags: - - -ldflags=-s -w - goos: - - aix - - linux - - freebsd - - netbsd - - windows - - darwin - goarch: - - 386 - - amd64 - - arm64 - goarm: - - 7 - -archives: - - - id: cpuid - name_template: "cpuid-{{ .Os }}_{{ .Arch }}_{{ .Version }}" - replacements: - aix: AIX - darwin: OSX - linux: Linux - windows: Windows - 386: i386 - amd64: x86_64 - freebsd: FreeBSD - netbsd: NetBSD - format_overrides: - - goos: windows - format: zip - files: - - LICENSE -checksum: - name_template: 'checksums.txt' -snapshot: - name_template: "{{ .Tag }}-next" -changelog: - sort: asc - filters: - exclude: - - '^doc:' - - '^docs:' - - '^test:' - - '^tests:' - - '^Update\sREADME.md' - -nfpms: - - - file_name_template: "cpuid_package_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - vendor: Klaus Post - homepage: https://github.com/klauspost/cpuid - maintainer: Klaus Post <klauspost@gmail.com> - description: CPUID Tool - license: BSD 3-Clause - formats: - - deb - - rpm - replacements: - darwin: Darwin - linux: Linux - freebsd: FreeBSD - amd64: x86_64 diff --git a/vendor/github.com/klauspost/cpuid/v2/CONTRIBUTING.txt b/vendor/github.com/klauspost/cpuid/v2/CONTRIBUTING.txt deleted file mode 100644 index 2ef4714f7..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/CONTRIBUTING.txt +++ /dev/null @@ -1,35 +0,0 @@ -Developer Certificate of Origin
-Version 1.1
-
-Copyright (C) 2015- Klaus Post & Contributors.
-Email: klauspost@gmail.com
-
-Everyone is permitted to copy and distribute verbatim copies of this
-license document, but changing it is not allowed.
-
-
-Developer's Certificate of Origin 1.1
-
-By making a contribution to this project, I certify that:
-
-(a) The contribution was created in whole or in part by me and I
- have the right to submit it under the open source license
- indicated in the file; or
-
-(b) The contribution is based upon previous work that, to the best
- of my knowledge, is covered under an appropriate open source
- license and I have the right under that license to submit that
- work with modifications, whether created in whole or in part
- by me, under the same open source license (unless I am
- permitted to submit under a different license), as indicated
- in the file; or
-
-(c) The contribution was provided directly to me by some other
- person who certified (a), (b) or (c) and I have not modified
- it.
-
-(d) I understand and agree that this project and the contribution
- are public and that a record of the contribution (including all
- personal information I submit with it, including my sign-off) is
- maintained indefinitely and may be redistributed consistent with
- this project or the open source license(s) involved.
diff --git a/vendor/github.com/klauspost/cpuid/v2/LICENSE b/vendor/github.com/klauspost/cpuid/v2/LICENSE deleted file mode 100644 index 5cec7ee94..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Klaus Post - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/klauspost/cpuid/v2/README.md b/vendor/github.com/klauspost/cpuid/v2/README.md deleted file mode 100644 index f06ba51c5..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/README.md +++ /dev/null @@ -1,499 +0,0 @@ -# cpuid -Package cpuid provides information about the CPU running the current program. - -CPU features are detected on startup, and kept for fast access through the life of the application. -Currently x86 / x64 (AMD64/i386) and ARM (ARM64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. - -You can access the CPU information by accessing the shared CPU variable of the cpuid library. - -Package home: https://github.com/klauspost/cpuid - -[](https://pkg.go.dev/github.com/klauspost/cpuid/v2) -[](https://github.com/klauspost/cpuid/actions/workflows/go.yml) - -## installing - -`go get -u github.com/klauspost/cpuid/v2` using modules. -Drop `v2` for others. - -Installing binary: - -`go install github.com/klauspost/cpuid/v2/cmd/cpuid@latest` - -Or download binaries from release page: https://github.com/klauspost/cpuid/releases - -### Homebrew - -For macOS/Linux users, you can install via [brew](https://brew.sh/) - -```sh -$ brew install cpuid -``` - -## example - -```Go -package main - -import ( - "fmt" - "strings" - - . "github.com/klauspost/cpuid/v2" -) - -func main() { - // Print basic CPU information: - fmt.Println("Name:", CPU.BrandName) - fmt.Println("PhysicalCores:", CPU.PhysicalCores) - fmt.Println("ThreadsPerCore:", CPU.ThreadsPerCore) - fmt.Println("LogicalCores:", CPU.LogicalCores) - fmt.Println("Family", CPU.Family, "Model:", CPU.Model, "Vendor ID:", CPU.VendorID) - fmt.Println("Features:", strings.Join(CPU.FeatureSet(), ",")) - fmt.Println("Cacheline bytes:", CPU.CacheLine) - fmt.Println("L1 Data Cache:", CPU.Cache.L1D, "bytes") - fmt.Println("L1 Instruction Cache:", CPU.Cache.L1I, "bytes") - fmt.Println("L2 Cache:", CPU.Cache.L2, "bytes") - fmt.Println("L3 Cache:", CPU.Cache.L3, "bytes") - fmt.Println("Frequency", CPU.Hz, "hz") - - // Test if we have these specific features: - if CPU.Supports(SSE, SSE2) { - fmt.Println("We have Streaming SIMD 2 Extensions") - } -} -``` - -Sample output: -``` ->go run main.go -Name: AMD Ryzen 9 3950X 16-Core Processor -PhysicalCores: 16 -ThreadsPerCore: 2 -LogicalCores: 32 -Family 23 Model: 113 Vendor ID: AMD -Features: ADX,AESNI,AVX,AVX2,BMI1,BMI2,CLMUL,CMOV,CX16,F16C,FMA3,HTT,HYPERVISOR,LZCNT,MMX,MMXEXT,NX,POPCNT,RDRAND,RDSEED,RDTSCP,SHA,SSE,SSE2,SSE3,SSE4,SSE42,SSE4A,SSSE3 -Cacheline bytes: 64 -L1 Data Cache: 32768 bytes -L1 Instruction Cache: 32768 bytes -L2 Cache: 524288 bytes -L3 Cache: 16777216 bytes -Frequency 0 hz -We have Streaming SIMD 2 Extensions -``` - -# usage - -The `cpuid.CPU` provides access to CPU features. Use `cpuid.CPU.Supports()` to check for CPU features. -A faster `cpuid.CPU.Has()` is provided which will usually be inlined by the gc compiler. - -To test a larger number of features, they can be combined using `f := CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2)`, etc. -This can be using with `cpuid.CPU.HasAll(f)` to quickly test if all features are supported. - -Note that for some cpu/os combinations some features will not be detected. -`amd64` has rather good support and should work reliably on all platforms. - -Note that hypervisors may not pass through all CPU features through to the guest OS, -so even if your host supports a feature it may not be visible on guests. - -## arm64 feature detection - -Not all operating systems provide ARM features directly -and there is no safe way to do so for the rest. - -Currently `arm64/linux` and `arm64/freebsd` should be quite reliable. -`arm64/darwin` adds features expected from the M1 processor, but a lot remains undetected. - -A `DetectARM()` can be used if you are able to control your deployment, -it will detect CPU features, but may crash if the OS doesn't intercept the calls. -A `-cpu.arm` flag for detecting unsafe ARM features can be added. See below. - -Note that currently only features are detected on ARM, -no additional information is currently available. - -## flags - -It is possible to add flags that affects cpu detection. - -For this the `Flags()` command is provided. - -This must be called *before* `flag.Parse()` AND after the flags have been parsed `Detect()` must be called. - -This means that any detection used in `init()` functions will not contain these flags. - -Example: - -```Go -package main - -import ( - "flag" - "fmt" - "strings" - - "github.com/klauspost/cpuid/v2" -) - -func main() { - cpuid.Flags() - flag.Parse() - cpuid.Detect() - - // Test if we have these specific features: - if cpuid.CPU.Supports(cpuid.SSE, cpuid.SSE2) { - fmt.Println("We have Streaming SIMD 2 Extensions") - } -} -``` - -## commandline - -Download as binary from: https://github.com/klauspost/cpuid/releases - -Install from source: - -`go install github.com/klauspost/cpuid/v2/cmd/cpuid@latest` - -### Example - -``` -λ cpuid -Name: AMD Ryzen 9 3950X 16-Core Processor -Vendor String: AuthenticAMD -Vendor ID: AMD -PhysicalCores: 16 -Threads Per Core: 2 -Logical Cores: 32 -CPU Family 23 Model: 113 -Features: ADX,AESNI,AVX,AVX2,BMI1,BMI2,CLMUL,CLZERO,CMOV,CMPXCHG8,CPBOOST,CX16,F16C,FMA3,FXSR,FXSROPT,HTT,HYPERVISOR,LAHF,LZCNT,MCAOVERFLOW,MMX,MMXEXT,MOVBE,NX,OSXSAVE,POPCNT,RDRAND,RDSEED,RDTSCP,SCE,SHA,SSE,SSE2,SSE3,SSE4,SSE42,SSE4A,SSSE3,SUCCOR,X87,XSAVE -Microarchitecture level: 3 -Cacheline bytes: 64 -L1 Instruction Cache: 32768 bytes -L1 Data Cache: 32768 bytes -L2 Cache: 524288 bytes -L3 Cache: 16777216 bytes - -``` -### JSON Output: - -``` -λ cpuid --json -{ - "BrandName": "AMD Ryzen 9 3950X 16-Core Processor", - "VendorID": 2, - "VendorString": "AuthenticAMD", - "PhysicalCores": 16, - "ThreadsPerCore": 2, - "LogicalCores": 32, - "Family": 23, - "Model": 113, - "CacheLine": 64, - "Hz": 0, - "BoostFreq": 0, - "Cache": { - "L1I": 32768, - "L1D": 32768, - "L2": 524288, - "L3": 16777216 - }, - "SGX": { - "Available": false, - "LaunchControl": false, - "SGX1Supported": false, - "SGX2Supported": false, - "MaxEnclaveSizeNot64": 0, - "MaxEnclaveSize64": 0, - "EPCSections": null - }, - "Features": [ - "ADX", - "AESNI", - "AVX", - "AVX2", - "BMI1", - "BMI2", - "CLMUL", - "CLZERO", - "CMOV", - "CMPXCHG8", - "CPBOOST", - "CX16", - "F16C", - "FMA3", - "FXSR", - "FXSROPT", - "HTT", - "HYPERVISOR", - "LAHF", - "LZCNT", - "MCAOVERFLOW", - "MMX", - "MMXEXT", - "MOVBE", - "NX", - "OSXSAVE", - "POPCNT", - "RDRAND", - "RDSEED", - "RDTSCP", - "SCE", - "SHA", - "SSE", - "SSE2", - "SSE3", - "SSE4", - "SSE42", - "SSE4A", - "SSSE3", - "SUCCOR", - "X87", - "XSAVE" - ], - "X64Level": 3 -} -``` - -### Check CPU microarch level - -``` -λ cpuid --check-level=3 -2022/03/18 17:04:40 AMD Ryzen 9 3950X 16-Core Processor -2022/03/18 17:04:40 Microarchitecture level 3 is supported. Max level is 3. -Exit Code 0 - -λ cpuid --check-level=4 -2022/03/18 17:06:18 AMD Ryzen 9 3950X 16-Core Processor -2022/03/18 17:06:18 Microarchitecture level 4 not supported. Max level is 3. -Exit Code 1 -``` - - -## Available flags - -### x86 & amd64 - -| Feature Flag | Description | -|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ADX | Intel ADX (Multi-Precision Add-Carry Instruction Extensions) | -| AESNI | Advanced Encryption Standard New Instructions | -| AMD3DNOW | AMD 3DNOW | -| AMD3DNOWEXT | AMD 3DNowExt | -| AMXBF16 | Tile computational operations on BFLOAT16 numbers | -| AMXINT8 | Tile computational operations on 8-bit integers | -| AMXFP16 | Tile computational operations on FP16 numbers | -| AMXFP8 | Tile computational operations on FP8 numbers | -| AMXTILE | Tile architecture | -| APX_F | Intel APX | -| AVX | AVX functions | -| AVX10 | If set the Intel AVX10 Converged Vector ISA is supported | -| AVX10_128 | If set indicates that AVX10 128-bit vector support is present | -| AVX10_256 | If set indicates that AVX10 256-bit vector support is present | -| AVX10_512 | If set indicates that AVX10 512-bit vector support is present | -| AVX2 | AVX2 functions | -| AVX512BF16 | AVX-512 BFLOAT16 Instructions | -| AVX512BITALG | AVX-512 Bit Algorithms | -| AVX512BW | AVX-512 Byte and Word Instructions | -| AVX512CD | AVX-512 Conflict Detection Instructions | -| AVX512DQ | AVX-512 Doubleword and Quadword Instructions | -| AVX512ER | AVX-512 Exponential and Reciprocal Instructions | -| AVX512F | AVX-512 Foundation | -| AVX512FP16 | AVX-512 FP16 Instructions | -| AVX512IFMA | AVX-512 Integer Fused Multiply-Add Instructions | -| AVX512PF | AVX-512 Prefetch Instructions | -| AVX512VBMI | AVX-512 Vector Bit Manipulation Instructions | -| AVX512VBMI2 | AVX-512 Vector Bit Manipulation Instructions, Version 2 | -| AVX512VL | AVX-512 Vector Length Extensions | -| AVX512VNNI | AVX-512 Vector Neural Network Instructions | -| AVX512VP2INTERSECT | AVX-512 Intersect for D/Q | -| AVX512VPOPCNTDQ | AVX-512 Vector Population Count Doubleword and Quadword | -| AVXIFMA | AVX-IFMA instructions | -| AVXNECONVERT | AVX-NE-CONVERT instructions | -| AVXSLOW | Indicates the CPU performs 2 128 bit operations instead of one | -| AVXVNNI | AVX (VEX encoded) VNNI neural network instructions | -| AVXVNNIINT8 | AVX-VNNI-INT8 instructions | -| AVXVNNIINT16 | AVX-VNNI-INT16 instructions | -| BHI_CTRL | Branch History Injection and Intra-mode Branch Target Injection / CVE-2022-0001, CVE-2022-0002 / INTEL-SA-00598 | -| BMI1 | Bit Manipulation Instruction Set 1 | -| BMI2 | Bit Manipulation Instruction Set 2 | -| CETIBT | Intel CET Indirect Branch Tracking | -| CETSS | Intel CET Shadow Stack | -| CLDEMOTE | Cache Line Demote | -| CLMUL | Carry-less Multiplication | -| CLZERO | CLZERO instruction supported | -| CMOV | i686 CMOV | -| CMPCCXADD | CMPCCXADD instructions | -| CMPSB_SCADBS_SHORT | Fast short CMPSB and SCASB | -| CMPXCHG8 | CMPXCHG8 instruction | -| CPBOOST | Core Performance Boost | -| CPPC | AMD: Collaborative Processor Performance Control | -| CX16 | CMPXCHG16B Instruction | -| EFER_LMSLE_UNS | AMD: =Core::X86::Msr::EFER[LMSLE] is not supported, and MBZ | -| ENQCMD | Enqueue Command | -| ERMS | Enhanced REP MOVSB/STOSB | -| F16C | Half-precision floating-point conversion | -| FLUSH_L1D | Flush L1D cache | -| FMA3 | Intel FMA 3. Does not imply AVX. | -| FMA4 | Bulldozer FMA4 functions | -| FP128 | AMD: When set, the internal FP/SIMD execution datapath is 128-bits wide | -| FP256 | AMD: When set, the internal FP/SIMD execution datapath is 256-bits wide | -| FSRM | Fast Short Rep Mov | -| FXSR | FXSAVE, FXRESTOR instructions, CR4 bit 9 | -| FXSROPT | FXSAVE/FXRSTOR optimizations | -| GFNI | Galois Field New Instructions. May require other features (AVX, AVX512VL,AVX512F) based on usage. | -| HLE | Hardware Lock Elision | -| HRESET | If set CPU supports history reset and the IA32_HRESET_ENABLE MSR | -| HTT | Hyperthreading (enabled) | -| HWA | Hardware assert supported. Indicates support for MSRC001_10 | -| HYBRID_CPU | This part has CPUs of more than one type. | -| HYPERVISOR | This bit has been reserved by Intel & AMD for use by hypervisors | -| IA32_ARCH_CAP | IA32_ARCH_CAPABILITIES MSR (Intel) | -| IA32_CORE_CAP | IA32_CORE_CAPABILITIES MSR | -| IBPB | Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB) | -| IBRS | AMD: Indirect Branch Restricted Speculation | -| IBRS_PREFERRED | AMD: IBRS is preferred over software solution | -| IBRS_PROVIDES_SMP | AMD: IBRS provides Same Mode Protection | -| IBS | Instruction Based Sampling (AMD) | -| IBSBRNTRGT | Instruction Based Sampling Feature (AMD) | -| IBSFETCHSAM | Instruction Based Sampling Feature (AMD) | -| IBSFFV | Instruction Based Sampling Feature (AMD) | -| IBSOPCNT | Instruction Based Sampling Feature (AMD) | -| IBSOPCNTEXT | Instruction Based Sampling Feature (AMD) | -| IBSOPSAM | Instruction Based Sampling Feature (AMD) | -| IBSRDWROPCNT | Instruction Based Sampling Feature (AMD) | -| IBSRIPINVALIDCHK | Instruction Based Sampling Feature (AMD) | -| IBS_FETCH_CTLX | AMD: IBS fetch control extended MSR supported | -| IBS_OPDATA4 | AMD: IBS op data 4 MSR supported | -| IBS_OPFUSE | AMD: Indicates support for IbsOpFuse | -| IBS_PREVENTHOST | Disallowing IBS use by the host supported | -| IBS_ZEN4 | Fetch and Op IBS support IBS extensions added with Zen4 | -| IDPRED_CTRL | IPRED_DIS | -| INT_WBINVD | WBINVD/WBNOINVD are interruptible. | -| INVLPGB | NVLPGB and TLBSYNC instruction supported | -| KEYLOCKER | Key locker | -| KEYLOCKERW | Key locker wide | -| LAHF | LAHF/SAHF in long mode | -| LAM | If set, CPU supports Linear Address Masking | -| LBRVIRT | LBR virtualization | -| LZCNT | LZCNT instruction | -| MCAOVERFLOW | MCA overflow recovery support. | -| MCDT_NO | Processor do not exhibit MXCSR Configuration Dependent Timing behavior and do not need to mitigate it. | -| MCOMMIT | MCOMMIT instruction supported | -| MD_CLEAR | VERW clears CPU buffers | -| MMX | standard MMX | -| MMXEXT | SSE integer functions or AMD MMX ext | -| MOVBE | MOVBE instruction (big-endian) | -| MOVDIR64B | Move 64 Bytes as Direct Store | -| MOVDIRI | Move Doubleword as Direct Store | -| MOVSB_ZL | Fast Zero-Length MOVSB | -| MPX | Intel MPX (Memory Protection Extensions) | -| MOVU | MOVU SSE instructions are more efficient and should be preferred to SSE MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD | -| MSRIRC | Instruction Retired Counter MSR available | -| MSRLIST | Read/Write List of Model Specific Registers | -| MSR_PAGEFLUSH | Page Flush MSR available | -| NRIPS | Indicates support for NRIP save on VMEXIT | -| NX | NX (No-Execute) bit | -| OSXSAVE | XSAVE enabled by OS | -| PCONFIG | PCONFIG for Intel Multi-Key Total Memory Encryption | -| POPCNT | POPCNT instruction | -| PPIN | AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled | -| PREFETCHI | PREFETCHIT0/1 instructions | -| PSFD | Predictive Store Forward Disable | -| RDPRU | RDPRU instruction supported | -| RDRAND | RDRAND instruction is available | -| RDSEED | RDSEED instruction is available | -| RDTSCP | RDTSCP Instruction | -| RRSBA_CTRL | Restricted RSB Alternate | -| RTM | Restricted Transactional Memory | -| RTM_ALWAYS_ABORT | Indicates that the loaded microcode is forcing RTM abort. | -| SERIALIZE | Serialize Instruction Execution | -| SEV | AMD Secure Encrypted Virtualization supported | -| SEV_64BIT | AMD SEV guest execution only allowed from a 64-bit host | -| SEV_ALTERNATIVE | AMD SEV Alternate Injection supported | -| SEV_DEBUGSWAP | Full debug state swap supported for SEV-ES guests | -| SEV_ES | AMD SEV Encrypted State supported | -| SEV_RESTRICTED | AMD SEV Restricted Injection supported | -| SEV_SNP | AMD SEV Secure Nested Paging supported | -| SGX | Software Guard Extensions | -| SGXLC | Software Guard Extensions Launch Control | -| SHA | Intel SHA Extensions | -| SME | AMD Secure Memory Encryption supported | -| SME_COHERENT | AMD Hardware cache coherency across encryption domains enforced | -| SPEC_CTRL_SSBD | Speculative Store Bypass Disable | -| SRBDS_CTRL | SRBDS mitigation MSR available | -| SSE | SSE functions | -| SSE2 | P4 SSE functions | -| SSE3 | Prescott SSE3 functions | -| SSE4 | Penryn SSE4.1 functions | -| SSE42 | Nehalem SSE4.2 functions | -| SSE4A | AMD Barcelona microarchitecture SSE4a instructions | -| SSSE3 | Conroe SSSE3 functions | -| STIBP | Single Thread Indirect Branch Predictors | -| STIBP_ALWAYSON | AMD: Single Thread Indirect Branch Prediction Mode has Enhanced Performance and may be left Always On | -| STOSB_SHORT | Fast short STOSB | -| SUCCOR | Software uncorrectable error containment and recovery capability. | -| SVM | AMD Secure Virtual Machine | -| SVMDA | Indicates support for the SVM decode assists. | -| SVMFBASID | SVM, Indicates that TLB flush events, including CR3 writes and CR4.PGE toggles, flush only the current ASID's TLB entries. Also indicates support for the extended VMCBTLB_Control | -| SVML | AMD SVM lock. Indicates support for SVM-Lock. | -| SVMNP | AMD SVM nested paging | -| SVMPF | SVM pause intercept filter. Indicates support for the pause intercept filter | -| SVMPFT | SVM PAUSE filter threshold. Indicates support for the PAUSE filter cycle count threshold | -| SYSCALL | System-Call Extension (SCE): SYSCALL and SYSRET instructions. | -| SYSEE | SYSENTER and SYSEXIT instructions | -| TBM | AMD Trailing Bit Manipulation | -| TDX_GUEST | Intel Trust Domain Extensions Guest | -| TLB_FLUSH_NESTED | AMD: Flushing includes all the nested translations for guest translations | -| TME | Intel Total Memory Encryption. The following MSRs are supported: IA32_TME_CAPABILITY, IA32_TME_ACTIVATE, IA32_TME_EXCLUDE_MASK, and IA32_TME_EXCLUDE_BASE. | -| TOPEXT | TopologyExtensions: topology extensions support. Indicates support for CPUID Fn8000_001D_EAX_x[N:0]-CPUID Fn8000_001E_EDX. | -| TSCRATEMSR | MSR based TSC rate control. Indicates support for MSR TSC ratio MSRC000_0104 | -| TSXLDTRK | Intel TSX Suspend Load Address Tracking | -| VAES | Vector AES. AVX(512) versions requires additional checks. | -| VMCBCLEAN | VMCB clean bits. Indicates support for VMCB clean bits. | -| VMPL | AMD VM Permission Levels supported | -| VMSA_REGPROT | AMD VMSA Register Protection supported | -| VMX | Virtual Machine Extensions | -| VPCLMULQDQ | Carry-Less Multiplication Quadword. Requires AVX for 3 register versions. | -| VTE | AMD Virtual Transparent Encryption supported | -| WAITPKG | TPAUSE, UMONITOR, UMWAIT | -| WBNOINVD | Write Back and Do Not Invalidate Cache | -| WRMSRNS | Non-Serializing Write to Model Specific Register | -| X87 | FPU | -| XGETBV1 | Supports XGETBV with ECX = 1 | -| XOP | Bulldozer XOP functions | -| XSAVE | XSAVE, XRESTOR, XSETBV, XGETBV | -| XSAVEC | Supports XSAVEC and the compacted form of XRSTOR. | -| XSAVEOPT | XSAVEOPT available | -| XSAVES | Supports XSAVES/XRSTORS and IA32_XSS | - -# ARM features: - -| Feature Flag | Description | -|--------------|------------------------------------------------------------------| -| AESARM | AES instructions | -| ARMCPUID | Some CPU ID registers readable at user-level | -| ASIMD | Advanced SIMD | -| ASIMDDP | SIMD Dot Product | -| ASIMDHP | Advanced SIMD half-precision floating point | -| ASIMDRDM | Rounding Double Multiply Accumulate/Subtract (SQRDMLAH/SQRDMLSH) | -| ATOMICS | Large System Extensions (LSE) | -| CRC32 | CRC32/CRC32C instructions | -| DCPOP | Data cache clean to Point of Persistence (DC CVAP) | -| EVTSTRM | Generic timer | -| FCMA | Floatin point complex number addition and multiplication | -| FP | Single-precision and double-precision floating point | -| FPHP | Half-precision floating point | -| GPA | Generic Pointer Authentication | -| JSCVT | Javascript-style double->int convert (FJCVTZS) | -| LRCPC | Weaker release consistency (LDAPR, etc) | -| PMULL | Polynomial Multiply instructions (PMULL/PMULL2) | -| SHA1 | SHA-1 instructions (SHA1C, etc) | -| SHA2 | SHA-2 instructions (SHA256H, etc) | -| SHA3 | SHA-3 instructions (EOR3, RAXI, XAR, BCAX) | -| SHA512 | SHA512 instructions | -| SM3 | SM3 instructions | -| SM4 | SM4 instructions | -| SVE | Scalable Vector Extension | - -# license - -This code is published under an MIT license. See LICENSE file for more information. diff --git a/vendor/github.com/klauspost/cpuid/v2/cpuid.go b/vendor/github.com/klauspost/cpuid/v2/cpuid.go deleted file mode 100644 index db99eb62f..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/cpuid.go +++ /dev/null @@ -1,1558 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -// Package cpuid provides information about the CPU running the current program. -// -// CPU features are detected on startup, and kept for fast access through the life of the application. -// Currently x86 / x64 (AMD64) as well as arm64 is supported. -// -// You can access the CPU information by accessing the shared CPU variable of the cpuid library. -// -// Package home: https://github.com/klauspost/cpuid -package cpuid - -import ( - "flag" - "fmt" - "math" - "math/bits" - "os" - "runtime" - "strings" -) - -// AMD refererence: https://www.amd.com/system/files/TechDocs/25481.pdf -// and Processor Programming Reference (PPR) - -// Vendor is a representation of a CPU vendor. -type Vendor int - -const ( - VendorUnknown Vendor = iota - Intel - AMD - VIA - Transmeta - NSC - KVM // Kernel-based Virtual Machine - MSVM // Microsoft Hyper-V or Windows Virtual PC - VMware - XenHVM - Bhyve - Hygon - SiS - RDC - - Ampere - ARM - Broadcom - Cavium - DEC - Fujitsu - Infineon - Motorola - NVIDIA - AMCC - Qualcomm - Marvell - - QEMU - QNX - ACRN - SRE - Apple - - lastVendor -) - -//go:generate stringer -type=FeatureID,Vendor - -// FeatureID is the ID of a specific cpu feature. -type FeatureID int - -const ( - // Keep index -1 as unknown - UNKNOWN = -1 - - // x86 features - ADX FeatureID = iota // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - AESNI // Advanced Encryption Standard New Instructions - AMD3DNOW // AMD 3DNOW - AMD3DNOWEXT // AMD 3DNowExt - AMXBF16 // Tile computational operations on BFLOAT16 numbers - AMXFP16 // Tile computational operations on FP16 numbers - AMXINT8 // Tile computational operations on 8-bit integers - AMXFP8 // Tile computational operations on FP8 numbers - AMXTILE // Tile architecture - APX_F // Intel APX - AVX // AVX functions - AVX10 // If set the Intel AVX10 Converged Vector ISA is supported - AVX10_128 // If set indicates that AVX10 128-bit vector support is present - AVX10_256 // If set indicates that AVX10 256-bit vector support is present - AVX10_512 // If set indicates that AVX10 512-bit vector support is present - AVX2 // AVX2 functions - AVX512BF16 // AVX-512 BFLOAT16 Instructions - AVX512BITALG // AVX-512 Bit Algorithms - AVX512BW // AVX-512 Byte and Word Instructions - AVX512CD // AVX-512 Conflict Detection Instructions - AVX512DQ // AVX-512 Doubleword and Quadword Instructions - AVX512ER // AVX-512 Exponential and Reciprocal Instructions - AVX512F // AVX-512 Foundation - AVX512FP16 // AVX-512 FP16 Instructions - AVX512IFMA // AVX-512 Integer Fused Multiply-Add Instructions - AVX512PF // AVX-512 Prefetch Instructions - AVX512VBMI // AVX-512 Vector Bit Manipulation Instructions - AVX512VBMI2 // AVX-512 Vector Bit Manipulation Instructions, Version 2 - AVX512VL // AVX-512 Vector Length Extensions - AVX512VNNI // AVX-512 Vector Neural Network Instructions - AVX512VP2INTERSECT // AVX-512 Intersect for D/Q - AVX512VPOPCNTDQ // AVX-512 Vector Population Count Doubleword and Quadword - AVXIFMA // AVX-IFMA instructions - AVXNECONVERT // AVX-NE-CONVERT instructions - AVXSLOW // Indicates the CPU performs 2 128 bit operations instead of one - AVXVNNI // AVX (VEX encoded) VNNI neural network instructions - AVXVNNIINT8 // AVX-VNNI-INT8 instructions - AVXVNNIINT16 // AVX-VNNI-INT16 instructions - BHI_CTRL // Branch History Injection and Intra-mode Branch Target Injection / CVE-2022-0001, CVE-2022-0002 / INTEL-SA-00598 - BMI1 // Bit Manipulation Instruction Set 1 - BMI2 // Bit Manipulation Instruction Set 2 - CETIBT // Intel CET Indirect Branch Tracking - CETSS // Intel CET Shadow Stack - CLDEMOTE // Cache Line Demote - CLMUL // Carry-less Multiplication - CLZERO // CLZERO instruction supported - CMOV // i686 CMOV - CMPCCXADD // CMPCCXADD instructions - CMPSB_SCADBS_SHORT // Fast short CMPSB and SCASB - CMPXCHG8 // CMPXCHG8 instruction - CPBOOST // Core Performance Boost - CPPC // AMD: Collaborative Processor Performance Control - CX16 // CMPXCHG16B Instruction - EFER_LMSLE_UNS // AMD: =Core::X86::Msr::EFER[LMSLE] is not supported, and MBZ - ENQCMD // Enqueue Command - ERMS // Enhanced REP MOVSB/STOSB - F16C // Half-precision floating-point conversion - FLUSH_L1D // Flush L1D cache - FMA3 // Intel FMA 3. Does not imply AVX. - FMA4 // Bulldozer FMA4 functions - FP128 // AMD: When set, the internal FP/SIMD execution datapath is no more than 128-bits wide - FP256 // AMD: When set, the internal FP/SIMD execution datapath is no more than 256-bits wide - FSRM // Fast Short Rep Mov - FXSR // FXSAVE, FXRESTOR instructions, CR4 bit 9 - FXSROPT // FXSAVE/FXRSTOR optimizations - GFNI // Galois Field New Instructions. May require other features (AVX, AVX512VL,AVX512F) based on usage. - HLE // Hardware Lock Elision - HRESET // If set CPU supports history reset and the IA32_HRESET_ENABLE MSR - HTT // Hyperthreading (enabled) - HWA // Hardware assert supported. Indicates support for MSRC001_10 - HYBRID_CPU // This part has CPUs of more than one type. - HYPERVISOR // This bit has been reserved by Intel & AMD for use by hypervisors - IA32_ARCH_CAP // IA32_ARCH_CAPABILITIES MSR (Intel) - IA32_CORE_CAP // IA32_CORE_CAPABILITIES MSR - IBPB // Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB) - IBPB_BRTYPE // Indicates that MSR 49h (PRED_CMD) bit 0 (IBPB) flushes all branch type predictions from the CPU branch predictor - IBRS // AMD: Indirect Branch Restricted Speculation - IBRS_PREFERRED // AMD: IBRS is preferred over software solution - IBRS_PROVIDES_SMP // AMD: IBRS provides Same Mode Protection - IBS // Instruction Based Sampling (AMD) - IBSBRNTRGT // Instruction Based Sampling Feature (AMD) - IBSFETCHSAM // Instruction Based Sampling Feature (AMD) - IBSFFV // Instruction Based Sampling Feature (AMD) - IBSOPCNT // Instruction Based Sampling Feature (AMD) - IBSOPCNTEXT // Instruction Based Sampling Feature (AMD) - IBSOPSAM // Instruction Based Sampling Feature (AMD) - IBSRDWROPCNT // Instruction Based Sampling Feature (AMD) - IBSRIPINVALIDCHK // Instruction Based Sampling Feature (AMD) - IBS_FETCH_CTLX // AMD: IBS fetch control extended MSR supported - IBS_OPDATA4 // AMD: IBS op data 4 MSR supported - IBS_OPFUSE // AMD: Indicates support for IbsOpFuse - IBS_PREVENTHOST // Disallowing IBS use by the host supported - IBS_ZEN4 // AMD: Fetch and Op IBS support IBS extensions added with Zen4 - IDPRED_CTRL // IPRED_DIS - INT_WBINVD // WBINVD/WBNOINVD are interruptible. - INVLPGB // NVLPGB and TLBSYNC instruction supported - KEYLOCKER // Key locker - KEYLOCKERW // Key locker wide - LAHF // LAHF/SAHF in long mode - LAM // If set, CPU supports Linear Address Masking - LBRVIRT // LBR virtualization - LZCNT // LZCNT instruction - MCAOVERFLOW // MCA overflow recovery support. - MCDT_NO // Processor do not exhibit MXCSR Configuration Dependent Timing behavior and do not need to mitigate it. - MCOMMIT // MCOMMIT instruction supported - MD_CLEAR // VERW clears CPU buffers - MMX // standard MMX - MMXEXT // SSE integer functions or AMD MMX ext - MOVBE // MOVBE instruction (big-endian) - MOVDIR64B // Move 64 Bytes as Direct Store - MOVDIRI // Move Doubleword as Direct Store - MOVSB_ZL // Fast Zero-Length MOVSB - MOVU // AMD: MOVU SSE instructions are more efficient and should be preferred to SSE MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD - MPX // Intel MPX (Memory Protection Extensions) - MSRIRC // Instruction Retired Counter MSR available - MSRLIST // Read/Write List of Model Specific Registers - MSR_PAGEFLUSH // Page Flush MSR available - NRIPS // Indicates support for NRIP save on VMEXIT - NX // NX (No-Execute) bit - OSXSAVE // XSAVE enabled by OS - PCONFIG // PCONFIG for Intel Multi-Key Total Memory Encryption - POPCNT // POPCNT instruction - PPIN // AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled - PREFETCHI // PREFETCHIT0/1 instructions - PSFD // Predictive Store Forward Disable - RDPRU // RDPRU instruction supported - RDRAND // RDRAND instruction is available - RDSEED // RDSEED instruction is available - RDTSCP // RDTSCP Instruction - RRSBA_CTRL // Restricted RSB Alternate - RTM // Restricted Transactional Memory - RTM_ALWAYS_ABORT // Indicates that the loaded microcode is forcing RTM abort. - SBPB // Indicates support for the Selective Branch Predictor Barrier - SERIALIZE // Serialize Instruction Execution - SEV // AMD Secure Encrypted Virtualization supported - SEV_64BIT // AMD SEV guest execution only allowed from a 64-bit host - SEV_ALTERNATIVE // AMD SEV Alternate Injection supported - SEV_DEBUGSWAP // Full debug state swap supported for SEV-ES guests - SEV_ES // AMD SEV Encrypted State supported - SEV_RESTRICTED // AMD SEV Restricted Injection supported - SEV_SNP // AMD SEV Secure Nested Paging supported - SGX // Software Guard Extensions - SGXLC // Software Guard Extensions Launch Control - SHA // Intel SHA Extensions - SME // AMD Secure Memory Encryption supported - SME_COHERENT // AMD Hardware cache coherency across encryption domains enforced - SPEC_CTRL_SSBD // Speculative Store Bypass Disable - SRBDS_CTRL // SRBDS mitigation MSR available - SRSO_MSR_FIX // Indicates that software may use MSR BP_CFG[BpSpecReduce] to mitigate SRSO. - SRSO_NO // Indicates the CPU is not subject to the SRSO vulnerability - SRSO_USER_KERNEL_NO // Indicates the CPU is not subject to the SRSO vulnerability across user/kernel boundaries - SSE // SSE functions - SSE2 // P4 SSE functions - SSE3 // Prescott SSE3 functions - SSE4 // Penryn SSE4.1 functions - SSE42 // Nehalem SSE4.2 functions - SSE4A // AMD Barcelona microarchitecture SSE4a instructions - SSSE3 // Conroe SSSE3 functions - STIBP // Single Thread Indirect Branch Predictors - STIBP_ALWAYSON // AMD: Single Thread Indirect Branch Prediction Mode has Enhanced Performance and may be left Always On - STOSB_SHORT // Fast short STOSB - SUCCOR // Software uncorrectable error containment and recovery capability. - SVM // AMD Secure Virtual Machine - SVMDA // Indicates support for the SVM decode assists. - SVMFBASID // SVM, Indicates that TLB flush events, including CR3 writes and CR4.PGE toggles, flush only the current ASID's TLB entries. Also indicates support for the extended VMCBTLB_Control - SVML // AMD SVM lock. Indicates support for SVM-Lock. - SVMNP // AMD SVM nested paging - SVMPF // SVM pause intercept filter. Indicates support for the pause intercept filter - SVMPFT // SVM PAUSE filter threshold. Indicates support for the PAUSE filter cycle count threshold - SYSCALL // System-Call Extension (SCE): SYSCALL and SYSRET instructions. - SYSEE // SYSENTER and SYSEXIT instructions - TBM // AMD Trailing Bit Manipulation - TDX_GUEST // Intel Trust Domain Extensions Guest - TLB_FLUSH_NESTED // AMD: Flushing includes all the nested translations for guest translations - TME // Intel Total Memory Encryption. The following MSRs are supported: IA32_TME_CAPABILITY, IA32_TME_ACTIVATE, IA32_TME_EXCLUDE_MASK, and IA32_TME_EXCLUDE_BASE. - TOPEXT // TopologyExtensions: topology extensions support. Indicates support for CPUID Fn8000_001D_EAX_x[N:0]-CPUID Fn8000_001E_EDX. - TSCRATEMSR // MSR based TSC rate control. Indicates support for MSR TSC ratio MSRC000_0104 - TSXLDTRK // Intel TSX Suspend Load Address Tracking - VAES // Vector AES. AVX(512) versions requires additional checks. - VMCBCLEAN // VMCB clean bits. Indicates support for VMCB clean bits. - VMPL // AMD VM Permission Levels supported - VMSA_REGPROT // AMD VMSA Register Protection supported - VMX // Virtual Machine Extensions - VPCLMULQDQ // Carry-Less Multiplication Quadword. Requires AVX for 3 register versions. - VTE // AMD Virtual Transparent Encryption supported - WAITPKG // TPAUSE, UMONITOR, UMWAIT - WBNOINVD // Write Back and Do Not Invalidate Cache - WRMSRNS // Non-Serializing Write to Model Specific Register - X87 // FPU - XGETBV1 // Supports XGETBV with ECX = 1 - XOP // Bulldozer XOP functions - XSAVE // XSAVE, XRESTOR, XSETBV, XGETBV - XSAVEC // Supports XSAVEC and the compacted form of XRSTOR. - XSAVEOPT // XSAVEOPT available - XSAVES // Supports XSAVES/XRSTORS and IA32_XSS - - // ARM features: - AESARM // AES instructions - ARMCPUID // Some CPU ID registers readable at user-level - ASIMD // Advanced SIMD - ASIMDDP // SIMD Dot Product - ASIMDHP // Advanced SIMD half-precision floating point - ASIMDRDM // Rounding Double Multiply Accumulate/Subtract (SQRDMLAH/SQRDMLSH) - ATOMICS // Large System Extensions (LSE) - CRC32 // CRC32/CRC32C instructions - DCPOP // Data cache clean to Point of Persistence (DC CVAP) - EVTSTRM // Generic timer - FCMA // Floatin point complex number addition and multiplication - FP // Single-precision and double-precision floating point - FPHP // Half-precision floating point - GPA // Generic Pointer Authentication - JSCVT // Javascript-style double->int convert (FJCVTZS) - LRCPC // Weaker release consistency (LDAPR, etc) - PMULL // Polynomial Multiply instructions (PMULL/PMULL2) - SHA1 // SHA-1 instructions (SHA1C, etc) - SHA2 // SHA-2 instructions (SHA256H, etc) - SHA3 // SHA-3 instructions (EOR3, RAXI, XAR, BCAX) - SHA512 // SHA512 instructions - SM3 // SM3 instructions - SM4 // SM4 instructions - SVE // Scalable Vector Extension - // Keep it last. It automatically defines the size of []flagSet - lastID - - firstID FeatureID = UNKNOWN + 1 -) - -// CPUInfo contains information about the detected system CPU. -type CPUInfo struct { - BrandName string // Brand name reported by the CPU - VendorID Vendor // Comparable CPU vendor ID - VendorString string // Raw vendor string. - HypervisorVendorID Vendor // Hypervisor vendor - HypervisorVendorString string // Raw hypervisor vendor string - featureSet flagSet // Features of the CPU - PhysicalCores int // Number of physical processor cores in your CPU. Will be 0 if undetectable. - ThreadsPerCore int // Number of threads per physical core. Will be 1 if undetectable. - LogicalCores int // Number of physical cores times threads that can run on each core through the use of hyperthreading. Will be 0 if undetectable. - Family int // CPU family number - Model int // CPU model number - Stepping int // CPU stepping info - CacheLine int // Cache line size in bytes. Will be 0 if undetectable. - Hz int64 // Clock speed, if known, 0 otherwise. Will attempt to contain base clock speed. - BoostFreq int64 // Max clock speed, if known, 0 otherwise - Cache struct { - L1I int // L1 Instruction Cache (per core or shared). Will be -1 if undetected - L1D int // L1 Data Cache (per core or shared). Will be -1 if undetected - L2 int // L2 Cache (per core or shared). Will be -1 if undetected - L3 int // L3 Cache (per core, per ccx or shared). Will be -1 if undetected - } - SGX SGXSupport - AMDMemEncryption AMDMemEncryptionSupport - AVX10Level uint8 - - maxFunc uint32 - maxExFunc uint32 -} - -var cpuid func(op uint32) (eax, ebx, ecx, edx uint32) -var cpuidex func(op, op2 uint32) (eax, ebx, ecx, edx uint32) -var xgetbv func(index uint32) (eax, edx uint32) -var rdtscpAsm func() (eax, ebx, ecx, edx uint32) -var darwinHasAVX512 = func() bool { return false } - -// CPU contains information about the CPU as detected on startup, -// or when Detect last was called. -// -// Use this as the primary entry point to you data. -var CPU CPUInfo - -func init() { - initCPU() - Detect() -} - -// Detect will re-detect current CPU info. -// This will replace the content of the exported CPU variable. -// -// Unless you expect the CPU to change while you are running your program -// you should not need to call this function. -// If you call this, you must ensure that no other goroutine is accessing the -// exported CPU variable. -func Detect() { - // Set defaults - CPU.ThreadsPerCore = 1 - CPU.Cache.L1I = -1 - CPU.Cache.L1D = -1 - CPU.Cache.L2 = -1 - CPU.Cache.L3 = -1 - safe := true - if detectArmFlag != nil { - safe = !*detectArmFlag - } - addInfo(&CPU, safe) - if displayFeats != nil && *displayFeats { - fmt.Println("cpu features:", strings.Join(CPU.FeatureSet(), ",")) - // Exit with non-zero so tests will print value. - os.Exit(1) - } - if disableFlag != nil { - s := strings.Split(*disableFlag, ",") - for _, feat := range s { - feat := ParseFeature(strings.TrimSpace(feat)) - if feat != UNKNOWN { - CPU.featureSet.unset(feat) - } - } - } -} - -// DetectARM will detect ARM64 features. -// This is NOT done automatically since it can potentially crash -// if the OS does not handle the command. -// If in the future this can be done safely this function may not -// do anything. -func DetectARM() { - addInfo(&CPU, false) -} - -var detectArmFlag *bool -var displayFeats *bool -var disableFlag *string - -// Flags will enable flags. -// This must be called *before* flag.Parse AND -// Detect must be called after the flags have been parsed. -// Note that this means that any detection used in init() functions -// will not contain these flags. -func Flags() { - disableFlag = flag.String("cpu.disable", "", "disable cpu features; comma separated list") - displayFeats = flag.Bool("cpu.features", false, "lists cpu features and exits") - detectArmFlag = flag.Bool("cpu.arm", false, "allow ARM features to be detected; can potentially crash") -} - -// Supports returns whether the CPU supports all of the requested features. -func (c CPUInfo) Supports(ids ...FeatureID) bool { - for _, id := range ids { - if !c.featureSet.inSet(id) { - return false - } - } - return true -} - -// Has allows for checking a single feature. -// Should be inlined by the compiler. -func (c *CPUInfo) Has(id FeatureID) bool { - return c.featureSet.inSet(id) -} - -// AnyOf returns whether the CPU supports one or more of the requested features. -func (c CPUInfo) AnyOf(ids ...FeatureID) bool { - for _, id := range ids { - if c.featureSet.inSet(id) { - return true - } - } - return false -} - -// Features contains several features combined for a fast check using -// CpuInfo.HasAll -type Features *flagSet - -// CombineFeatures allows to combine several features for a close to constant time lookup. -func CombineFeatures(ids ...FeatureID) Features { - var v flagSet - for _, id := range ids { - v.set(id) - } - return &v -} - -func (c *CPUInfo) HasAll(f Features) bool { - return c.featureSet.hasSetP(f) -} - -// https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels -var oneOfLevel = CombineFeatures(SYSEE, SYSCALL) -var level1Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2) -var level2Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3) -var level3Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE) -var level4Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE, AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL) - -// X64Level returns the microarchitecture level detected on the CPU. -// If features are lacking or non x64 mode, 0 is returned. -// See https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels -func (c CPUInfo) X64Level() int { - if !c.featureSet.hasOneOf(oneOfLevel) { - return 0 - } - if c.featureSet.hasSetP(level4Features) { - return 4 - } - if c.featureSet.hasSetP(level3Features) { - return 3 - } - if c.featureSet.hasSetP(level2Features) { - return 2 - } - if c.featureSet.hasSetP(level1Features) { - return 1 - } - return 0 -} - -// Disable will disable one or several features. -func (c *CPUInfo) Disable(ids ...FeatureID) bool { - for _, id := range ids { - c.featureSet.unset(id) - } - return true -} - -// Enable will disable one or several features even if they were undetected. -// This is of course not recommended for obvious reasons. -func (c *CPUInfo) Enable(ids ...FeatureID) bool { - for _, id := range ids { - c.featureSet.set(id) - } - return true -} - -// IsVendor returns true if vendor is recognized as Intel -func (c CPUInfo) IsVendor(v Vendor) bool { - return c.VendorID == v -} - -// FeatureSet returns all available features as strings. -func (c CPUInfo) FeatureSet() []string { - s := make([]string, 0, c.featureSet.nEnabled()) - s = append(s, c.featureSet.Strings()...) - return s -} - -// RTCounter returns the 64-bit time-stamp counter -// Uses the RDTSCP instruction. The value 0 is returned -// if the CPU does not support the instruction. -func (c CPUInfo) RTCounter() uint64 { - if !c.Has(RDTSCP) { - return 0 - } - a, _, _, d := rdtscpAsm() - return uint64(a) | (uint64(d) << 32) -} - -// Ia32TscAux returns the IA32_TSC_AUX part of the RDTSCP. -// This variable is OS dependent, but on Linux contains information -// about the current cpu/core the code is running on. -// If the RDTSCP instruction isn't supported on the CPU, the value 0 is returned. -func (c CPUInfo) Ia32TscAux() uint32 { - if !c.Has(RDTSCP) { - return 0 - } - _, _, ecx, _ := rdtscpAsm() - return ecx -} - -// SveLengths returns arm SVE vector and predicate lengths. -// Will return 0, 0 if SVE is not enabled or otherwise unable to detect. -func (c CPUInfo) SveLengths() (vl, pl uint64) { - if !c.Has(SVE) { - return 0, 0 - } - return getVectorLength() -} - -// LogicalCPU will return the Logical CPU the code is currently executing on. -// This is likely to change when the OS re-schedules the running thread -// to another CPU. -// If the current core cannot be detected, -1 will be returned. -func (c CPUInfo) LogicalCPU() int { - if c.maxFunc < 1 { - return -1 - } - _, ebx, _, _ := cpuid(1) - return int(ebx >> 24) -} - -// frequencies tries to compute the clock speed of the CPU. If leaf 15 is -// supported, use it, otherwise parse the brand string. Yes, really. -func (c *CPUInfo) frequencies() { - c.Hz, c.BoostFreq = 0, 0 - mfi := maxFunctionID() - if mfi >= 0x15 { - eax, ebx, ecx, _ := cpuid(0x15) - if eax != 0 && ebx != 0 && ecx != 0 { - c.Hz = (int64(ecx) * int64(ebx)) / int64(eax) - } - } - if mfi >= 0x16 { - a, b, _, _ := cpuid(0x16) - // Base... - if a&0xffff > 0 { - c.Hz = int64(a&0xffff) * 1_000_000 - } - // Boost... - if b&0xffff > 0 { - c.BoostFreq = int64(b&0xffff) * 1_000_000 - } - } - if c.Hz > 0 { - return - } - - // computeHz determines the official rated speed of a CPU from its brand - // string. This insanity is *actually the official documented way to do - // this according to Intel*, prior to leaf 0x15 existing. The official - // documentation only shows this working for exactly `x.xx` or `xxxx` - // cases, e.g., `2.50GHz` or `1300MHz`; this parser will accept other - // sizes. - model := c.BrandName - hz := strings.LastIndex(model, "Hz") - if hz < 3 { - return - } - var multiplier int64 - switch model[hz-1] { - case 'M': - multiplier = 1000 * 1000 - case 'G': - multiplier = 1000 * 1000 * 1000 - case 'T': - multiplier = 1000 * 1000 * 1000 * 1000 - } - if multiplier == 0 { - return - } - freq := int64(0) - divisor := int64(0) - decimalShift := int64(1) - var i int - for i = hz - 2; i >= 0 && model[i] != ' '; i-- { - if model[i] >= '0' && model[i] <= '9' { - freq += int64(model[i]-'0') * decimalShift - decimalShift *= 10 - } else if model[i] == '.' { - if divisor != 0 { - return - } - divisor = decimalShift - } else { - return - } - } - // we didn't find a space - if i < 0 { - return - } - if divisor != 0 { - c.Hz = (freq * multiplier) / divisor - return - } - c.Hz = freq * multiplier -} - -// VM Will return true if the cpu id indicates we are in -// a virtual machine. -func (c CPUInfo) VM() bool { - return CPU.featureSet.inSet(HYPERVISOR) -} - -// flags contains detected cpu features and characteristics -type flags uint64 - -// log2(bits_in_uint64) -const flagBitsLog2 = 6 -const flagBits = 1 << flagBitsLog2 -const flagMask = flagBits - 1 - -// flagSet contains detected cpu features and characteristics in an array of flags -type flagSet [(lastID + flagMask) / flagBits]flags - -func (s *flagSet) inSet(feat FeatureID) bool { - return s[feat>>flagBitsLog2]&(1<<(feat&flagMask)) != 0 -} - -func (s *flagSet) set(feat FeatureID) { - s[feat>>flagBitsLog2] |= 1 << (feat & flagMask) -} - -// setIf will set a feature if boolean is true. -func (s *flagSet) setIf(cond bool, features ...FeatureID) { - if cond { - for _, offset := range features { - s[offset>>flagBitsLog2] |= 1 << (offset & flagMask) - } - } -} - -func (s *flagSet) unset(offset FeatureID) { - bit := flags(1 << (offset & flagMask)) - s[offset>>flagBitsLog2] = s[offset>>flagBitsLog2] & ^bit -} - -// or with another flagset. -func (s *flagSet) or(other flagSet) { - for i, v := range other[:] { - s[i] |= v - } -} - -// hasSet returns whether all features are present. -func (s *flagSet) hasSet(other flagSet) bool { - for i, v := range other[:] { - if s[i]&v != v { - return false - } - } - return true -} - -// hasSet returns whether all features are present. -func (s *flagSet) hasSetP(other *flagSet) bool { - for i, v := range other[:] { - if s[i]&v != v { - return false - } - } - return true -} - -// hasOneOf returns whether one or more features are present. -func (s *flagSet) hasOneOf(other *flagSet) bool { - for i, v := range other[:] { - if s[i]&v != 0 { - return true - } - } - return false -} - -// nEnabled will return the number of enabled flags. -func (s *flagSet) nEnabled() (n int) { - for _, v := range s[:] { - n += bits.OnesCount64(uint64(v)) - } - return n -} - -func flagSetWith(feat ...FeatureID) flagSet { - var res flagSet - for _, f := range feat { - res.set(f) - } - return res -} - -// ParseFeature will parse the string and return the ID of the matching feature. -// Will return UNKNOWN if not found. -func ParseFeature(s string) FeatureID { - s = strings.ToUpper(s) - for i := firstID; i < lastID; i++ { - if i.String() == s { - return i - } - } - return UNKNOWN -} - -// Strings returns an array of the detected features for FlagsSet. -func (s flagSet) Strings() []string { - if len(s) == 0 { - return []string{""} - } - r := make([]string, 0) - for i := firstID; i < lastID; i++ { - if s.inSet(i) { - r = append(r, i.String()) - } - } - return r -} - -func maxExtendedFunction() uint32 { - eax, _, _, _ := cpuid(0x80000000) - return eax -} - -func maxFunctionID() uint32 { - a, _, _, _ := cpuid(0) - return a -} - -func brandName() string { - if maxExtendedFunction() >= 0x80000004 { - v := make([]uint32, 0, 48) - for i := uint32(0); i < 3; i++ { - a, b, c, d := cpuid(0x80000002 + i) - v = append(v, a, b, c, d) - } - return strings.Trim(string(valAsString(v...)), " ") - } - return "unknown" -} - -func threadsPerCore() int { - mfi := maxFunctionID() - vend, _ := vendorID() - - if mfi < 0x4 || (vend != Intel && vend != AMD) { - return 1 - } - - if mfi < 0xb { - if vend != Intel { - return 1 - } - _, b, _, d := cpuid(1) - if (d & (1 << 28)) != 0 { - // v will contain logical core count - v := (b >> 16) & 255 - if v > 1 { - a4, _, _, _ := cpuid(4) - // physical cores - v2 := (a4 >> 26) + 1 - if v2 > 0 { - return int(v) / int(v2) - } - } - } - return 1 - } - _, b, _, _ := cpuidex(0xb, 0) - if b&0xffff == 0 { - if vend == AMD { - // if >= Zen 2 0x8000001e EBX 15-8 bits means threads per core. - // The number of threads per core is ThreadsPerCore+1 - // See PPR for AMD Family 17h Models 00h-0Fh (page 82) - fam, _, _ := familyModel() - _, _, _, d := cpuid(1) - if (d&(1<<28)) != 0 && fam >= 23 { - if maxExtendedFunction() >= 0x8000001e { - _, b, _, _ := cpuid(0x8000001e) - return int((b>>8)&0xff) + 1 - } - return 2 - } - } - return 1 - } - return int(b & 0xffff) -} - -func logicalCores() int { - mfi := maxFunctionID() - v, _ := vendorID() - switch v { - case Intel: - // Use this on old Intel processors - if mfi < 0xb { - if mfi < 1 { - return 0 - } - // CPUID.1:EBX[23:16] represents the maximum number of addressable IDs (initial APIC ID) - // that can be assigned to logical processors in a physical package. - // The value may not be the same as the number of logical processors that are present in the hardware of a physical package. - _, ebx, _, _ := cpuid(1) - logical := (ebx >> 16) & 0xff - return int(logical) - } - _, b, _, _ := cpuidex(0xb, 1) - return int(b & 0xffff) - case AMD, Hygon: - _, b, _, _ := cpuid(1) - return int((b >> 16) & 0xff) - default: - return 0 - } -} - -func familyModel() (family, model, stepping int) { - if maxFunctionID() < 0x1 { - return 0, 0, 0 - } - eax, _, _, _ := cpuid(1) - // If BaseFamily[3:0] is less than Fh then ExtendedFamily[7:0] is reserved and Family is equal to BaseFamily[3:0]. - family = int((eax >> 8) & 0xf) - extFam := family == 0x6 // Intel is 0x6, needs extended model. - if family == 0xf { - // Add ExtFamily - family += int((eax >> 20) & 0xff) - extFam = true - } - // If BaseFamily[3:0] is less than 0Fh then ExtendedModel[3:0] is reserved and Model is equal to BaseModel[3:0]. - model = int((eax >> 4) & 0xf) - if extFam { - // Add ExtModel - model += int((eax >> 12) & 0xf0) - } - stepping = int(eax & 0xf) - return family, model, stepping -} - -func physicalCores() int { - v, _ := vendorID() - switch v { - case Intel: - return logicalCores() / threadsPerCore() - case AMD, Hygon: - lc := logicalCores() - tpc := threadsPerCore() - if lc > 0 && tpc > 0 { - return lc / tpc - } - - // The following is inaccurate on AMD EPYC 7742 64-Core Processor - if maxExtendedFunction() >= 0x80000008 { - _, _, c, _ := cpuid(0x80000008) - if c&0xff > 0 { - return int(c&0xff) + 1 - } - } - } - return 0 -} - -// Except from http://en.wikipedia.org/wiki/CPUID#EAX.3D0:_Get_vendor_ID -var vendorMapping = map[string]Vendor{ - "AMDisbetter!": AMD, - "AuthenticAMD": AMD, - "CentaurHauls": VIA, - "GenuineIntel": Intel, - "TransmetaCPU": Transmeta, - "GenuineTMx86": Transmeta, - "Geode by NSC": NSC, - "VIA VIA VIA ": VIA, - "KVMKVMKVM": KVM, - "Linux KVM Hv": KVM, - "TCGTCGTCGTCG": QEMU, - "Microsoft Hv": MSVM, - "VMwareVMware": VMware, - "XenVMMXenVMM": XenHVM, - "bhyve bhyve ": Bhyve, - "HygonGenuine": Hygon, - "Vortex86 SoC": SiS, - "SiS SiS SiS ": SiS, - "RiseRiseRise": SiS, - "Genuine RDC": RDC, - "QNXQVMBSQG": QNX, - "ACRNACRNACRN": ACRN, - "SRESRESRESRE": SRE, - "Apple VZ": Apple, -} - -func vendorID() (Vendor, string) { - _, b, c, d := cpuid(0) - v := string(valAsString(b, d, c)) - vend, ok := vendorMapping[v] - if !ok { - return VendorUnknown, v - } - return vend, v -} - -func hypervisorVendorID() (Vendor, string) { - // https://lwn.net/Articles/301888/ - _, b, c, d := cpuid(0x40000000) - v := string(valAsString(b, c, d)) - vend, ok := vendorMapping[v] - if !ok { - return VendorUnknown, v - } - return vend, v -} - -func cacheLine() int { - if maxFunctionID() < 0x1 { - return 0 - } - - _, ebx, _, _ := cpuid(1) - cache := (ebx & 0xff00) >> 5 // cflush size - if cache == 0 && maxExtendedFunction() >= 0x80000006 { - _, _, ecx, _ := cpuid(0x80000006) - cache = ecx & 0xff // cacheline size - } - // TODO: Read from Cache and TLB Information - return int(cache) -} - -func (c *CPUInfo) cacheSize() { - c.Cache.L1D = -1 - c.Cache.L1I = -1 - c.Cache.L2 = -1 - c.Cache.L3 = -1 - vendor, _ := vendorID() - switch vendor { - case Intel: - if maxFunctionID() < 4 { - return - } - c.Cache.L1I, c.Cache.L1D, c.Cache.L2, c.Cache.L3 = 0, 0, 0, 0 - for i := uint32(0); ; i++ { - eax, ebx, ecx, _ := cpuidex(4, i) - cacheType := eax & 15 - if cacheType == 0 { - break - } - cacheLevel := (eax >> 5) & 7 - coherency := int(ebx&0xfff) + 1 - partitions := int((ebx>>12)&0x3ff) + 1 - associativity := int((ebx>>22)&0x3ff) + 1 - sets := int(ecx) + 1 - size := associativity * partitions * coherency * sets - switch cacheLevel { - case 1: - if cacheType == 1 { - // 1 = Data Cache - c.Cache.L1D = size - } else if cacheType == 2 { - // 2 = Instruction Cache - c.Cache.L1I = size - } else { - if c.Cache.L1D < 0 { - c.Cache.L1I = size - } - if c.Cache.L1I < 0 { - c.Cache.L1I = size - } - } - case 2: - c.Cache.L2 = size - case 3: - c.Cache.L3 = size - } - } - case AMD, Hygon: - // Untested. - if maxExtendedFunction() < 0x80000005 { - return - } - _, _, ecx, edx := cpuid(0x80000005) - c.Cache.L1D = int(((ecx >> 24) & 0xFF) * 1024) - c.Cache.L1I = int(((edx >> 24) & 0xFF) * 1024) - - if maxExtendedFunction() < 0x80000006 { - return - } - _, _, ecx, _ = cpuid(0x80000006) - c.Cache.L2 = int(((ecx >> 16) & 0xFFFF) * 1024) - - // CPUID Fn8000_001D_EAX_x[N:0] Cache Properties - if maxExtendedFunction() < 0x8000001D || !c.Has(TOPEXT) { - return - } - - // Xen Hypervisor is buggy and returns the same entry no matter ECX value. - // Hack: When we encounter the same entry 100 times we break. - nSame := 0 - var last uint32 - for i := uint32(0); i < math.MaxUint32; i++ { - eax, ebx, ecx, _ := cpuidex(0x8000001D, i) - - level := (eax >> 5) & 7 - cacheNumSets := ecx + 1 - cacheLineSize := 1 + (ebx & 2047) - cachePhysPartitions := 1 + ((ebx >> 12) & 511) - cacheNumWays := 1 + ((ebx >> 22) & 511) - - typ := eax & 15 - size := int(cacheNumSets * cacheLineSize * cachePhysPartitions * cacheNumWays) - if typ == 0 { - return - } - - // Check for the same value repeated. - comb := eax ^ ebx ^ ecx - if comb == last { - nSame++ - if nSame == 100 { - return - } - } - last = comb - - switch level { - case 1: - switch typ { - case 1: - // Data cache - c.Cache.L1D = size - case 2: - // Inst cache - c.Cache.L1I = size - default: - if c.Cache.L1D < 0 { - c.Cache.L1I = size - } - if c.Cache.L1I < 0 { - c.Cache.L1I = size - } - } - case 2: - c.Cache.L2 = size - case 3: - c.Cache.L3 = size - } - } - } -} - -type SGXEPCSection struct { - BaseAddress uint64 - EPCSize uint64 -} - -type SGXSupport struct { - Available bool - LaunchControl bool - SGX1Supported bool - SGX2Supported bool - MaxEnclaveSizeNot64 int64 - MaxEnclaveSize64 int64 - EPCSections []SGXEPCSection -} - -func hasSGX(available, lc bool) (rval SGXSupport) { - rval.Available = available - - if !available { - return - } - - rval.LaunchControl = lc - - a, _, _, d := cpuidex(0x12, 0) - rval.SGX1Supported = a&0x01 != 0 - rval.SGX2Supported = a&0x02 != 0 - rval.MaxEnclaveSizeNot64 = 1 << (d & 0xFF) // pow 2 - rval.MaxEnclaveSize64 = 1 << ((d >> 8) & 0xFF) // pow 2 - rval.EPCSections = make([]SGXEPCSection, 0) - - for subleaf := uint32(2); subleaf < 2+8; subleaf++ { - eax, ebx, ecx, edx := cpuidex(0x12, subleaf) - leafType := eax & 0xf - - if leafType == 0 { - // Invalid subleaf, stop iterating - break - } else if leafType == 1 { - // EPC Section subleaf - baseAddress := uint64(eax&0xfffff000) + (uint64(ebx&0x000fffff) << 32) - size := uint64(ecx&0xfffff000) + (uint64(edx&0x000fffff) << 32) - - section := SGXEPCSection{BaseAddress: baseAddress, EPCSize: size} - rval.EPCSections = append(rval.EPCSections, section) - } - } - - return -} - -type AMDMemEncryptionSupport struct { - Available bool - CBitPossition uint32 - NumVMPL uint32 - PhysAddrReduction uint32 - NumEntryptedGuests uint32 - MinSevNoEsAsid uint32 -} - -func hasAMDMemEncryption(available bool) (rval AMDMemEncryptionSupport) { - rval.Available = available - if !available { - return - } - - _, b, c, d := cpuidex(0x8000001f, 0) - - rval.CBitPossition = b & 0x3f - rval.PhysAddrReduction = (b >> 6) & 0x3F - rval.NumVMPL = (b >> 12) & 0xf - rval.NumEntryptedGuests = c - rval.MinSevNoEsAsid = d - - return -} - -func support() flagSet { - var fs flagSet - mfi := maxFunctionID() - vend, _ := vendorID() - if mfi < 0x1 { - return fs - } - family, model, _ := familyModel() - - _, _, c, d := cpuid(1) - fs.setIf((d&(1<<0)) != 0, X87) - fs.setIf((d&(1<<8)) != 0, CMPXCHG8) - fs.setIf((d&(1<<11)) != 0, SYSEE) - fs.setIf((d&(1<<15)) != 0, CMOV) - fs.setIf((d&(1<<23)) != 0, MMX) - fs.setIf((d&(1<<24)) != 0, FXSR) - fs.setIf((d&(1<<25)) != 0, FXSROPT) - fs.setIf((d&(1<<25)) != 0, SSE) - fs.setIf((d&(1<<26)) != 0, SSE2) - fs.setIf((c&1) != 0, SSE3) - fs.setIf((c&(1<<5)) != 0, VMX) - fs.setIf((c&(1<<9)) != 0, SSSE3) - fs.setIf((c&(1<<19)) != 0, SSE4) - fs.setIf((c&(1<<20)) != 0, SSE42) - fs.setIf((c&(1<<25)) != 0, AESNI) - fs.setIf((c&(1<<1)) != 0, CLMUL) - fs.setIf(c&(1<<22) != 0, MOVBE) - fs.setIf(c&(1<<23) != 0, POPCNT) - fs.setIf(c&(1<<30) != 0, RDRAND) - - // This bit has been reserved by Intel & AMD for use by hypervisors, - // and indicates the presence of a hypervisor. - fs.setIf(c&(1<<31) != 0, HYPERVISOR) - fs.setIf(c&(1<<29) != 0, F16C) - fs.setIf(c&(1<<13) != 0, CX16) - - if vend == Intel && (d&(1<<28)) != 0 && mfi >= 4 { - fs.setIf(threadsPerCore() > 1, HTT) - } - if vend == AMD && (d&(1<<28)) != 0 && mfi >= 4 { - fs.setIf(threadsPerCore() > 1, HTT) - } - fs.setIf(c&1<<26 != 0, XSAVE) - fs.setIf(c&1<<27 != 0, OSXSAVE) - // Check XGETBV/XSAVE (26), OXSAVE (27) and AVX (28) bits - const avxCheck = 1<<26 | 1<<27 | 1<<28 - if c&avxCheck == avxCheck { - // Check for OS support - eax, _ := xgetbv(0) - if (eax & 0x6) == 0x6 { - fs.set(AVX) - switch vend { - case Intel: - // Older than Haswell. - fs.setIf(family == 6 && model < 60, AVXSLOW) - case AMD: - // Older than Zen 2 - fs.setIf(family < 23 || (family == 23 && model < 49), AVXSLOW) - } - } - } - // FMA3 can be used with SSE registers, so no OS support is strictly needed. - // fma3 and OSXSAVE needed. - const fma3Check = 1<<12 | 1<<27 - fs.setIf(c&fma3Check == fma3Check, FMA3) - - // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. - if mfi >= 7 { - _, ebx, ecx, edx := cpuidex(7, 0) - if fs.inSet(AVX) && (ebx&0x00000020) != 0 { - fs.set(AVX2) - } - // CPUID.(EAX=7, ECX=0).EBX - if (ebx & 0x00000008) != 0 { - fs.set(BMI1) - fs.setIf((ebx&0x00000100) != 0, BMI2) - } - fs.setIf(ebx&(1<<2) != 0, SGX) - fs.setIf(ebx&(1<<4) != 0, HLE) - fs.setIf(ebx&(1<<9) != 0, ERMS) - fs.setIf(ebx&(1<<11) != 0, RTM) - fs.setIf(ebx&(1<<14) != 0, MPX) - fs.setIf(ebx&(1<<18) != 0, RDSEED) - fs.setIf(ebx&(1<<19) != 0, ADX) - fs.setIf(ebx&(1<<29) != 0, SHA) - - // CPUID.(EAX=7, ECX=0).ECX - fs.setIf(ecx&(1<<5) != 0, WAITPKG) - fs.setIf(ecx&(1<<7) != 0, CETSS) - fs.setIf(ecx&(1<<8) != 0, GFNI) - fs.setIf(ecx&(1<<9) != 0, VAES) - fs.setIf(ecx&(1<<10) != 0, VPCLMULQDQ) - fs.setIf(ecx&(1<<13) != 0, TME) - fs.setIf(ecx&(1<<25) != 0, CLDEMOTE) - fs.setIf(ecx&(1<<23) != 0, KEYLOCKER) - fs.setIf(ecx&(1<<27) != 0, MOVDIRI) - fs.setIf(ecx&(1<<28) != 0, MOVDIR64B) - fs.setIf(ecx&(1<<29) != 0, ENQCMD) - fs.setIf(ecx&(1<<30) != 0, SGXLC) - - // CPUID.(EAX=7, ECX=0).EDX - fs.setIf(edx&(1<<4) != 0, FSRM) - fs.setIf(edx&(1<<9) != 0, SRBDS_CTRL) - fs.setIf(edx&(1<<10) != 0, MD_CLEAR) - fs.setIf(edx&(1<<11) != 0, RTM_ALWAYS_ABORT) - fs.setIf(edx&(1<<14) != 0, SERIALIZE) - fs.setIf(edx&(1<<15) != 0, HYBRID_CPU) - fs.setIf(edx&(1<<16) != 0, TSXLDTRK) - fs.setIf(edx&(1<<18) != 0, PCONFIG) - fs.setIf(edx&(1<<20) != 0, CETIBT) - fs.setIf(edx&(1<<26) != 0, IBPB) - fs.setIf(edx&(1<<27) != 0, STIBP) - fs.setIf(edx&(1<<28) != 0, FLUSH_L1D) - fs.setIf(edx&(1<<29) != 0, IA32_ARCH_CAP) - fs.setIf(edx&(1<<30) != 0, IA32_CORE_CAP) - fs.setIf(edx&(1<<31) != 0, SPEC_CTRL_SSBD) - - // CPUID.(EAX=7, ECX=1).EAX - eax1, _, _, edx1 := cpuidex(7, 1) - fs.setIf(fs.inSet(AVX) && eax1&(1<<4) != 0, AVXVNNI) - fs.setIf(eax1&(1<<7) != 0, CMPCCXADD) - fs.setIf(eax1&(1<<10) != 0, MOVSB_ZL) - fs.setIf(eax1&(1<<11) != 0, STOSB_SHORT) - fs.setIf(eax1&(1<<12) != 0, CMPSB_SCADBS_SHORT) - fs.setIf(eax1&(1<<22) != 0, HRESET) - fs.setIf(eax1&(1<<23) != 0, AVXIFMA) - fs.setIf(eax1&(1<<26) != 0, LAM) - - // CPUID.(EAX=7, ECX=1).EDX - fs.setIf(edx1&(1<<4) != 0, AVXVNNIINT8) - fs.setIf(edx1&(1<<5) != 0, AVXNECONVERT) - fs.setIf(edx1&(1<<10) != 0, AVXVNNIINT16) - fs.setIf(edx1&(1<<14) != 0, PREFETCHI) - fs.setIf(edx1&(1<<19) != 0, AVX10) - fs.setIf(edx1&(1<<21) != 0, APX_F) - - // Only detect AVX-512 features if XGETBV is supported - if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { - // Check for OS support - eax, _ := xgetbv(0) - - // Verify that XCR0[7:5] = ‘111b’ (OPMASK state, upper 256-bit of ZMM0-ZMM15 and - // ZMM16-ZMM31 state are enabled by OS) - /// and that XCR0[2:1] = ‘11b’ (XMM state and YMM state are enabled by OS). - hasAVX512 := (eax>>5)&7 == 7 && (eax>>1)&3 == 3 - if runtime.GOOS == "darwin" { - hasAVX512 = fs.inSet(AVX) && darwinHasAVX512() - } - if hasAVX512 { - fs.setIf(ebx&(1<<16) != 0, AVX512F) - fs.setIf(ebx&(1<<17) != 0, AVX512DQ) - fs.setIf(ebx&(1<<21) != 0, AVX512IFMA) - fs.setIf(ebx&(1<<26) != 0, AVX512PF) - fs.setIf(ebx&(1<<27) != 0, AVX512ER) - fs.setIf(ebx&(1<<28) != 0, AVX512CD) - fs.setIf(ebx&(1<<30) != 0, AVX512BW) - fs.setIf(ebx&(1<<31) != 0, AVX512VL) - // ecx - fs.setIf(ecx&(1<<1) != 0, AVX512VBMI) - fs.setIf(ecx&(1<<3) != 0, AMXFP8) - fs.setIf(ecx&(1<<6) != 0, AVX512VBMI2) - fs.setIf(ecx&(1<<11) != 0, AVX512VNNI) - fs.setIf(ecx&(1<<12) != 0, AVX512BITALG) - fs.setIf(ecx&(1<<14) != 0, AVX512VPOPCNTDQ) - // edx - fs.setIf(edx&(1<<8) != 0, AVX512VP2INTERSECT) - fs.setIf(edx&(1<<22) != 0, AMXBF16) - fs.setIf(edx&(1<<23) != 0, AVX512FP16) - fs.setIf(edx&(1<<24) != 0, AMXTILE) - fs.setIf(edx&(1<<25) != 0, AMXINT8) - // eax1 = CPUID.(EAX=7, ECX=1).EAX - fs.setIf(eax1&(1<<5) != 0, AVX512BF16) - fs.setIf(eax1&(1<<19) != 0, WRMSRNS) - fs.setIf(eax1&(1<<21) != 0, AMXFP16) - fs.setIf(eax1&(1<<27) != 0, MSRLIST) - } - } - - // CPUID.(EAX=7, ECX=2) - _, _, _, edx = cpuidex(7, 2) - fs.setIf(edx&(1<<0) != 0, PSFD) - fs.setIf(edx&(1<<1) != 0, IDPRED_CTRL) - fs.setIf(edx&(1<<2) != 0, RRSBA_CTRL) - fs.setIf(edx&(1<<4) != 0, BHI_CTRL) - fs.setIf(edx&(1<<5) != 0, MCDT_NO) - - // Add keylocker features. - if fs.inSet(KEYLOCKER) && mfi >= 0x19 { - _, ebx, _, _ := cpuidex(0x19, 0) - fs.setIf(ebx&5 == 5, KEYLOCKERW) // Bit 0 and 2 (1+4) - } - - // Add AVX10 features. - if fs.inSet(AVX10) && mfi >= 0x24 { - _, ebx, _, _ := cpuidex(0x24, 0) - fs.setIf(ebx&(1<<16) != 0, AVX10_128) - fs.setIf(ebx&(1<<17) != 0, AVX10_256) - fs.setIf(ebx&(1<<18) != 0, AVX10_512) - } - } - - // Processor Extended State Enumeration Sub-leaf (EAX = 0DH, ECX = 1) - // EAX - // Bit 00: XSAVEOPT is available. - // Bit 01: Supports XSAVEC and the compacted form of XRSTOR if set. - // Bit 02: Supports XGETBV with ECX = 1 if set. - // Bit 03: Supports XSAVES/XRSTORS and IA32_XSS if set. - // Bits 31 - 04: Reserved. - // EBX - // Bits 31 - 00: The size in bytes of the XSAVE area containing all states enabled by XCRO | IA32_XSS. - // ECX - // Bits 31 - 00: Reports the supported bits of the lower 32 bits of the IA32_XSS MSR. IA32_XSS[n] can be set to 1 only if ECX[n] is 1. - // EDX? - // Bits 07 - 00: Used for XCR0. Bit 08: PT state. Bit 09: Used for XCR0. Bits 12 - 10: Reserved. Bit 13: HWP state. Bits 31 - 14: Reserved. - if mfi >= 0xd { - if fs.inSet(XSAVE) { - eax, _, _, _ := cpuidex(0xd, 1) - fs.setIf(eax&(1<<0) != 0, XSAVEOPT) - fs.setIf(eax&(1<<1) != 0, XSAVEC) - fs.setIf(eax&(1<<2) != 0, XGETBV1) - fs.setIf(eax&(1<<3) != 0, XSAVES) - } - } - if maxExtendedFunction() >= 0x80000001 { - _, _, c, d := cpuid(0x80000001) - if (c & (1 << 5)) != 0 { - fs.set(LZCNT) - fs.set(POPCNT) - } - // ECX - fs.setIf((c&(1<<0)) != 0, LAHF) - fs.setIf((c&(1<<2)) != 0, SVM) - fs.setIf((c&(1<<6)) != 0, SSE4A) - fs.setIf((c&(1<<10)) != 0, IBS) - fs.setIf((c&(1<<22)) != 0, TOPEXT) - - // EDX - fs.setIf(d&(1<<11) != 0, SYSCALL) - fs.setIf(d&(1<<20) != 0, NX) - fs.setIf(d&(1<<22) != 0, MMXEXT) - fs.setIf(d&(1<<23) != 0, MMX) - fs.setIf(d&(1<<24) != 0, FXSR) - fs.setIf(d&(1<<25) != 0, FXSROPT) - fs.setIf(d&(1<<27) != 0, RDTSCP) - fs.setIf(d&(1<<30) != 0, AMD3DNOWEXT) - fs.setIf(d&(1<<31) != 0, AMD3DNOW) - - /* XOP and FMA4 use the AVX instruction coding scheme, so they can't be - * used unless the OS has AVX support. */ - if fs.inSet(AVX) { - fs.setIf((c&(1<<11)) != 0, XOP) - fs.setIf((c&(1<<16)) != 0, FMA4) - } - - } - if maxExtendedFunction() >= 0x80000007 { - _, b, _, d := cpuid(0x80000007) - fs.setIf((b&(1<<0)) != 0, MCAOVERFLOW) - fs.setIf((b&(1<<1)) != 0, SUCCOR) - fs.setIf((b&(1<<2)) != 0, HWA) - fs.setIf((d&(1<<9)) != 0, CPBOOST) - } - - if maxExtendedFunction() >= 0x80000008 { - _, b, _, _ := cpuid(0x80000008) - fs.setIf(b&(1<<28) != 0, PSFD) - fs.setIf(b&(1<<27) != 0, CPPC) - fs.setIf(b&(1<<24) != 0, SPEC_CTRL_SSBD) - fs.setIf(b&(1<<23) != 0, PPIN) - fs.setIf(b&(1<<21) != 0, TLB_FLUSH_NESTED) - fs.setIf(b&(1<<20) != 0, EFER_LMSLE_UNS) - fs.setIf(b&(1<<19) != 0, IBRS_PROVIDES_SMP) - fs.setIf(b&(1<<18) != 0, IBRS_PREFERRED) - fs.setIf(b&(1<<17) != 0, STIBP_ALWAYSON) - fs.setIf(b&(1<<15) != 0, STIBP) - fs.setIf(b&(1<<14) != 0, IBRS) - fs.setIf((b&(1<<13)) != 0, INT_WBINVD) - fs.setIf(b&(1<<12) != 0, IBPB) - fs.setIf((b&(1<<9)) != 0, WBNOINVD) - fs.setIf((b&(1<<8)) != 0, MCOMMIT) - fs.setIf((b&(1<<4)) != 0, RDPRU) - fs.setIf((b&(1<<3)) != 0, INVLPGB) - fs.setIf((b&(1<<1)) != 0, MSRIRC) - fs.setIf((b&(1<<0)) != 0, CLZERO) - } - - if fs.inSet(SVM) && maxExtendedFunction() >= 0x8000000A { - _, _, _, edx := cpuid(0x8000000A) - fs.setIf((edx>>0)&1 == 1, SVMNP) - fs.setIf((edx>>1)&1 == 1, LBRVIRT) - fs.setIf((edx>>2)&1 == 1, SVML) - fs.setIf((edx>>3)&1 == 1, NRIPS) - fs.setIf((edx>>4)&1 == 1, TSCRATEMSR) - fs.setIf((edx>>5)&1 == 1, VMCBCLEAN) - fs.setIf((edx>>6)&1 == 1, SVMFBASID) - fs.setIf((edx>>7)&1 == 1, SVMDA) - fs.setIf((edx>>10)&1 == 1, SVMPF) - fs.setIf((edx>>12)&1 == 1, SVMPFT) - } - - if maxExtendedFunction() >= 0x8000001a { - eax, _, _, _ := cpuid(0x8000001a) - fs.setIf((eax>>0)&1 == 1, FP128) - fs.setIf((eax>>1)&1 == 1, MOVU) - fs.setIf((eax>>2)&1 == 1, FP256) - } - - if maxExtendedFunction() >= 0x8000001b && fs.inSet(IBS) { - eax, _, _, _ := cpuid(0x8000001b) - fs.setIf((eax>>0)&1 == 1, IBSFFV) - fs.setIf((eax>>1)&1 == 1, IBSFETCHSAM) - fs.setIf((eax>>2)&1 == 1, IBSOPSAM) - fs.setIf((eax>>3)&1 == 1, IBSRDWROPCNT) - fs.setIf((eax>>4)&1 == 1, IBSOPCNT) - fs.setIf((eax>>5)&1 == 1, IBSBRNTRGT) - fs.setIf((eax>>6)&1 == 1, IBSOPCNTEXT) - fs.setIf((eax>>7)&1 == 1, IBSRIPINVALIDCHK) - fs.setIf((eax>>8)&1 == 1, IBS_OPFUSE) - fs.setIf((eax>>9)&1 == 1, IBS_FETCH_CTLX) - fs.setIf((eax>>10)&1 == 1, IBS_OPDATA4) // Doc says "Fixed,0. IBS op data 4 MSR supported", but assuming they mean 1. - fs.setIf((eax>>11)&1 == 1, IBS_ZEN4) - } - - if maxExtendedFunction() >= 0x8000001f && vend == AMD { - a, _, _, _ := cpuid(0x8000001f) - fs.setIf((a>>0)&1 == 1, SME) - fs.setIf((a>>1)&1 == 1, SEV) - fs.setIf((a>>2)&1 == 1, MSR_PAGEFLUSH) - fs.setIf((a>>3)&1 == 1, SEV_ES) - fs.setIf((a>>4)&1 == 1, SEV_SNP) - fs.setIf((a>>5)&1 == 1, VMPL) - fs.setIf((a>>10)&1 == 1, SME_COHERENT) - fs.setIf((a>>11)&1 == 1, SEV_64BIT) - fs.setIf((a>>12)&1 == 1, SEV_RESTRICTED) - fs.setIf((a>>13)&1 == 1, SEV_ALTERNATIVE) - fs.setIf((a>>14)&1 == 1, SEV_DEBUGSWAP) - fs.setIf((a>>15)&1 == 1, IBS_PREVENTHOST) - fs.setIf((a>>16)&1 == 1, VTE) - fs.setIf((a>>24)&1 == 1, VMSA_REGPROT) - } - - if maxExtendedFunction() >= 0x80000021 && vend == AMD { - a, _, _, _ := cpuid(0x80000021) - fs.setIf((a>>31)&1 == 1, SRSO_MSR_FIX) - fs.setIf((a>>30)&1 == 1, SRSO_USER_KERNEL_NO) - fs.setIf((a>>29)&1 == 1, SRSO_NO) - fs.setIf((a>>28)&1 == 1, IBPB_BRTYPE) - fs.setIf((a>>27)&1 == 1, SBPB) - } - - if mfi >= 0x20 { - // Microsoft has decided to purposefully hide the information - // of the guest TEE when VMs are being created using Hyper-V. - // - // This leads us to check for the Hyper-V cpuid features - // (0x4000000C), and then for the `ebx` value set. - // - // For Intel TDX, `ebx` is set as `0xbe3`, being 3 the part - // we're mostly interested about,according to: - // https://github.com/torvalds/linux/blob/d2f51b3516dade79269ff45eae2a7668ae711b25/arch/x86/include/asm/hyperv-tlfs.h#L169-L174 - _, ebx, _, _ := cpuid(0x4000000C) - fs.setIf(ebx == 0xbe3, TDX_GUEST) - } - - if mfi >= 0x21 { - // Intel Trusted Domain Extensions Guests have their own cpuid leaf (0x21). - _, ebx, ecx, edx := cpuid(0x21) - identity := string(valAsString(ebx, edx, ecx)) - fs.setIf(identity == "IntelTDX ", TDX_GUEST) - } - - return fs -} - -func (c *CPUInfo) supportAVX10() uint8 { - if c.maxFunc >= 0x24 && c.featureSet.inSet(AVX10) { - _, ebx, _, _ := cpuidex(0x24, 0) - return uint8(ebx) - } - return 0 -} - -func valAsString(values ...uint32) []byte { - r := make([]byte, 4*len(values)) - for i, v := range values { - dst := r[i*4:] - dst[0] = byte(v & 0xff) - dst[1] = byte((v >> 8) & 0xff) - dst[2] = byte((v >> 16) & 0xff) - dst[3] = byte((v >> 24) & 0xff) - switch { - case dst[0] == 0: - return r[:i*4] - case dst[1] == 0: - return r[:i*4+1] - case dst[2] == 0: - return r[:i*4+2] - case dst[3] == 0: - return r[:i*4+3] - } - } - return r -} diff --git a/vendor/github.com/klauspost/cpuid/v2/cpuid_386.s b/vendor/github.com/klauspost/cpuid/v2/cpuid_386.s deleted file mode 100644 index 8587c3a1f..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/cpuid_386.s +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//+build 386,!gccgo,!noasm,!appengine - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORL CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+4(FP) - MOVL BX, ebx+8(FP) - MOVL CX, ecx+12(FP) - MOVL DX, edx+16(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func xgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+4(FP) - MOVL DX, edx+8(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET - -// func asmDarwinHasAVX512() bool -TEXT ·asmDarwinHasAVX512(SB), 7, $0 - MOVL $0, eax+0(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/v2/cpuid_amd64.s b/vendor/github.com/klauspost/cpuid/v2/cpuid_amd64.s deleted file mode 100644 index bc11f8942..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/cpuid_amd64.s +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//+build amd64,!gccgo,!noasm,!appengine - -// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuid(SB), 7, $0 - XORQ CX, CX - MOVL op+0(FP), AX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -TEXT ·asmCpuidex(SB), 7, $0 - MOVL op+0(FP), AX - MOVL op2+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func asmXgetbv(index uint32) (eax, edx uint32) -TEXT ·asmXgetbv(SB), 7, $0 - MOVL index+0(FP), CX - BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV - MOVL AX, eax+8(FP) - MOVL DX, edx+12(FP) - RET - -// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -TEXT ·asmRdtscpAsm(SB), 7, $0 - BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP - MOVL AX, eax+0(FP) - MOVL BX, ebx+4(FP) - MOVL CX, ecx+8(FP) - MOVL DX, edx+12(FP) - RET - -// From https://go-review.googlesource.com/c/sys/+/285572/ -// func asmDarwinHasAVX512() bool -TEXT ·asmDarwinHasAVX512(SB), 7, $0-1 - MOVB $0, ret+0(FP) // default to false - -#ifdef GOOS_darwin // return if not darwin -#ifdef GOARCH_amd64 // return if not amd64 -// These values from: -// https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h -#define commpage64_base_address 0x00007fffffe00000 -#define commpage64_cpu_capabilities64 (commpage64_base_address+0x010) -#define commpage64_version (commpage64_base_address+0x01E) -#define hasAVX512F 0x0000004000000000 - MOVQ $commpage64_version, BX - MOVW (BX), AX - CMPW AX, $13 // versions < 13 do not support AVX512 - JL no_avx512 - MOVQ $commpage64_cpu_capabilities64, BX - MOVQ (BX), AX - MOVQ $hasAVX512F, CX - ANDQ CX, AX - JZ no_avx512 - MOVB $1, ret+0(FP) - -no_avx512: -#endif -#endif - RET - diff --git a/vendor/github.com/klauspost/cpuid/v2/cpuid_arm64.s b/vendor/github.com/klauspost/cpuid/v2/cpuid_arm64.s deleted file mode 100644 index b196f78eb..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/cpuid_arm64.s +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//+build arm64,!gccgo,!noasm,!appengine - -// See https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt - -// func getMidr -TEXT ·getMidr(SB), 7, $0 - WORD $0xd5380000 // mrs x0, midr_el1 /* Main ID Register */ - MOVD R0, midr+0(FP) - RET - -// func getProcFeatures -TEXT ·getProcFeatures(SB), 7, $0 - WORD $0xd5380400 // mrs x0, id_aa64pfr0_el1 /* Processor Feature Register 0 */ - MOVD R0, procFeatures+0(FP) - RET - -// func getInstAttributes -TEXT ·getInstAttributes(SB), 7, $0 - WORD $0xd5380600 // mrs x0, id_aa64isar0_el1 /* Instruction Set Attribute Register 0 */ - WORD $0xd5380621 // mrs x1, id_aa64isar1_el1 /* Instruction Set Attribute Register 1 */ - MOVD R0, instAttrReg0+0(FP) - MOVD R1, instAttrReg1+8(FP) - RET - -TEXT ·getVectorLength(SB), 7, $0 - WORD $0xd2800002 // mov x2, #0 - WORD $0x04225022 // addvl x2, x2, #1 - WORD $0xd37df042 // lsl x2, x2, #3 - WORD $0xd2800003 // mov x3, #0 - WORD $0x04635023 // addpl x3, x3, #1 - WORD $0xd37df063 // lsl x3, x3, #3 - MOVD R2, vl+0(FP) - MOVD R3, pl+8(FP) - RET diff --git a/vendor/github.com/klauspost/cpuid/v2/detect_arm64.go b/vendor/github.com/klauspost/cpuid/v2/detect_arm64.go deleted file mode 100644 index 566743d22..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/detect_arm64.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//go:build arm64 && !gccgo && !noasm && !appengine -// +build arm64,!gccgo,!noasm,!appengine - -package cpuid - -import "runtime" - -func getMidr() (midr uint64) -func getProcFeatures() (procFeatures uint64) -func getInstAttributes() (instAttrReg0, instAttrReg1 uint64) -func getVectorLength() (vl, pl uint64) - -func initCPU() { - cpuid = func(uint32) (a, b, c, d uint32) { return 0, 0, 0, 0 } - cpuidex = func(x, y uint32) (a, b, c, d uint32) { return 0, 0, 0, 0 } - xgetbv = func(uint32) (a, b uint32) { return 0, 0 } - rdtscpAsm = func() (a, b, c, d uint32) { return 0, 0, 0, 0 } -} - -func addInfo(c *CPUInfo, safe bool) { - // Seems to be safe to assume on ARM64 - c.CacheLine = 64 - detectOS(c) - - // ARM64 disabled since it may crash if interrupt is not intercepted by OS. - if safe && !c.Has(ARMCPUID) && runtime.GOOS != "freebsd" { - return - } - midr := getMidr() - - // MIDR_EL1 - Main ID Register - // https://developer.arm.com/docs/ddi0595/h/aarch64-system-registers/midr_el1 - // x--------------------------------------------------x - // | Name | bits | visible | - // |--------------------------------------------------| - // | Implementer | [31-24] | y | - // |--------------------------------------------------| - // | Variant | [23-20] | y | - // |--------------------------------------------------| - // | Architecture | [19-16] | y | - // |--------------------------------------------------| - // | PartNum | [15-4] | y | - // |--------------------------------------------------| - // | Revision | [3-0] | y | - // x--------------------------------------------------x - - switch (midr >> 24) & 0xff { - case 0xC0: - c.VendorString = "Ampere Computing" - c.VendorID = Ampere - case 0x41: - c.VendorString = "Arm Limited" - c.VendorID = ARM - case 0x42: - c.VendorString = "Broadcom Corporation" - c.VendorID = Broadcom - case 0x43: - c.VendorString = "Cavium Inc" - c.VendorID = Cavium - case 0x44: - c.VendorString = "Digital Equipment Corporation" - c.VendorID = DEC - case 0x46: - c.VendorString = "Fujitsu Ltd" - c.VendorID = Fujitsu - case 0x49: - c.VendorString = "Infineon Technologies AG" - c.VendorID = Infineon - case 0x4D: - c.VendorString = "Motorola or Freescale Semiconductor Inc" - c.VendorID = Motorola - case 0x4E: - c.VendorString = "NVIDIA Corporation" - c.VendorID = NVIDIA - case 0x50: - c.VendorString = "Applied Micro Circuits Corporation" - c.VendorID = AMCC - case 0x51: - c.VendorString = "Qualcomm Inc" - c.VendorID = Qualcomm - case 0x56: - c.VendorString = "Marvell International Ltd" - c.VendorID = Marvell - case 0x69: - c.VendorString = "Intel Corporation" - c.VendorID = Intel - } - - // Lower 4 bits: Architecture - // Architecture Meaning - // 0b0001 Armv4. - // 0b0010 Armv4T. - // 0b0011 Armv5 (obsolete). - // 0b0100 Armv5T. - // 0b0101 Armv5TE. - // 0b0110 Armv5TEJ. - // 0b0111 Armv6. - // 0b1111 Architectural features are individually identified in the ID_* registers, see 'ID registers'. - // Upper 4 bit: Variant - // An IMPLEMENTATION DEFINED variant number. - // Typically, this field is used to distinguish between different product variants, or major revisions of a product. - c.Family = int(midr>>16) & 0xff - - // PartNum, bits [15:4] - // An IMPLEMENTATION DEFINED primary part number for the device. - // On processors implemented by Arm, if the top four bits of the primary - // part number are 0x0 or 0x7, the variant and architecture are encoded differently. - // Revision, bits [3:0] - // An IMPLEMENTATION DEFINED revision number for the device. - c.Model = int(midr) & 0xffff - - procFeatures := getProcFeatures() - - // ID_AA64PFR0_EL1 - Processor Feature Register 0 - // x--------------------------------------------------x - // | Name | bits | visible | - // |--------------------------------------------------| - // | DIT | [51-48] | y | - // |--------------------------------------------------| - // | SVE | [35-32] | y | - // |--------------------------------------------------| - // | GIC | [27-24] | n | - // |--------------------------------------------------| - // | AdvSIMD | [23-20] | y | - // |--------------------------------------------------| - // | FP | [19-16] | y | - // |--------------------------------------------------| - // | EL3 | [15-12] | n | - // |--------------------------------------------------| - // | EL2 | [11-8] | n | - // |--------------------------------------------------| - // | EL1 | [7-4] | n | - // |--------------------------------------------------| - // | EL0 | [3-0] | n | - // x--------------------------------------------------x - - var f flagSet - // if procFeatures&(0xf<<48) != 0 { - // fmt.Println("DIT") - // } - f.setIf(procFeatures&(0xf<<32) != 0, SVE) - if procFeatures&(0xf<<20) != 15<<20 { - f.set(ASIMD) - // https://developer.arm.com/docs/ddi0595/b/aarch64-system-registers/id_aa64pfr0_el1 - // 0b0001 --> As for 0b0000, and also includes support for half-precision floating-point arithmetic. - f.setIf(procFeatures&(0xf<<20) == 1<<20, FPHP, ASIMDHP) - } - f.setIf(procFeatures&(0xf<<16) != 0, FP) - - instAttrReg0, instAttrReg1 := getInstAttributes() - - // https://developer.arm.com/docs/ddi0595/b/aarch64-system-registers/id_aa64isar0_el1 - // - // ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 - // x--------------------------------------------------x - // | Name | bits | visible | - // |--------------------------------------------------| - // | TS | [55-52] | y | - // |--------------------------------------------------| - // | FHM | [51-48] | y | - // |--------------------------------------------------| - // | DP | [47-44] | y | - // |--------------------------------------------------| - // | SM4 | [43-40] | y | - // |--------------------------------------------------| - // | SM3 | [39-36] | y | - // |--------------------------------------------------| - // | SHA3 | [35-32] | y | - // |--------------------------------------------------| - // | RDM | [31-28] | y | - // |--------------------------------------------------| - // | ATOMICS | [23-20] | y | - // |--------------------------------------------------| - // | CRC32 | [19-16] | y | - // |--------------------------------------------------| - // | SHA2 | [15-12] | y | - // |--------------------------------------------------| - // | SHA1 | [11-8] | y | - // |--------------------------------------------------| - // | AES | [7-4] | y | - // x--------------------------------------------------x - - // if instAttrReg0&(0xf<<52) != 0 { - // fmt.Println("TS") - // } - // if instAttrReg0&(0xf<<48) != 0 { - // fmt.Println("FHM") - // } - f.setIf(instAttrReg0&(0xf<<44) != 0, ASIMDDP) - f.setIf(instAttrReg0&(0xf<<40) != 0, SM4) - f.setIf(instAttrReg0&(0xf<<36) != 0, SM3) - f.setIf(instAttrReg0&(0xf<<32) != 0, SHA3) - f.setIf(instAttrReg0&(0xf<<28) != 0, ASIMDRDM) - f.setIf(instAttrReg0&(0xf<<20) != 0, ATOMICS) - f.setIf(instAttrReg0&(0xf<<16) != 0, CRC32) - f.setIf(instAttrReg0&(0xf<<12) != 0, SHA2) - // https://developer.arm.com/docs/ddi0595/b/aarch64-system-registers/id_aa64isar0_el1 - // 0b0010 --> As 0b0001, plus SHA512H, SHA512H2, SHA512SU0, and SHA512SU1 instructions implemented. - f.setIf(instAttrReg0&(0xf<<12) == 2<<12, SHA512) - f.setIf(instAttrReg0&(0xf<<8) != 0, SHA1) - f.setIf(instAttrReg0&(0xf<<4) != 0, AESARM) - // https://developer.arm.com/docs/ddi0595/b/aarch64-system-registers/id_aa64isar0_el1 - // 0b0010 --> As for 0b0001, plus PMULL/PMULL2 instructions operating on 64-bit data quantities. - f.setIf(instAttrReg0&(0xf<<4) == 2<<4, PMULL) - - // https://developer.arm.com/docs/ddi0595/b/aarch64-system-registers/id_aa64isar1_el1 - // - // ID_AA64ISAR1_EL1 - Instruction set attribute register 1 - // x--------------------------------------------------x - // | Name | bits | visible | - // |--------------------------------------------------| - // | GPI | [31-28] | y | - // |--------------------------------------------------| - // | GPA | [27-24] | y | - // |--------------------------------------------------| - // | LRCPC | [23-20] | y | - // |--------------------------------------------------| - // | FCMA | [19-16] | y | - // |--------------------------------------------------| - // | JSCVT | [15-12] | y | - // |--------------------------------------------------| - // | API | [11-8] | y | - // |--------------------------------------------------| - // | APA | [7-4] | y | - // |--------------------------------------------------| - // | DPB | [3-0] | y | - // x--------------------------------------------------x - - // if instAttrReg1&(0xf<<28) != 0 { - // fmt.Println("GPI") - // } - f.setIf(instAttrReg1&(0xf<<28) != 24, GPA) - f.setIf(instAttrReg1&(0xf<<20) != 0, LRCPC) - f.setIf(instAttrReg1&(0xf<<16) != 0, FCMA) - f.setIf(instAttrReg1&(0xf<<12) != 0, JSCVT) - // if instAttrReg1&(0xf<<8) != 0 { - // fmt.Println("API") - // } - // if instAttrReg1&(0xf<<4) != 0 { - // fmt.Println("APA") - // } - f.setIf(instAttrReg1&(0xf<<0) != 0, DCPOP) - - // Store - c.featureSet.or(f) -} diff --git a/vendor/github.com/klauspost/cpuid/v2/detect_ref.go b/vendor/github.com/klauspost/cpuid/v2/detect_ref.go deleted file mode 100644 index 574f9389c..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/detect_ref.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//go:build (!amd64 && !386 && !arm64) || gccgo || noasm || appengine -// +build !amd64,!386,!arm64 gccgo noasm appengine - -package cpuid - -func initCPU() { - cpuid = func(uint32) (a, b, c, d uint32) { return 0, 0, 0, 0 } - cpuidex = func(x, y uint32) (a, b, c, d uint32) { return 0, 0, 0, 0 } - xgetbv = func(uint32) (a, b uint32) { return 0, 0 } - rdtscpAsm = func() (a, b, c, d uint32) { return 0, 0, 0, 0 } - -} - -func addInfo(info *CPUInfo, safe bool) {} -func getVectorLength() (vl, pl uint64) { return 0, 0 } diff --git a/vendor/github.com/klauspost/cpuid/v2/detect_x86.go b/vendor/github.com/klauspost/cpuid/v2/detect_x86.go deleted file mode 100644 index f924c9d83..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/detect_x86.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. - -//go:build (386 && !gccgo && !noasm && !appengine) || (amd64 && !gccgo && !noasm && !appengine) -// +build 386,!gccgo,!noasm,!appengine amd64,!gccgo,!noasm,!appengine - -package cpuid - -func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) -func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) -func asmXgetbv(index uint32) (eax, edx uint32) -func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) -func asmDarwinHasAVX512() bool - -func initCPU() { - cpuid = asmCpuid - cpuidex = asmCpuidex - xgetbv = asmXgetbv - rdtscpAsm = asmRdtscpAsm - darwinHasAVX512 = asmDarwinHasAVX512 -} - -func addInfo(c *CPUInfo, safe bool) { - c.maxFunc = maxFunctionID() - c.maxExFunc = maxExtendedFunction() - c.BrandName = brandName() - c.CacheLine = cacheLine() - c.Family, c.Model, c.Stepping = familyModel() - c.featureSet = support() - c.SGX = hasSGX(c.featureSet.inSet(SGX), c.featureSet.inSet(SGXLC)) - c.AMDMemEncryption = hasAMDMemEncryption(c.featureSet.inSet(SME) || c.featureSet.inSet(SEV)) - c.ThreadsPerCore = threadsPerCore() - c.LogicalCores = logicalCores() - c.PhysicalCores = physicalCores() - c.VendorID, c.VendorString = vendorID() - c.HypervisorVendorID, c.HypervisorVendorString = hypervisorVendorID() - c.AVX10Level = c.supportAVX10() - c.cacheSize() - c.frequencies() -} - -func getVectorLength() (vl, pl uint64) { return 0, 0 } diff --git a/vendor/github.com/klauspost/cpuid/v2/featureid_string.go b/vendor/github.com/klauspost/cpuid/v2/featureid_string.go deleted file mode 100644 index e7f874a7e..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/featureid_string.go +++ /dev/null @@ -1,291 +0,0 @@ -// Code generated by "stringer -type=FeatureID,Vendor"; DO NOT EDIT. - -package cpuid - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[ADX-1] - _ = x[AESNI-2] - _ = x[AMD3DNOW-3] - _ = x[AMD3DNOWEXT-4] - _ = x[AMXBF16-5] - _ = x[AMXFP16-6] - _ = x[AMXINT8-7] - _ = x[AMXFP8-8] - _ = x[AMXTILE-9] - _ = x[APX_F-10] - _ = x[AVX-11] - _ = x[AVX10-12] - _ = x[AVX10_128-13] - _ = x[AVX10_256-14] - _ = x[AVX10_512-15] - _ = x[AVX2-16] - _ = x[AVX512BF16-17] - _ = x[AVX512BITALG-18] - _ = x[AVX512BW-19] - _ = x[AVX512CD-20] - _ = x[AVX512DQ-21] - _ = x[AVX512ER-22] - _ = x[AVX512F-23] - _ = x[AVX512FP16-24] - _ = x[AVX512IFMA-25] - _ = x[AVX512PF-26] - _ = x[AVX512VBMI-27] - _ = x[AVX512VBMI2-28] - _ = x[AVX512VL-29] - _ = x[AVX512VNNI-30] - _ = x[AVX512VP2INTERSECT-31] - _ = x[AVX512VPOPCNTDQ-32] - _ = x[AVXIFMA-33] - _ = x[AVXNECONVERT-34] - _ = x[AVXSLOW-35] - _ = x[AVXVNNI-36] - _ = x[AVXVNNIINT8-37] - _ = x[AVXVNNIINT16-38] - _ = x[BHI_CTRL-39] - _ = x[BMI1-40] - _ = x[BMI2-41] - _ = x[CETIBT-42] - _ = x[CETSS-43] - _ = x[CLDEMOTE-44] - _ = x[CLMUL-45] - _ = x[CLZERO-46] - _ = x[CMOV-47] - _ = x[CMPCCXADD-48] - _ = x[CMPSB_SCADBS_SHORT-49] - _ = x[CMPXCHG8-50] - _ = x[CPBOOST-51] - _ = x[CPPC-52] - _ = x[CX16-53] - _ = x[EFER_LMSLE_UNS-54] - _ = x[ENQCMD-55] - _ = x[ERMS-56] - _ = x[F16C-57] - _ = x[FLUSH_L1D-58] - _ = x[FMA3-59] - _ = x[FMA4-60] - _ = x[FP128-61] - _ = x[FP256-62] - _ = x[FSRM-63] - _ = x[FXSR-64] - _ = x[FXSROPT-65] - _ = x[GFNI-66] - _ = x[HLE-67] - _ = x[HRESET-68] - _ = x[HTT-69] - _ = x[HWA-70] - _ = x[HYBRID_CPU-71] - _ = x[HYPERVISOR-72] - _ = x[IA32_ARCH_CAP-73] - _ = x[IA32_CORE_CAP-74] - _ = x[IBPB-75] - _ = x[IBPB_BRTYPE-76] - _ = x[IBRS-77] - _ = x[IBRS_PREFERRED-78] - _ = x[IBRS_PROVIDES_SMP-79] - _ = x[IBS-80] - _ = x[IBSBRNTRGT-81] - _ = x[IBSFETCHSAM-82] - _ = x[IBSFFV-83] - _ = x[IBSOPCNT-84] - _ = x[IBSOPCNTEXT-85] - _ = x[IBSOPSAM-86] - _ = x[IBSRDWROPCNT-87] - _ = x[IBSRIPINVALIDCHK-88] - _ = x[IBS_FETCH_CTLX-89] - _ = x[IBS_OPDATA4-90] - _ = x[IBS_OPFUSE-91] - _ = x[IBS_PREVENTHOST-92] - _ = x[IBS_ZEN4-93] - _ = x[IDPRED_CTRL-94] - _ = x[INT_WBINVD-95] - _ = x[INVLPGB-96] - _ = x[KEYLOCKER-97] - _ = x[KEYLOCKERW-98] - _ = x[LAHF-99] - _ = x[LAM-100] - _ = x[LBRVIRT-101] - _ = x[LZCNT-102] - _ = x[MCAOVERFLOW-103] - _ = x[MCDT_NO-104] - _ = x[MCOMMIT-105] - _ = x[MD_CLEAR-106] - _ = x[MMX-107] - _ = x[MMXEXT-108] - _ = x[MOVBE-109] - _ = x[MOVDIR64B-110] - _ = x[MOVDIRI-111] - _ = x[MOVSB_ZL-112] - _ = x[MOVU-113] - _ = x[MPX-114] - _ = x[MSRIRC-115] - _ = x[MSRLIST-116] - _ = x[MSR_PAGEFLUSH-117] - _ = x[NRIPS-118] - _ = x[NX-119] - _ = x[OSXSAVE-120] - _ = x[PCONFIG-121] - _ = x[POPCNT-122] - _ = x[PPIN-123] - _ = x[PREFETCHI-124] - _ = x[PSFD-125] - _ = x[RDPRU-126] - _ = x[RDRAND-127] - _ = x[RDSEED-128] - _ = x[RDTSCP-129] - _ = x[RRSBA_CTRL-130] - _ = x[RTM-131] - _ = x[RTM_ALWAYS_ABORT-132] - _ = x[SBPB-133] - _ = x[SERIALIZE-134] - _ = x[SEV-135] - _ = x[SEV_64BIT-136] - _ = x[SEV_ALTERNATIVE-137] - _ = x[SEV_DEBUGSWAP-138] - _ = x[SEV_ES-139] - _ = x[SEV_RESTRICTED-140] - _ = x[SEV_SNP-141] - _ = x[SGX-142] - _ = x[SGXLC-143] - _ = x[SHA-144] - _ = x[SME-145] - _ = x[SME_COHERENT-146] - _ = x[SPEC_CTRL_SSBD-147] - _ = x[SRBDS_CTRL-148] - _ = x[SRSO_MSR_FIX-149] - _ = x[SRSO_NO-150] - _ = x[SRSO_USER_KERNEL_NO-151] - _ = x[SSE-152] - _ = x[SSE2-153] - _ = x[SSE3-154] - _ = x[SSE4-155] - _ = x[SSE42-156] - _ = x[SSE4A-157] - _ = x[SSSE3-158] - _ = x[STIBP-159] - _ = x[STIBP_ALWAYSON-160] - _ = x[STOSB_SHORT-161] - _ = x[SUCCOR-162] - _ = x[SVM-163] - _ = x[SVMDA-164] - _ = x[SVMFBASID-165] - _ = x[SVML-166] - _ = x[SVMNP-167] - _ = x[SVMPF-168] - _ = x[SVMPFT-169] - _ = x[SYSCALL-170] - _ = x[SYSEE-171] - _ = x[TBM-172] - _ = x[TDX_GUEST-173] - _ = x[TLB_FLUSH_NESTED-174] - _ = x[TME-175] - _ = x[TOPEXT-176] - _ = x[TSCRATEMSR-177] - _ = x[TSXLDTRK-178] - _ = x[VAES-179] - _ = x[VMCBCLEAN-180] - _ = x[VMPL-181] - _ = x[VMSA_REGPROT-182] - _ = x[VMX-183] - _ = x[VPCLMULQDQ-184] - _ = x[VTE-185] - _ = x[WAITPKG-186] - _ = x[WBNOINVD-187] - _ = x[WRMSRNS-188] - _ = x[X87-189] - _ = x[XGETBV1-190] - _ = x[XOP-191] - _ = x[XSAVE-192] - _ = x[XSAVEC-193] - _ = x[XSAVEOPT-194] - _ = x[XSAVES-195] - _ = x[AESARM-196] - _ = x[ARMCPUID-197] - _ = x[ASIMD-198] - _ = x[ASIMDDP-199] - _ = x[ASIMDHP-200] - _ = x[ASIMDRDM-201] - _ = x[ATOMICS-202] - _ = x[CRC32-203] - _ = x[DCPOP-204] - _ = x[EVTSTRM-205] - _ = x[FCMA-206] - _ = x[FP-207] - _ = x[FPHP-208] - _ = x[GPA-209] - _ = x[JSCVT-210] - _ = x[LRCPC-211] - _ = x[PMULL-212] - _ = x[SHA1-213] - _ = x[SHA2-214] - _ = x[SHA3-215] - _ = x[SHA512-216] - _ = x[SM3-217] - _ = x[SM4-218] - _ = x[SVE-219] - _ = x[lastID-220] - _ = x[firstID-0] -} - -const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXFP8AMXTILEAPX_FAVXAVX10AVX10_128AVX10_256AVX10_512AVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8AVXVNNIINT16BHI_CTRLBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBPB_BRTYPEIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4IDPRED_CTRLINT_WBINVDINVLPGBKEYLOCKERKEYLOCKERWLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSRLISTMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRRSBA_CTRLRTMRTM_ALWAYS_ABORTSBPBSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSRSO_MSR_FIXSRSO_NOSRSO_USER_KERNEL_NOSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTDX_GUESTTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDWRMSRNSX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID" - -var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 61, 68, 73, 76, 81, 90, 99, 108, 112, 122, 134, 142, 150, 158, 166, 173, 183, 193, 201, 211, 222, 230, 240, 258, 273, 280, 292, 299, 306, 317, 329, 337, 341, 345, 351, 356, 364, 369, 375, 379, 388, 406, 414, 421, 425, 429, 443, 449, 453, 457, 466, 470, 474, 479, 484, 488, 492, 499, 503, 506, 512, 515, 518, 528, 538, 551, 564, 568, 579, 583, 597, 614, 617, 627, 638, 644, 652, 663, 671, 683, 699, 713, 724, 734, 749, 757, 768, 778, 785, 794, 804, 808, 811, 818, 823, 834, 841, 848, 856, 859, 865, 870, 879, 886, 894, 898, 901, 907, 914, 927, 932, 934, 941, 948, 954, 958, 967, 971, 976, 982, 988, 994, 1004, 1007, 1023, 1027, 1036, 1039, 1048, 1063, 1076, 1082, 1096, 1103, 1106, 1111, 1114, 1117, 1129, 1143, 1153, 1165, 1172, 1191, 1194, 1198, 1202, 1206, 1211, 1216, 1221, 1226, 1240, 1251, 1257, 1260, 1265, 1274, 1278, 1283, 1288, 1294, 1301, 1306, 1309, 1318, 1334, 1337, 1343, 1353, 1361, 1365, 1374, 1378, 1390, 1393, 1403, 1406, 1413, 1421, 1428, 1431, 1438, 1441, 1446, 1452, 1460, 1466, 1472, 1480, 1485, 1492, 1499, 1507, 1514, 1519, 1524, 1531, 1535, 1537, 1541, 1544, 1549, 1554, 1559, 1563, 1567, 1571, 1577, 1580, 1583, 1586, 1592} - -func (i FeatureID) String() string { - if i < 0 || i >= FeatureID(len(_FeatureID_index)-1) { - return "FeatureID(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _FeatureID_name[_FeatureID_index[i]:_FeatureID_index[i+1]] -} -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[VendorUnknown-0] - _ = x[Intel-1] - _ = x[AMD-2] - _ = x[VIA-3] - _ = x[Transmeta-4] - _ = x[NSC-5] - _ = x[KVM-6] - _ = x[MSVM-7] - _ = x[VMware-8] - _ = x[XenHVM-9] - _ = x[Bhyve-10] - _ = x[Hygon-11] - _ = x[SiS-12] - _ = x[RDC-13] - _ = x[Ampere-14] - _ = x[ARM-15] - _ = x[Broadcom-16] - _ = x[Cavium-17] - _ = x[DEC-18] - _ = x[Fujitsu-19] - _ = x[Infineon-20] - _ = x[Motorola-21] - _ = x[NVIDIA-22] - _ = x[AMCC-23] - _ = x[Qualcomm-24] - _ = x[Marvell-25] - _ = x[QEMU-26] - _ = x[QNX-27] - _ = x[ACRN-28] - _ = x[SRE-29] - _ = x[Apple-30] - _ = x[lastVendor-31] -} - -const _Vendor_name = "VendorUnknownIntelAMDVIATransmetaNSCKVMMSVMVMwareXenHVMBhyveHygonSiSRDCAmpereARMBroadcomCaviumDECFujitsuInfineonMotorolaNVIDIAAMCCQualcommMarvellQEMUQNXACRNSREApplelastVendor" - -var _Vendor_index = [...]uint8{0, 13, 18, 21, 24, 33, 36, 39, 43, 49, 55, 60, 65, 68, 71, 77, 80, 88, 94, 97, 104, 112, 120, 126, 130, 138, 145, 149, 152, 156, 159, 164, 174} - -func (i Vendor) String() string { - if i < 0 || i >= Vendor(len(_Vendor_index)-1) { - return "Vendor(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _Vendor_name[_Vendor_index[i]:_Vendor_index[i+1]] -} diff --git a/vendor/github.com/klauspost/cpuid/v2/os_darwin_arm64.go b/vendor/github.com/klauspost/cpuid/v2/os_darwin_arm64.go deleted file mode 100644 index 84b1acd21..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/os_darwin_arm64.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2020 Klaus Post, released under MIT License. See LICENSE file. - -package cpuid - -import ( - "runtime" - "strings" - - "golang.org/x/sys/unix" -) - -func detectOS(c *CPUInfo) bool { - if runtime.GOOS != "ios" { - tryToFillCPUInfoFomSysctl(c) - } - // There are no hw.optional sysctl values for the below features on Mac OS 11.0 - // to detect their supported state dynamically. Assume the CPU features that - // Apple Silicon M1 supports to be available as a minimal set of features - // to all Go programs running on darwin/arm64. - // TODO: Add more if we know them. - c.featureSet.setIf(runtime.GOOS != "ios", AESARM, PMULL, SHA1, SHA2) - - return true -} - -func sysctlGetBool(name string) bool { - value, err := unix.SysctlUint32(name) - if err != nil { - return false - } - return value != 0 -} - -func sysctlGetString(name string) string { - value, err := unix.Sysctl(name) - if err != nil { - return "" - } - return value -} - -func sysctlGetInt(unknown int, names ...string) int { - for _, name := range names { - value, err := unix.SysctlUint32(name) - if err != nil { - continue - } - if value != 0 { - return int(value) - } - } - return unknown -} - -func sysctlGetInt64(unknown int, names ...string) int { - for _, name := range names { - value64, err := unix.SysctlUint64(name) - if err != nil { - continue - } - if int(value64) != unknown { - return int(value64) - } - } - return unknown -} - -func setFeature(c *CPUInfo, name string, feature FeatureID) { - c.featureSet.setIf(sysctlGetBool(name), feature) -} -func tryToFillCPUInfoFomSysctl(c *CPUInfo) { - c.BrandName = sysctlGetString("machdep.cpu.brand_string") - - if len(c.BrandName) != 0 { - c.VendorString = strings.Fields(c.BrandName)[0] - } - - c.PhysicalCores = sysctlGetInt(runtime.NumCPU(), "hw.physicalcpu") - c.ThreadsPerCore = sysctlGetInt(1, "machdep.cpu.thread_count", "kern.num_threads") / - sysctlGetInt(1, "hw.physicalcpu") - c.LogicalCores = sysctlGetInt(runtime.NumCPU(), "machdep.cpu.core_count") - c.Family = sysctlGetInt(0, "machdep.cpu.family", "hw.cpufamily") - c.Model = sysctlGetInt(0, "machdep.cpu.model") - c.CacheLine = sysctlGetInt64(0, "hw.cachelinesize") - c.Cache.L1I = sysctlGetInt64(-1, "hw.l1icachesize") - c.Cache.L1D = sysctlGetInt64(-1, "hw.l1dcachesize") - c.Cache.L2 = sysctlGetInt64(-1, "hw.l2cachesize") - c.Cache.L3 = sysctlGetInt64(-1, "hw.l3cachesize") - - // from https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile - setFeature(c, "hw.optional.arm.FEAT_AES", AESARM) - setFeature(c, "hw.optional.AdvSIMD", ASIMD) - setFeature(c, "hw.optional.arm.FEAT_DotProd", ASIMDDP) - setFeature(c, "hw.optional.arm.FEAT_RDM", ASIMDRDM) - setFeature(c, "hw.optional.FEAT_CRC32", CRC32) - setFeature(c, "hw.optional.arm.FEAT_DPB", DCPOP) - // setFeature(c, "", EVTSTRM) - setFeature(c, "hw.optional.arm.FEAT_FCMA", FCMA) - setFeature(c, "hw.optional.arm.FEAT_FP", FP) - setFeature(c, "hw.optional.arm.FEAT_FP16", FPHP) - setFeature(c, "hw.optional.arm.FEAT_PAuth", GPA) - setFeature(c, "hw.optional.arm.FEAT_JSCVT", JSCVT) - setFeature(c, "hw.optional.arm.FEAT_LRCPC", LRCPC) - setFeature(c, "hw.optional.arm.FEAT_PMULL", PMULL) - setFeature(c, "hw.optional.arm.FEAT_SHA1", SHA1) - setFeature(c, "hw.optional.arm.FEAT_SHA256", SHA2) - setFeature(c, "hw.optional.arm.FEAT_SHA3", SHA3) - setFeature(c, "hw.optional.arm.FEAT_SHA512", SHA512) - // setFeature(c, "", SM3) - // setFeature(c, "", SM4) - setFeature(c, "hw.optional.arm.FEAT_SVE", SVE) - - // from empirical observation - setFeature(c, "hw.optional.AdvSIMD_HPFPCvt", ASIMDHP) - setFeature(c, "hw.optional.armv8_1_atomics", ATOMICS) - setFeature(c, "hw.optional.floatingpoint", FP) - setFeature(c, "hw.optional.armv8_2_sha3", SHA3) - setFeature(c, "hw.optional.armv8_2_sha512", SHA512) - setFeature(c, "hw.optional.armv8_3_compnum", FCMA) - setFeature(c, "hw.optional.armv8_crc32", CRC32) -} diff --git a/vendor/github.com/klauspost/cpuid/v2/os_linux_arm64.go b/vendor/github.com/klauspost/cpuid/v2/os_linux_arm64.go deleted file mode 100644 index ee278b9e4..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/os_linux_arm64.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2020 Klaus Post, released under MIT License. See LICENSE file. - -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file located -// here https://github.com/golang/sys/blob/master/LICENSE - -package cpuid - -import ( - "encoding/binary" - "io/ioutil" - "runtime" -) - -// HWCAP bits. -const ( - hwcap_FP = 1 << 0 - hwcap_ASIMD = 1 << 1 - hwcap_EVTSTRM = 1 << 2 - hwcap_AES = 1 << 3 - hwcap_PMULL = 1 << 4 - hwcap_SHA1 = 1 << 5 - hwcap_SHA2 = 1 << 6 - hwcap_CRC32 = 1 << 7 - hwcap_ATOMICS = 1 << 8 - hwcap_FPHP = 1 << 9 - hwcap_ASIMDHP = 1 << 10 - hwcap_CPUID = 1 << 11 - hwcap_ASIMDRDM = 1 << 12 - hwcap_JSCVT = 1 << 13 - hwcap_FCMA = 1 << 14 - hwcap_LRCPC = 1 << 15 - hwcap_DCPOP = 1 << 16 - hwcap_SHA3 = 1 << 17 - hwcap_SM3 = 1 << 18 - hwcap_SM4 = 1 << 19 - hwcap_ASIMDDP = 1 << 20 - hwcap_SHA512 = 1 << 21 - hwcap_SVE = 1 << 22 - hwcap_ASIMDFHM = 1 << 23 -) - -func detectOS(c *CPUInfo) bool { - // For now assuming no hyperthreading is reasonable. - c.LogicalCores = runtime.NumCPU() - c.PhysicalCores = c.LogicalCores - c.ThreadsPerCore = 1 - if hwcap == 0 { - // We did not get values from the runtime. - // Try reading /proc/self/auxv - - // From https://github.com/golang/sys - const ( - _AT_HWCAP = 16 - _AT_HWCAP2 = 26 - - uintSize = int(32 << (^uint(0) >> 63)) - ) - - buf, err := ioutil.ReadFile("/proc/self/auxv") - if err != nil { - // e.g. on android /proc/self/auxv is not accessible, so silently - // ignore the error and leave Initialized = false. On some - // architectures (e.g. arm64) doinit() implements a fallback - // readout and will set Initialized = true again. - return false - } - bo := binary.LittleEndian - for len(buf) >= 2*(uintSize/8) { - var tag, val uint - switch uintSize { - case 32: - tag = uint(bo.Uint32(buf[0:])) - val = uint(bo.Uint32(buf[4:])) - buf = buf[8:] - case 64: - tag = uint(bo.Uint64(buf[0:])) - val = uint(bo.Uint64(buf[8:])) - buf = buf[16:] - } - switch tag { - case _AT_HWCAP: - hwcap = val - case _AT_HWCAP2: - // Not used - } - } - if hwcap == 0 { - return false - } - } - - // HWCap was populated by the runtime from the auxiliary vector. - // Use HWCap information since reading aarch64 system registers - // is not supported in user space on older linux kernels. - c.featureSet.setIf(isSet(hwcap, hwcap_AES), AESARM) - c.featureSet.setIf(isSet(hwcap, hwcap_ASIMD), ASIMD) - c.featureSet.setIf(isSet(hwcap, hwcap_ASIMDDP), ASIMDDP) - c.featureSet.setIf(isSet(hwcap, hwcap_ASIMDHP), ASIMDHP) - c.featureSet.setIf(isSet(hwcap, hwcap_ASIMDRDM), ASIMDRDM) - c.featureSet.setIf(isSet(hwcap, hwcap_CPUID), ARMCPUID) - c.featureSet.setIf(isSet(hwcap, hwcap_CRC32), CRC32) - c.featureSet.setIf(isSet(hwcap, hwcap_DCPOP), DCPOP) - c.featureSet.setIf(isSet(hwcap, hwcap_EVTSTRM), EVTSTRM) - c.featureSet.setIf(isSet(hwcap, hwcap_FCMA), FCMA) - c.featureSet.setIf(isSet(hwcap, hwcap_FP), FP) - c.featureSet.setIf(isSet(hwcap, hwcap_FPHP), FPHP) - c.featureSet.setIf(isSet(hwcap, hwcap_JSCVT), JSCVT) - c.featureSet.setIf(isSet(hwcap, hwcap_LRCPC), LRCPC) - c.featureSet.setIf(isSet(hwcap, hwcap_PMULL), PMULL) - c.featureSet.setIf(isSet(hwcap, hwcap_SHA1), SHA1) - c.featureSet.setIf(isSet(hwcap, hwcap_SHA2), SHA2) - c.featureSet.setIf(isSet(hwcap, hwcap_SHA3), SHA3) - c.featureSet.setIf(isSet(hwcap, hwcap_SHA512), SHA512) - c.featureSet.setIf(isSet(hwcap, hwcap_SM3), SM3) - c.featureSet.setIf(isSet(hwcap, hwcap_SM4), SM4) - c.featureSet.setIf(isSet(hwcap, hwcap_SVE), SVE) - - // The Samsung S9+ kernel reports support for atomics, but not all cores - // actually support them, resulting in SIGILL. See issue #28431. - // TODO(elias.naur): Only disable the optimization on bad chipsets on android. - c.featureSet.setIf(isSet(hwcap, hwcap_ATOMICS) && runtime.GOOS != "android", ATOMICS) - - return true -} - -func isSet(hwc uint, value uint) bool { - return hwc&value != 0 -} diff --git a/vendor/github.com/klauspost/cpuid/v2/os_other_arm64.go b/vendor/github.com/klauspost/cpuid/v2/os_other_arm64.go deleted file mode 100644 index 8733ba343..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/os_other_arm64.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2020 Klaus Post, released under MIT License. See LICENSE file. - -//go:build arm64 && !linux && !darwin -// +build arm64,!linux,!darwin - -package cpuid - -import "runtime" - -func detectOS(c *CPUInfo) bool { - c.PhysicalCores = runtime.NumCPU() - // For now assuming 1 thread per core... - c.ThreadsPerCore = 1 - c.LogicalCores = c.PhysicalCores - return false -} diff --git a/vendor/github.com/klauspost/cpuid/v2/os_safe_linux_arm64.go b/vendor/github.com/klauspost/cpuid/v2/os_safe_linux_arm64.go deleted file mode 100644 index f8f201b5f..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/os_safe_linux_arm64.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2021 Klaus Post, released under MIT License. See LICENSE file. - -//go:build nounsafe -// +build nounsafe - -package cpuid - -var hwcap uint diff --git a/vendor/github.com/klauspost/cpuid/v2/os_unsafe_linux_arm64.go b/vendor/github.com/klauspost/cpuid/v2/os_unsafe_linux_arm64.go deleted file mode 100644 index 92af622eb..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/os_unsafe_linux_arm64.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2021 Klaus Post, released under MIT License. See LICENSE file. - -//go:build !nounsafe -// +build !nounsafe - -package cpuid - -import _ "unsafe" // needed for go:linkname - -//go:linkname hwcap internal/cpu.HWCap -var hwcap uint diff --git a/vendor/github.com/klauspost/cpuid/v2/test-architectures.sh b/vendor/github.com/klauspost/cpuid/v2/test-architectures.sh deleted file mode 100644 index 471d986d2..000000000 --- a/vendor/github.com/klauspost/cpuid/v2/test-architectures.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -e - -go tool dist list | while IFS=/ read os arch; do - echo "Checking $os/$arch..." - echo " normal" - GOARCH=$arch GOOS=$os go build -o /dev/null . - echo " noasm" - GOARCH=$arch GOOS=$os go build -tags noasm -o /dev/null . - echo " appengine" - GOARCH=$arch GOOS=$os go build -tags appengine -o /dev/null . - echo " noasm,appengine" - GOARCH=$arch GOOS=$os go build -tags 'appengine noasm' -o /dev/null . -done |