From 37ee93f62dc7daf5863b4864e81a8acff63ca361 Mon Sep 17 00:00:00 2001 From: xzl Date: Fri, 8 Dec 2023 11:13:29 +0800 Subject: [PATCH] feat: update golang-github-eapache-go-xerial-snappy to 0.0_git20180814.776d571 Xerial-compatible Snappy framing support for Go Issue: https://github.com/linuxdeepin/developer-center/issues/3965 Log: update repo --- .travis.yml | 7 + LICENSE | 21 ++ README.md | 14 +- ...020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 | Bin 0 -> 8 bytes ...05979b224be0294bf350310d4ba5257c9bb815db-3 | 1 + ...0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 | Bin 0 -> 56 bytes corpus/1 | Bin 0 -> 31 bytes ...361a1c6d2a8f80780826c3d83ad391d0475c922f-4 | Bin 0 -> 50 bytes ...117af68228fa64339d362cf980c68ffadff96c8-12 | Bin 0 -> 248 bytes ...4142249be82c8a617cf838eef05394ece39becd3-9 | Bin 0 -> 76 bytes ...1ea8c7d904f1cd913b52e9ead4a96c639d76802-10 | Bin 0 -> 110 bytes ...44083e1447694980c0ee682576e32358c9ee883f-2 | Bin 0 -> 40 bytes ...4d6b359bd538feaa7d36c89235d07d0a443797ac-1 | Bin 0 -> 29 bytes ...521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 | 1 + ...526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 | Bin 0 -> 61 bytes .../581b8fe7088f921567811fdf30e1f527c9f48e5e | 1 + ...0cd10738158020f5843b43960158c3d116b3a71-11 | Bin 0 -> 195 bytes ...652b031b4b9d601235f86ef62523e63d733b8623-3 | Bin 0 -> 45 bytes ...684a011f6fdfc7ae9863e12381165e82d2a2e356-9 | Bin 0 -> 111 bytes ...72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 | Bin 0 -> 8 bytes ...80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 | 1 + ...8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 | Bin 0 -> 81 bytes ...9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 | Bin 0 -> 8 bytes ...99d36b0b5b1be7151a508dd440ec725a2576c41c-1 | Bin 0 -> 8 bytes ...9d339eddb4e2714ea319c3fb571311cb95fdb067-6 | Bin 0 -> 55 bytes ...b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 | 1 + ...c1951b29109ec1017f63535ce3699630f46f54e1-5 | Bin 0 -> 50 bytes ...cb806bc4f67316af02d6ae677332a3b6005a18da-5 | 1 + ...d7dd228703739e9252c7ea76f1c5f82ab44686a-10 | Bin 0 -> 96 bytes ...ce3671e91907349cea04fc3f2a4b91c65b99461d-3 | Bin 0 -> 36 bytes ...ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 | 1 + ...da39a3ee5e6b4b0d3255bfef95601890afd80709-1 | 0 ...e2230aa0ecaebb9b890440effa13f501a89247b2-1 | Bin 0 -> 35 bytes ...fa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 | Bin 0 -> 116 bytes ...f0445ac39e03978bbc8011316ac8468015ddb72c-1 | Bin 0 -> 20 bytes ...f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 | Bin 0 -> 20 bytes debian/changelog | 33 ++- debian/compat | 1 - debian/control | 39 ++- debian/copyright | 44 ++-- debian/gbp.conf | 2 + debian/gitlab-ci.yml | 6 + debian/rules | 2 +- debian/upstream/metadata | 4 + debian/watch | 4 + fuzz.go | 16 ++ snappy.go | 131 +++++++++ snappy_test.go | 249 ++++++++++++++++++ 48 files changed, 544 insertions(+), 36 deletions(-) create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 create mode 100644 corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 create mode 100644 corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 create mode 100644 corpus/1 create mode 100644 corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 create mode 100644 corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 create mode 100644 corpus/4142249be82c8a617cf838eef05394ece39becd3-9 create mode 100644 corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10 create mode 100644 corpus/44083e1447694980c0ee682576e32358c9ee883f-2 create mode 100644 corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1 create mode 100644 corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 create mode 100644 corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 create mode 100644 corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e create mode 100644 corpus/60cd10738158020f5843b43960158c3d116b3a71-11 create mode 100644 corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 create mode 100644 corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9 create mode 100644 corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 create mode 100644 corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 create mode 100644 corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 create mode 100644 corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 create mode 100644 corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1 create mode 100644 corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6 create mode 100644 corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 create mode 100644 corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5 create mode 100644 corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5 create mode 100644 corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10 create mode 100644 corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3 create mode 100644 corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 create mode 100644 corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1 create mode 100644 corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1 create mode 100644 corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 create mode 100644 corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1 create mode 100644 corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 delete mode 100644 debian/compat create mode 100644 debian/gbp.conf create mode 100644 debian/gitlab-ci.yml create mode 100644 debian/upstream/metadata create mode 100644 debian/watch create mode 100644 fuzz.go create mode 100644 snappy.go create mode 100644 snappy_test.go diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d6cf4f1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: go + +go: +- 1.5.4 +- 1.6.1 + +sudo: false diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5bf3688 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 9ebb840..3f2695c 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ -# template-repository \ No newline at end of file +# go-xerial-snappy + +[![Build Status](https://travis-ci.org/eapache/go-xerial-snappy.svg?branch=master)](https://travis-ci.org/eapache/go-xerial-snappy) + +Xerial-compatible Snappy framing support for golang. + +Packages using Xerial for snappy encoding use a framing format incompatible with +basically everything else in existence. This package wraps Go's built-in snappy +package to support it. + +Apps that use this format include Apache Kafka (see +https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for +details). diff --git a/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 b/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 new file mode 100644 index 0000000000000000000000000000000000000000..1b704ae0ea46b0fd879bebf2ed4b1638b3016f4f GIT binary patch literal 8 PcmaFAfA9PKdm|YD8@B~8 literal 0 HcmV?d00001 diff --git a/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 b/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 new file mode 100644 index 0000000..8c321a6 --- /dev/null +++ b/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 @@ -0,0 +1 @@ +����Y \ No newline at end of file diff --git a/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 b/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 new file mode 100644 index 0000000000000000000000000000000000000000..d413114b2d7a6b16ef8fcbf79853b24a4747dacb GIT binary patch literal 56 vcmZn)_Hzsfh-6@pV_;-p0Ap@Wg#aH%Pd^vO5TFQHhyf@DWr0*NFe(55)U*Xx literal 0 HcmV?d00001 diff --git a/corpus/1 b/corpus/1 new file mode 100644 index 0000000000000000000000000000000000000000..5e3abaf35b463142c5e53036bf70063aa37f18ca GIT binary patch literal 31 fcmZn)_Hzsfh-3f)MkwayR0!~K^z?Ia3~>YiQUnDe literal 0 HcmV?d00001 diff --git a/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 b/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 new file mode 100644 index 0000000000000000000000000000000000000000..d2528bad484a4fe7372fc3858461660fef90fba3 GIT binary patch literal 50 jcmZn)_Hzsfh-3f)MkwayR0!~K^z?Ia41o!Q<)8`xu9*bo literal 0 HcmV?d00001 diff --git a/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 b/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 new file mode 100644 index 0000000000000000000000000000000000000000..38ee90f859b809677a61199f7c11b5f8af80b33c GIT binary patch literal 248 zcmZn)_Hzsfh-6@pV_;-p0Ap@W1sBHW~Em literal 0 HcmV?d00001 diff --git a/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e b/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e new file mode 100644 index 0000000..59c77e4 --- /dev/null +++ b/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e @@ -0,0 +1 @@ +package \ No newline at end of file diff --git a/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 b/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 new file mode 100644 index 0000000000000000000000000000000000000000..801e2cf29ca389c2b507273e2b5bb7a330433c8e GIT binary patch literal 195 bcmZn)_Hzsfh-3f)KNrUk5Y0HG2u20~rP%|i literal 0 HcmV?d00001 diff --git a/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 b/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 new file mode 100644 index 0000000000000000000000000000000000000000..76bd9d9ebc765a0a6c28aece7027fa0e2b77b5c8 GIT binary patch literal 45 kcmZn)_Hzsfh-3f)MkwayR0!~K^z?Ia41o!Qa>0sv{022=n5 literal 0 HcmV?d00001 diff --git a/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 b/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 new file mode 100644 index 0000000000000000000000000000000000000000..2031634884f95d331e0cd7b237ae3e0131f43f6e GIT binary patch literal 116 zcmZn)_Hzsfh-6@pV_;-p0Ap@W1sBH Sat, 28 Jan 2023 13:46:49 +0800 + [ Alexandre Viau ] + * Point Vcs-* urls to salsa.debian.org. + + [ Debian Janitor ] + * Bump debhelper from old 11 to 12. + * Set debhelper-compat version in Build-Depends. + * Set upstream metadata fields: Bug-Database, Bug-Submit, Repository, + Repository-Browse. + * Apply multi-arch hints. + + [ Shengjing Zhu ] + * Change section to golang + * Update maintainer to team+pkg-go@tracker.debian.org + * Bump debhelper-compat to 13 + * Replace transitional package golang-snappy-go-dev + * Add Rules-Requires-Root + * Update Standards-Version to 4.6.2 (no changes) + * Add uscan watch file + * New upstream version 0.0~git20180814.776d571 + + -- Shengjing Zhu Sun, 01 Jan 2023 17:22:02 +0800 + +golang-github-eapache-go-xerial-snappy (0.0~git20160609.bb955e0-1) unstable; urgency=medium + + * Initial release (Closes: #840626) + Thanks to Guillem Jover for his initial work. + + -- Christos Trochalakis Wed, 24 Jan 2018 13:00:44 +0200 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index b4de394..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -11 diff --git a/debian/control b/debian/control index cb7c4a0..0879e6e 100644 --- a/debian/control +++ b/debian/control @@ -1,15 +1,28 @@ -Source: template-repository -Section: unknown +Source: golang-github-eapache-go-xerial-snappy +Section: golang Priority: optional -Maintainer: Tsic404 -Build-Depends: debhelper (>= 11) -Standards-Version: 4.1.3 -Homepage: https://github.com/deepin-community/template-repository -#Vcs-Browser: https://salsa.debian.org/debian/deepin-community-template-repository -#Vcs-Git: https://salsa.debian.org/debian/deepin-community-template-repository.git +Maintainer: Debian Go Packaging Team +Uploaders: Christos Trochalakis , +Build-Depends: debhelper-compat (= 13), + dh-golang, + golang-any, + golang-github-golang-snappy-dev, +Standards-Version: 4.6.2 +Homepage: https://github.com/eapache/go-xerial-snappy +Vcs-Browser: https://salsa.debian.org/go-team/packages/golang-github-eapache-go-xerial-snappy +Vcs-Git: https://salsa.debian.org/go-team/packages/golang-github-eapache-go-xerial-snappy.git +XS-Go-Import-Path: github.com/eapache/go-xerial-snappy +Rules-Requires-Root: no +Testsuite: autopkgtest-pkg-go -Package: template-repository -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends} -Description: - +Package: golang-github-eapache-go-xerial-snappy-dev +Architecture: all +Depends: golang-github-golang-snappy-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Multi-Arch: foreign +Description: Xerial-compatible Snappy framing support for Go + The package wraps Go's built-in Snappy package (github.com/golang/snappy) + adding support for a Xerial compatible framing format. + . + The package was extracted from Shopify's sarama Kafka client. diff --git a/debian/copyright b/debian/copyright index f5c805e..9fa1207 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,22 +1,32 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: template-repository -Source: https://github.com/deepin-community/template-repository +Upstream-Name: go-xerial-snappy +Source: https://github.com/eapache/go-xerial-snappy Files: * -Copyright: 2023 Tsic404 -License: GPL-2+ - This package is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - . - This package is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +Copyright: + Copyright © 2016 Evan Huus +License: MIT + +Files: debian/* +Copyright: + Copyright 2018 Christos Trochalakis +License: MIT + +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: . - You should have received a copy of the GNU General Public License - along with this program. If not, see + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. . - On Debian systems, the complete text of the GNU General - Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..cec628c --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,2 @@ +[DEFAULT] +pristine-tar = True diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml new file mode 100644 index 0000000..594e14e --- /dev/null +++ b/debian/gitlab-ci.yml @@ -0,0 +1,6 @@ +# auto-generated, DO NOT MODIFY. +# The authoritative copy of this file lives at: +# https://salsa.debian.org/go-team/infra/pkg-go-tools/blob/master/config/gitlabciyml.go +--- +include: + - https://salsa.debian.org/go-team/infra/pkg-go-tools/-/raw/master/pipeline/test-archive.yml diff --git a/debian/rules b/debian/rules index 2d33f6a..8cce5e0 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,4 @@ #!/usr/bin/make -f %: - dh $@ + dh $@ --buildsystem=golang --with=golang diff --git a/debian/upstream/metadata b/debian/upstream/metadata new file mode 100644 index 0000000..c8b41c7 --- /dev/null +++ b/debian/upstream/metadata @@ -0,0 +1,4 @@ +Bug-Database: https://github.com/eapache/go-xerial-snappy/issues +Bug-Submit: https://github.com/eapache/go-xerial-snappy/issues/new +Repository: https://github.com/eapache/go-xerial-snappy.git +Repository-Browse: https://github.com/eapache/go-xerial-snappy diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..5549cd0 --- /dev/null +++ b/debian/watch @@ -0,0 +1,4 @@ +version=4 +opts="mode=git, pgpmode=none" \ + https://github.com/eapache/go-xerial-snappy \ + HEAD debian diff --git a/fuzz.go b/fuzz.go new file mode 100644 index 0000000..6a46f47 --- /dev/null +++ b/fuzz.go @@ -0,0 +1,16 @@ +// +build gofuzz + +package snappy + +func Fuzz(data []byte) int { + decode, err := Decode(data) + if decode == nil && err == nil { + panic("nil error with nil result") + } + + if err != nil { + return 0 + } + + return 1 +} diff --git a/snappy.go b/snappy.go new file mode 100644 index 0000000..ea8f7af --- /dev/null +++ b/snappy.go @@ -0,0 +1,131 @@ +package snappy + +import ( + "bytes" + "encoding/binary" + "errors" + + master "github.com/golang/snappy" +) + +const ( + sizeOffset = 16 + sizeBytes = 4 +) + +var ( + xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0} + + // This is xerial version 1 and minimally compatible with version 1 + xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1} + + // ErrMalformed is returned by the decoder when the xerial framing + // is malformed + ErrMalformed = errors.New("malformed xerial framing") +) + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +// Encode encodes data as snappy with no framing header. +func Encode(src []byte) []byte { + return master.Encode(nil, src) +} + +// EncodeStream *appends* to the specified 'dst' the compressed +// 'src' in xerial framing format. If 'dst' does not have enough +// capacity, then a new slice will be allocated. If 'dst' has +// non-zero length, then if *must* have been built using this function. +func EncodeStream(dst, src []byte) []byte { + if len(dst) == 0 { + dst = append(dst, xerialHeader...) + dst = append(dst, xerialVersionInfo...) + } + + // Snappy encode in blocks of maximum 32KB + var ( + max = len(src) + blockSize = 32 * 1024 + pos = 0 + chunk []byte + ) + + for pos < max { + newPos := min(pos + blockSize, max) + chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos]) + + // First encode the compressed size (big-endian) + // Put* panics if the buffer is too small, so pad 4 bytes first + origLen := len(dst) + dst = append(dst, dst[0:4]...) + binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk))) + + // And now the compressed data + dst = append(dst, chunk...) + pos = newPos + } + return dst +} + +// Decode decodes snappy data whether it is traditional unframed +// or includes the xerial framing format. +func Decode(src []byte) ([]byte, error) { + return DecodeInto(nil, src) +} + +// DecodeInto decodes snappy data whether it is traditional unframed +// or includes the xerial framing format into the specified `dst`. +// It is assumed that the entirety of `dst` including all capacity is available +// for use by this function. If `dst` is nil *or* insufficiently large to hold +// the decoded `src`, new space will be allocated. +func DecodeInto(dst, src []byte) ([]byte, error) { + var max = len(src) + if max < len(xerialHeader) { + return nil, ErrMalformed + } + + if !bytes.Equal(src[:8], xerialHeader) { + return master.Decode(dst[:cap(dst)], src) + } + + if max < sizeOffset+sizeBytes { + return nil, ErrMalformed + } + + if dst == nil { + dst = make([]byte, 0, len(src)) + } + + dst = dst[:0] + var ( + pos = sizeOffset + chunk []byte + err error + ) + + for pos+sizeBytes <= max { + size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes])) + pos += sizeBytes + + nextPos := pos + size + // On architectures where int is 32-bytes wide size + pos could + // overflow so we need to check the low bound as well as the + // high + if nextPos < pos || nextPos > max { + return nil, ErrMalformed + } + + chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos]) + + if err != nil { + return nil, err + } + pos = nextPos + dst = append(dst, chunk...) + } + return dst, nil +} diff --git a/snappy_test.go b/snappy_test.go new file mode 100644 index 0000000..bdc41bf --- /dev/null +++ b/snappy_test.go @@ -0,0 +1,249 @@ +package snappy + +import ( + "bytes" + "testing" +) + +const largeString = `Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias except` + +var snappyTestCases = map[string][]byte{ + "REPEATREPEATREPEATREPEATREPEATREPEAT": {36, 20, 82, 69, 80, 69, 65, 84, 118, 6, 0}, + "REALLY SHORT": {12, 44, 82, 69, 65, 76, 76, 89, 32, 83, 72, 79, 82, 84}, + "AXBXCXDXEXFX": {12, 44, 65, 88, 66, 88, 67, 88, 68, 88, 69, 88, 70, 88}, +} + +var snappyStreamTestCases = map[string][]byte{ + "PLAINDATA": {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 11, 9, 32, 80, 76, 65, 73, 78, 68, 65, 84, 65}, + `{"a":"UtaitILHMDAAAAfU","b":"日本"}`: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 39, 37, 144, 123, 34, 97, 34, 58, 34, 85, 116, 97, 105, 116, 73, 76, 72, 77, 68, 65, 65, 65, 65, 102, 85, 34, 44, 34, 98, 34, 58, 34, 230, 151, 165, 230, 156, 172, 34, 125}, + largeString: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 89, 128, 8, 240, 90, 83, 101, 100, 32, 117, 116, 32, 112, 101, 114, 115, 112, 105, 99, 105, 97, 116, 105, 115, 32, 117, 110, 100, 101, 32, 111, 109, 110, 105, 115, 32, 105, 115, 116, 101, 32, 110, 97, 116, 117, 115, 32, 101, 114, 114, 111, 114, 32, 115, 105, 116, 32, 118, 111, 108, 117, 112, 116, 97, 116, 101, 109, 32, 97, 99, 99, 117, 115, 97, 110, 116, 105, 117, 109, 32, 100, 111, 108, 111, 114, 101, 109, 113, 117, 101, 32, 108, 97, 117, 100, 97, 5, 22, 240, 60, 44, 32, 116, 111, 116, 97, 109, 32, 114, 101, 109, 32, 97, 112, 101, 114, 105, 97, 109, 44, 32, 101, 97, 113, 117, 101, 32, 105, 112, 115, 97, 32, 113, 117, 97, 101, 32, 97, 98, 32, 105, 108, 108, 111, 32, 105, 110, 118, 101, 110, 116, 111, 114, 101, 32, 118, 101, 114, 105, 116, 97, 1, 141, 4, 101, 116, 1, 36, 88, 115, 105, 32, 97, 114, 99, 104, 105, 116, 101, 99, 116, 111, 32, 98, 101, 97, 116, 97, 101, 32, 118, 105, 1, 6, 120, 100, 105, 99, 116, 97, 32, 115, 117, 110, 116, 32, 101, 120, 112, 108, 105, 99, 97, 98, 111, 46, 32, 78, 101, 109, 111, 32, 101, 110, 105, 109, 5, 103, 0, 109, 46, 180, 0, 12, 113, 117, 105, 97, 17, 16, 0, 115, 5, 209, 72, 97, 115, 112, 101, 114, 110, 97, 116, 117, 114, 32, 97, 117, 116, 32, 111, 100, 105, 116, 5, 9, 36, 102, 117, 103, 105, 116, 44, 32, 115, 101, 100, 9, 53, 32, 99, 111, 110, 115, 101, 113, 117, 117, 110, 1, 42, 20, 109, 97, 103, 110, 105, 32, 9, 245, 16, 115, 32, 101, 111, 115, 1, 36, 28, 32, 114, 97, 116, 105, 111, 110, 101, 17, 96, 33, 36, 1, 51, 36, 105, 32, 110, 101, 115, 99, 105, 117, 110, 116, 1, 155, 1, 254, 16, 112, 111, 114, 114, 111, 1, 51, 36, 115, 113, 117, 97, 109, 32, 101, 115, 116, 44, 1, 14, 13, 81, 5, 183, 4, 117, 109, 1, 18, 0, 97, 9, 19, 4, 32, 115, 1, 149, 12, 109, 101, 116, 44, 9, 135, 76, 99, 116, 101, 116, 117, 114, 44, 32, 97, 100, 105, 112, 105, 115, 99, 105, 32, 118, 101, 108, 50, 173, 0, 24, 110, 111, 110, 32, 110, 117, 109, 9, 94, 84, 105, 117, 115, 32, 109, 111, 100, 105, 32, 116, 101, 109, 112, 111, 114, 97, 32, 105, 110, 99, 105, 100, 33, 52, 20, 117, 116, 32, 108, 97, 98, 33, 116, 4, 101, 116, 9, 106, 0, 101, 5, 219, 20, 97, 109, 32, 97, 108, 105, 5, 62, 33, 164, 8, 114, 97, 116, 29, 212, 12, 46, 32, 85, 116, 41, 94, 52, 97, 100, 32, 109, 105, 110, 105, 109, 97, 32, 118, 101, 110, 105, 33, 221, 72, 113, 117, 105, 115, 32, 110, 111, 115, 116, 114, 117, 109, 32, 101, 120, 101, 114, 99, 105, 33, 202, 104, 111, 110, 101, 109, 32, 117, 108, 108, 97, 109, 32, 99, 111, 114, 112, 111, 114, 105, 115, 32, 115, 117, 115, 99, 105, 112, 105, 13, 130, 8, 105, 111, 115, 1, 64, 12, 110, 105, 115, 105, 1, 150, 5, 126, 44, 105, 100, 32, 101, 120, 32, 101, 97, 32, 99, 111, 109, 5, 192, 0, 99, 41, 131, 33, 172, 8, 63, 32, 81, 1, 107, 4, 97, 117, 33, 101, 96, 118, 101, 108, 32, 101, 117, 109, 32, 105, 117, 114, 101, 32, 114, 101, 112, 114, 101, 104, 101, 110, 100, 101, 114, 105, 65, 63, 12, 105, 32, 105, 110, 1, 69, 16, 118, 111, 108, 117, 112, 65, 185, 1, 47, 24, 105, 116, 32, 101, 115, 115, 101, 1, 222, 64, 109, 32, 110, 105, 104, 105, 108, 32, 109, 111, 108, 101, 115, 116, 105, 97, 101, 46, 103, 0, 0, 44, 1, 45, 16, 32, 105, 108, 108, 117, 37, 143, 45, 36, 0, 109, 5, 110, 65, 33, 20, 97, 116, 32, 113, 117, 111, 17, 92, 44, 115, 32, 110, 117, 108, 108, 97, 32, 112, 97, 114, 105, 9, 165, 24, 65, 116, 32, 118, 101, 114, 111, 69, 34, 44, 101, 116, 32, 97, 99, 99, 117, 115, 97, 109, 117, 115, 1, 13, 104, 105, 117, 115, 116, 111, 32, 111, 100, 105, 111, 32, 100, 105, 103, 110, 105, 115, 115, 105, 109, 111, 115, 32, 100, 117, 99, 105, 1, 34, 80, 113, 117, 105, 32, 98, 108, 97, 110, 100, 105, 116, 105, 105, 115, 32, 112, 114, 97, 101, 115, 101, 101, 87, 17, 111, 56, 116, 117, 109, 32, 100, 101, 108, 101, 110, 105, 116, 105, 32, 97, 116, 65, 89, 28, 99, 111, 114, 114, 117, 112, 116, 105, 1, 150, 0, 115, 13, 174, 5, 109, 8, 113, 117, 97, 65, 5, 52, 108, 101, 115, 116, 105, 97, 115, 32, 101, 120, 99, 101, 112, 116, 0, 0, 0, 1, 0}, +} + +func makeMassive(input string, numCopies int) string { + outBuff := make([]byte, len(input) * numCopies) + + for i := 0; i < numCopies; i++ { + copy(outBuff[len(outBuff):], input) + } + + return string(outBuff) +} + +func TestSnappyEncode(t *testing.T) { + for src, exp := range snappyTestCases { + dst := Encode([]byte(src)) + if !bytes.Equal(dst, exp) { + t.Errorf("Expected %s to generate %v, but was %v", src, exp, dst) + } + } +} + +func TestSnappyEncodeStream(t *testing.T) { + for src, _ := range snappyStreamTestCases { + dst := EncodeStream(nil, []byte(src)) + + // Block size can change the bytes generated, so let's just decode and make sure in matches out + dec, err := Decode(dst) + if err != nil { + t.Error(err) + } + if src != string(dec) { + t.Errorf("Expected decode to match encode orig = %s, decoded = %s", src, string(dec)) + } + } +} + +func TestSnappyLargeStringEncodeStream(t *testing.T) { + massiveString := makeMassive(largeString, 10000) + dst := EncodeStream(nil, []byte(massiveString)) + dec, err := Decode(dst) + if err != nil { + t.Error(err) + } + if massiveString != string(dec) { + t.Errorf("Decoded string didn't match original input (not printing due to size)") + } +} + +func TestSnappyDecode(t *testing.T) { + for exp, src := range snappyTestCases { + dst, err := Decode(src) + if err != nil { + t.Error("Encoding error: ", err) + } else if !bytes.Equal(dst, []byte(exp)) { + t.Errorf("Expected %s to be generated from %v, but was %s", exp, src, string(dst)) + } + } +} + +func TestSnappyDecodeStreams(t *testing.T) { + for exp, src := range snappyStreamTestCases { + dst, err := Decode(src) + if err != nil { + t.Error("Encoding error: ", err) + } else if !bytes.Equal(dst, []byte(exp)) { + t.Errorf("Expected %s to be generated from [%d]byte, but was %s", exp, len(src), string(dst)) + } + } +} + +func TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) { + // Truncated headers should not cause a panic. + for i := 0; i < len(xerialHeader); i++ { + buf := make([]byte, i) + copy(buf, xerialHeader[:i]) + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } + } +} + +func TestSnappyDecodeMalformedTruncatedSize(t *testing.T) { + // Inputs with valid Xerial header but truncated "size" field + sizes := []int{sizeOffset + 1, sizeOffset + 2, sizeOffset + 3} + for _, size := range sizes { + buf := make([]byte, size) + copy(buf, xerialHeader) + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } + } +} + +func TestSnappyDecodeMalformedBNoData(t *testing.T) { + // No data after the size field + buf := make([]byte, 20) + copy(buf, xerialHeader) + // indicate that there's one byte of data to be read + buf[len(buf)-1] = 1 + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } +} + +func TestSnappyMasterDecodeFailed(t *testing.T) { + buf := make([]byte, 21) + copy(buf, xerialHeader) + // indicate that there's one byte of data to be read + buf[len(buf)-2] = 1 + // A payload which will not decode + buf[len(buf)-1] = 1 + if _, err := Decode(buf); err == ErrMalformed || err == nil { + t.Errorf("unexpected err: %v", err) + } +} + +func BenchmarkSnappyDecode(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyTestCases { + dst, err := Decode(test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyDecodeInto(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + dst []byte + err error + ) + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyTestCases { + + dst, err = DecodeInto(dst, test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyStreamDecode(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyStreamTestCases { + dst, err := Decode(test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyStreamDecodeInto(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + dst = make([]byte, 1024, 1024) + err error + ) + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyStreamTestCases { + dst, err = DecodeInto(dst, test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} +func BenchmarkSnappyStreamDecodeMassive(b *testing.B) { + massiveString := makeMassive(largeString, 10000) + enc := EncodeStream(nil, []byte(massiveString)) + + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(int64(len(massiveString))) + + for n := 0; n < b.N; n++ { + _, err := Decode(enc) + if err != nil { + b.Error("Decoding error: ", err) + } + } +} + +func BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) { + massiveString := makeMassive(largeString, 10000) + enc := EncodeStream(nil, []byte(massiveString)) + + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(int64(len(massiveString))) + + var ( + dst = make([]byte, 1024, 1024) + err error + ) + + for n := 0; n < b.N; n++ { + dst, err = DecodeInto(dst, enc) + if err != nil { + b.Error("Decoding error: ", err) + } + } +}