Skip to content

Commit 05df110

Browse files
jonathanunderwoodjonathanunderwood
authored
Merge pull request python#30 from python-lz4/development
Development branch fixes for 0.9.1 release
2 parents b77b080 + 1b60148 commit 05df110

File tree

11 files changed

+78
-63
lines changed

11 files changed

+78
-63
lines changed

‎.travis.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ python:
66
- 3.3
77
- 3.4
88
- 3.5
9+
- 3.6
910
install:
1011
script: python setup.py test

‎appveyor.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ environment:
1717
- PYTHON: "C:\\Python34-x64"
1818
DISTUTILS_USE_SDK: "1"
1919
- PYTHON: "C:\\Python35-x64"
20+
- PYTHON: "C:\\Python36-x64"
2021

2122
matrix:
2223
# Immediately finish build once one of the jobs fails.

‎lz4/block/_block.c

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,6 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs)
121121
return NULL;
122122
}
123123

124-
if (source_size <= 0) {
125-
PyErr_Format(PyExc_ValueError, "Input source data size invalid: %d bytes", source_size);
126-
return NULL;
127-
}
128-
129124
if (!strncmp (mode, "default", sizeof ("default")))
130125
{
131126
comp = DEFAULT;
@@ -258,7 +253,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs)
258253
source_size -= hdr_size;
259254
}
260255

261-
if (dest_size <= 0 || dest_size > PY_SSIZE_T_MAX)
256+
if (dest_size < 0 || dest_size > PY_SSIZE_T_MAX)
262257
{
263258
PyErr_Format (PyExc_ValueError, "Invalid size in header: 0x%zu",
264259
dest_size);

‎lz4/frame/__init__.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@ class LZ4FrameCompressor(object):
3535
- lz4.frame.CONTENTCHECKSUM_DISABLED or 0: disables checksumming
3636
- lz4.frame.CONTENTCHECKSUM_ENABLED or 1: enables checksumming
3737
The default is CONTENTCHECKSUM_DISABLED.
38-
content_size (bool): Specifies whether to include an optional 8-byte header
39-
field that is the uncompressed size of data included within the frame.
40-
Including the content-size header is optional, and is enabled by default.
4138
frame_type (int): Specifies whether user data can be injected between
4239
frames. Options:
4340
- lz4.frame.FRAMETYPE_FRAME or 0: disables user data injection
@@ -53,14 +50,12 @@ def __init__(self,
5350
block_mode=BLOCKMODE_LINKED,
5451
compression_level=COMPRESSIONLEVEL_MIN,
5552
content_checksum=CONTENTCHECKSUM_DISABLED,
56-
content_size=True,
5753
frame_type=FRAMETYPE_FRAME,
5854
auto_flush=True):
5955
self.block_size = block_size
6056
self.block_mode = block_mode
6157
self.compression_level = compression_level
6258
self.content_checksum = content_checksum
63-
self.content_size = content_size
6459
self.frame_type = frame_type
6560
self.auto_flush = auto_flush
6661
self._context = create_compression_context()
@@ -82,9 +77,10 @@ def compress_begin(self, source_size=0):
8277
8378
Args:
8479
data (bytes): data to compress
85-
source_size (int): Optionally specified the total size of the
80+
source_size (int): Optionally specify the total size of the
8681
uncompressed data. If specified, will be stored in the
87-
compressed frame header for later use in decompression.
82+
compressed frame header as an 8-byte field for later use
83+
during decompression.
8884
8985
Returns:
9086
bytes: frame header data
@@ -97,7 +93,6 @@ def compress_begin(self, source_size=0):
9793
frame_type=self.frame_type,
9894
compression_level=self.compression_level,
9995
content_checksum=self.content_checksum,
100-
content_size=self.content_size,
10196
auto_flush=self.auto_flush,
10297
source_size=source_size)
10398

‎lz4/frame/_frame.c

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,9 @@ create_compression_context (PyObject * Py_UNUSED (self))
121121
return PyCapsule_New (context, capsule_name, destruct_compression_context);
122122
}
123123

124-
/******************
125-
* compress_frame *
126-
******************/
124+
/************
125+
* compress *
126+
************/
127127
#define __COMPRESS_KWARGS_DOCSTRING \
128128
" block_size (int): Sepcifies the maximum blocksize to use.\n" \
129129
" Options:\n\n" \
@@ -152,25 +152,25 @@ create_compression_context (PyObject * Py_UNUSED (self))
152152
" - lz4.frame.CONTENTCHECKSUM_DISABLED or 0: disables checksumming\n" \
153153
" - lz4.frame.CONTENTCHECKSUM_ENABLED or 1: enables checksumming\n\n" \
154154
" The default is CONTENTCHECKSUM_DISABLED.\n" \
155-
" content_size (bool): Specifies whether to include an optional 8-byte header\n" \
156-
" field that is the uncompressed size of data included within the frame.\n" \
157-
" Including the content-size header is optional, and is enabled by default.\n" \
158155
" frame_type (int): Specifies whether user data can be injected between\n" \
159156
" frames. Options:\n\n" \
160157
" - lz4.frame.FRAMETYPE_FRAME or 0: disables user data injection\n" \
161158
" - lz4.frame.FRAMETYPE_SKIPPABLEFRAME or 1: enables user data injection\n\n" \
162159
" The default is lz4.frame.FRAMETYPE_FRAME.\n" \
163160

164161
PyDoc_STRVAR(compress__doc,
165-
"compress(source, compression_level=0, block_size=0, content_checksum=0, content_size=1, block_mode=0, frame_type=0)\n\n" \
162+
"compress(source, compression_level=0, block_size=0, content_checksum=0, block_mode=0, frame_type=0, content_size_header=1)\n\n" \
166163
"Accepts a string, and compresses the string in one go, returning the\n" \
167164
"compressed string as a string of bytes. The compressed string includes\n" \
168165
"a header and endmark and so is suitable for writing to a file.\n\n" \
169166
"Args:\n" \
170167
" source (str): String to compress\n\n" \
171168
"Keyword Args:\n" \
172169
__COMPRESS_KWARGS_DOCSTRING \
173-
"\n" \
170+
" content_size_header (bool): Specifies whether to include an optional\n" \
171+
" 8-byte header field that is the uncompressed size of data included\n" \
172+
" within the frame. Including the content-size header is optional\n" \
173+
" and is enabled by default.\n\n" \
174174
"Returns:\n" \
175175
" str: Compressed data as a string\n"
176176
);
@@ -192,9 +192,9 @@ compress (PyObject * Py_UNUSED (self), PyObject * args,
192192
"compression_level",
193193
"block_size",
194194
"content_checksum",
195-
"content_size",
196195
"block_mode",
197196
"frame_type",
197+
"content_size_header",
198198
NULL
199199
};
200200

@@ -205,11 +205,10 @@ compress (PyObject * Py_UNUSED (self), PyObject * args,
205205
&source, &source_size,
206206
&preferences.compressionLevel,
207207
&preferences.frameInfo.blockSizeID,
208-
&preferences.
209-
frameInfo.contentChecksumFlag,
210-
&content_size_header,
208+
&preferences.frameInfo.contentChecksumFlag,
211209
&preferences.frameInfo.blockMode,
212-
&preferences.frameInfo.frameType))
210+
&preferences.frameInfo.frameType,
211+
&content_size_header))
213212
{
214213
return NULL;
215214
}
@@ -295,7 +294,7 @@ PyDoc_STRVAR(compress_begin__doc,
295294
" When autoFlush is disabled, the LZ4 library may buffer data\n" \
296295
" until a block is full\n\n" \
297296
" source_size (int): This optionally specifies the uncompressed size\n" \
298-
" of the source content. This arument is optional, but can if specified\n" \
297+
" of the source content. This arument is optional, but if specified\n" \
299298
" will be stored in the frame header for use during decompression.\n"
300299
"Returns:\n" \
301300
" str (str): Frame header.\n"
@@ -309,7 +308,6 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args,
309308
{
310309
PyObject *py_context = NULL;
311310
unsigned long source_size = 0;
312-
int content_size_header = 1;
313311
LZ4F_preferences_t preferences;
314312
/* Only needs to be large enough for a header, which is 15 bytes.
315313
* Unfortunately, the lz4 library doesn't provide a #define for this.
@@ -322,7 +320,6 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args,
322320
"compression_level",
323321
"block_size",
324322
"content_checksum",
325-
"content_size",
326323
"block_mode",
327324
"frame_type",
328325
"auto_flush",
@@ -335,13 +332,12 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args,
335332
argument */
336333
preferences.autoFlush = 1;
337334

338-
if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|kiiiiiii", kwlist,
335+
if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|kiiiiii", kwlist,
339336
&py_context,
340337
&source_size,
341338
&preferences.compressionLevel,
342339
&preferences.frameInfo.blockSizeID,
343340
&preferences.frameInfo.contentChecksumFlag,
344-
&content_size_header,
345341
&preferences.frameInfo.blockMode,
346342
&preferences.frameInfo.frameType,
347343
&preferences.autoFlush
@@ -350,14 +346,7 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args,
350346
return NULL;
351347
}
352348

353-
if (content_size_header)
354-
{
355-
preferences.frameInfo.contentSize = source_size;
356-
}
357-
else
358-
{
359-
preferences.frameInfo.contentSize = 0;
360-
}
349+
preferences.frameInfo.contentSize = source_size;
361350

362351
context =
363352
(struct compression_context *) PyCapsule_GetPointer (py_context, capsule_name);
@@ -798,16 +787,17 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * keywds)
798787
to estimate the new size of the destination buffer. */
799788
char * destination_buffer_new;
800789
destination_size += 3 * result;
790+
Py_BLOCK_THREADS
801791
destination_buffer_new = PyMem_Realloc(destination_buffer, destination_size);
802792
if (!destination_buffer_new)
803793
{
804794
LZ4F_freeDecompressionContext (context);
805-
Py_BLOCK_THREADS
806795
PyErr_SetString (PyExc_RuntimeError,
807796
"Failed to increase destination buffer size");
808797
PyMem_Free (destination_buffer);
809798
return NULL;
810799
}
800+
Py_UNBLOCK_THREADS
811801
destination_buffer = destination_buffer_new;
812802
}
813803
/* Data still remaining to be decompressed, so increment the source and

‎py3c/py3c.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,6 @@ SOFTWARE.
2929

3030
#include <py3c/comparison.h>
3131
#include <py3c/compat.h>
32-
32+
#include <py3c/py3shims.h>
3333

3434
#endif

‎py3c/py3c/compat.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
#define PyStr_InternFromString PyString_InternFromString
7474
#define PyStr_Decode PyString_Decode
7575

76-
static inline PyObject *PyStr_Concat(PyObject *left, PyObject *right) {
76+
static PyObject *PyStr_Concat(PyObject *left, PyObject *right) {
7777
PyObject *str = left;
7878
Py_INCREF(left); // reference to old left will be stolen
7979
PyString_Concat(&str, right);

‎py3c/py3c/py3shims.h

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/* Copyright (c) 2016, Red Hat, Inc. and/or its affiliates
2+
* Licensed under the MIT license; see py3c.h
3+
*/
4+
5+
/*
6+
* Shims for the PyMem_Raw* functions added inPython 3.3
7+
*
8+
* See https://docs.python.org/3/c-api/memory.html#raw-memory-interface
9+
*/
10+
11+
#ifndef _PY3C_RAWMALLOC_H_
12+
#define _PY3C_RAWMALLOC_H_
13+
#include <Python.h>
14+
#include <stdlib.h>
15+
16+
17+
#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 4)
18+
#define PyMem_RawMalloc(n) malloc((n) || 1)
19+
#define PyMem_RawRealloc(p, n) realloc(p, (n) || 1)
20+
#define PyMem_RawFree(p) free(p)
21+
#endif /* version < 3.4 */
22+
23+
24+
#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 5)
25+
#define PyMem_RawCalloc(n, s) calloc((n) || 1, (s) || 1)
26+
#endif /* version < 3.5 */
27+
28+
29+
#endif /* _PY3C_RAWMALLOC_H_ */

‎tests/bench.py

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,24 @@
11
import uuid
22
import timeit
33
import lz4
4-
import snappy
54
import os
65
from timeit import Timer
6+
import sys
7+
import blosc
78

8-
DATA = open("../src/lz4.c", "rb").read()
9-
LZ4_DATA = lz4.compress(DATA)
10-
SNAPPY_DATA = snappy.compress(DATA)
11-
LOOPS = 200000
9+
DATA = open(sys.argv[1], "rb").read()
10+
LZ4_DATA = lz4.block.compress(DATA)
11+
BLOSC_DATA = blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)
12+
LOOPS = 100
1213

1314
print("Data Size:")
1415
print(" Input: %d" % len(DATA))
1516
print(" LZ4: %d (%.2f)" % (len(LZ4_DATA), len(LZ4_DATA) / float(len(DATA))))
16-
print(" Snappy: %d (%.2f)" % (len(SNAPPY_DATA), len(SNAPPY_DATA) / float(len(DATA))))
17-
print(" LZ4 / Snappy: %f" % (float(len(LZ4_DATA)) / float(len(SNAPPY_DATA))))
17+
print(" Blosc: %d (%.2f)" % (len(BLOSC_DATA), len(BLOSC_DATA) / float(len(DATA))))
18+
print(" LZ4 / Blosc: %f" % (float(len(LZ4_DATA)) / float(len(BLOSC_DATA))))
1819

1920
print("Benchmark: %d calls" % LOOPS)
20-
print(" LZ4 Compression: %fs" % Timer("lz4.compress(DATA)", "from __main__ import DATA; import lz4").timeit(number=LOOPS))
21-
print(" Snappy Compression: %fs" % Timer("snappy.compress(DATA)", "from __main__ import DATA; import snappy").timeit(number=LOOPS))
22-
print(" LZ4 Decompression: %fs" % Timer("lz4.uncompress(LZ4_DATA)", "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS))
23-
print(" Snappy Decompression : %fs" % Timer("snappy.uncompress(SNAPPY_DATA)", "from __main__ import SNAPPY_DATA; import snappy").timeit(number=LOOPS))
21+
print(" LZ4 Compression: %fs" % (Timer("lz4.block.compress(DATA)", "from __main__ import DATA; import lz4").timeit(number=LOOPS)/LOOPS))
22+
print(" Blosc Compression: %fs" % (Timer("blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)", "from __main__ import DATA; import blosc").timeit(number=LOOPS)/LOOPS))
23+
print(" LZ4 Decompression: %fs" % (Timer("lz4.block.decompress(LZ4_DATA)", "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)/LOOPS))
24+
print(" Blosc Decompression : %fs" % (Timer("blosc.decompress(BLOSC_DATA)", "from __main__ import BLOSC_DATA; import blosc").timeit(number=LOOPS)/LOOPS))

‎tests/test_block.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88

99
class TestLZ4Block(unittest.TestCase):
1010

11+
def test_empty_string(self):
12+
DATA = b''
13+
self.assertEqual(DATA, lz4.block.decompress(lz4.block.compress(DATA)))
14+
1115
def test_random(self):
1216
DATA = os.urandom(128 * 1024) # Read 128kb
1317
self.assertEqual(DATA, lz4.block.decompress(lz4.block.compress(DATA)))

‎tests/test_frame.py

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,6 @@ def test_compress_begin_update_end_not_defaults(self):
138138
block_size=lz4frame.BLOCKSIZE_MAX256KB,
139139
block_mode=lz4frame.BLOCKMODE_LINKED,
140140
compression_level=lz4frame.COMPRESSIONLEVEL_MINHC,
141-
content_size=False,
142141
auto_flush=1
143142
)
144143
chunk_size = 128 * 1024 # 128 kb, half of block size
@@ -265,6 +264,14 @@ def test_LZ4FrameCompressor_reset(self):
265264
decompressed = lz4frame.decompress(compressed)
266265
self.assertEqual(input_data, decompressed)
267266

267+
def test_compress_without_content_size(self):
268+
input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
269+
compressed = lz4frame.compress(input_data, content_size_header=False)
270+
frame = lz4frame.get_frame_info(compressed)
271+
self.assertEqual(frame['contentSize'], 0)
272+
decompressed = lz4frame.decompress(compressed)
273+
self.assertEqual(input_data, decompressed)
274+
268275
class TestLZ4FrameModern(unittest.TestCase):
269276
def test_decompress_truncated(self):
270277
input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
@@ -306,14 +313,6 @@ def test_LZ4FrameCompressor_fails(self):
306313
compressed += compressor.flush()
307314
compressed = compressor.compress(input_data)
308315

309-
def test_compress_without_content_size(self):
310-
input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
311-
compressed = lz4frame.compress(input_data, content_size=False)
312-
frame = lz4frame.get_frame_info(compressed)
313-
self.assertEqual(frame['contentSize'], 0)
314-
decompressed = lz4frame.decompress(compressed)
315-
self.assertEqual(input_data, decompressed)
316-
317316

318317
if sys.version_info < (2, 7):
319318
# Poor-man unittest.TestCase.skip for Python 2.6

0 commit comments

Comments
 (0)