Skip to content

Commit f110d56

Browse files
committed
gh-103477: Write gzip trailer with zlib
RHEL, SLES and Ubuntu for IBM zSystems (aka s390x) ship with a zlib optimization [1] that significantly improves deflate performance by using a specialized CPU instruction. This instruction not only compresses the data, but also computes a checksum. At the moment Pyhton's gzip support performs compression and checksum calculation separately, which creates unnecessary overhead. The reason is that Python needs to write specific values into gzip header, so it uses a raw stream instead of a gzip stream, and zlib does not compute a checksum for raw streams. The challenge with using gzip streams instead of zlib streams is dealing with zlib-generated gzip header, which we need to rather generate manually. Implement the method proposed by @rhpvorderman: use Z_BLOCK on the first deflate() call in order to stop before the first deflate block is emitted. The data that is emitted up until this point is zlib-generated gzip header, which should be discarded. Expose this new functionality by adding a boolean gzip_trailer argument to zlib.compress() and zlib.compressobj(). Make use of it in gzip.compress(), GzipFile and TarFile. The performance improvement varies depending on data being compressed, but it's in the ballpark of 40%. An alternative approach is to use the deflateSetHeader() function, introduced in zlib v1.2.2.1 (2011). This also works, but the change was deemed too intrusive [2]. 📜🤖 Added by blurb_it. [1] madler/zlib#410 [2] #103478
1 parent fb4cddb commit f110d56

File tree

9 files changed

+141
-49
lines changed

9 files changed

+141
-49
lines changed

Include/internal/pycore_global_objects_fini_generated.h

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Include/internal/pycore_global_strings.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ struct _Py_global_strings {
451451
STRUCT_FOR_ID(globals)
452452
STRUCT_FOR_ID(groupindex)
453453
STRUCT_FOR_ID(groups)
454+
STRUCT_FOR_ID(gzip_trailer)
454455
STRUCT_FOR_ID(h)
455456
STRUCT_FOR_ID(handle)
456457
STRUCT_FOR_ID(hash_name)

Include/internal/pycore_runtime_init_generated.h

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Include/internal/pycore_unicodeobject_generated.h

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Lib/gzip.py

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,8 @@ def __init__(self, filename=None, mode=None,
221221
zlib.DEFLATED,
222222
-zlib.MAX_WBITS,
223223
zlib.DEF_MEM_LEVEL,
224-
0)
224+
0,
225+
gzip_trailer=True)
225226
self._write_mtime = mtime
226227
self._buffer_size = _WRITE_BUFFER_SIZE
227228
self._buffer = io.BufferedWriter(_WriteBufferStream(self),
@@ -245,8 +246,6 @@ def __repr__(self):
245246

246247
def _init_write(self, filename):
247248
self.name = filename
248-
self.crc = zlib.crc32(b"")
249-
self.size = 0
250249
self.writebuf = []
251250
self.bufsize = 0
252251
self.offset = 0 # Current file offset for seek(), tell(), etc
@@ -310,8 +309,6 @@ def _write_raw(self, data):
310309

311310
if length > 0:
312311
self.fileobj.write(self.compress.compress(data))
313-
self.size += length
314-
self.crc = zlib.crc32(data, self.crc)
315312
self.offset += length
316313

317314
return length
@@ -355,9 +352,6 @@ def close(self):
355352
if self.mode == WRITE:
356353
self._buffer.flush()
357354
fileobj.write(self.compress.flush())
358-
write32u(fileobj, self.crc)
359-
# self.size may exceed 2 GiB, or even 4 GiB
360-
write32u(fileobj, self.size & 0xffffffff)
361355
elif self.mode == READ:
362356
self._buffer.close()
363357
finally:
@@ -611,10 +605,11 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
611605
# This is faster and with less overhead.
612606
return zlib.compress(data, level=compresslevel, wbits=31)
613607
header = _create_simple_gzip_header(compresslevel, mtime)
614-
trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
615-
# Wbits=-15 creates a raw deflate block.
616-
return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
617-
trailer)
608+
# Wbits=-15 creates a raw deflate block. Gzip_trailer=True computes CRC32
609+
# and writes gzip trailer with zlib, which on some platforms is faster
610+
# than doing it manually.
611+
return (header + zlib.compress(data, level=compresslevel, wbits=-15,
612+
gzip_trailer=True))
618613

619614

620615
def decompress(data):

Lib/tarfile.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -369,7 +369,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize,
369369
except ImportError:
370370
raise CompressionError("zlib module is not available") from None
371371
self.zlib = zlib
372-
self.crc = zlib.crc32(b"")
373372
if mode == "r":
374373
self.exception = zlib.error
375374
self._init_read_gz()
@@ -420,7 +419,8 @@ def _init_write_gz(self, compresslevel):
420419
self.zlib.DEFLATED,
421420
-self.zlib.MAX_WBITS,
422421
self.zlib.DEF_MEM_LEVEL,
423-
0)
422+
0,
423+
gzip_trailer=True)
424424
timestamp = struct.pack("<L", int(time.time()))
425425
self.__write(b"\037\213\010\010" + timestamp + b"\002\377")
426426
if self.name.endswith(".gz"):
@@ -433,8 +433,6 @@ def _init_write_gz(self, compresslevel):
433433
def write(self, s):
434434
"""Write string s to the stream.
435435
"""
436-
if self.comptype == "gz":
437-
self.crc = self.zlib.crc32(s, self.crc)
438436
self.pos += len(s)
439437
if self.comptype != "tar":
440438
s = self.cmp.compress(s)
@@ -464,9 +462,6 @@ def close(self):
464462
if self.mode == "w" and self.buf:
465463
self.fileobj.write(self.buf)
466464
self.buf = b""
467-
if self.comptype == "gz":
468-
self.fileobj.write(struct.pack("<L", self.crc))
469-
self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF))
470465
finally:
471466
if not self._extfileobj:
472467
self.fileobj.close()
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Write gzip trailer with zlib, improving gzip compression performance on s390x by roughly 40%.

Modules/clinic/zlibmodule.c.h

Lines changed: 49 additions & 22 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)