Release 0.5.0

.github/release_checklist.md

@@ -2,9 +2,7 @@ Release checklist
 - [ ] Check outstanding issues on JIRA and Github.
 - [ ] Check [latest documentation](https://python-zlib-ng.readthedocs.io/en/latest/) looks fine.
 - [ ] Create a release branch.
-  - [ ] Set version to a stable number.
   - [ ] Change current development version in `CHANGELOG.rst` to stable version.
-  - [ ] Change the version in `__init__.py`
 - [ ] Merge the release branch into `main`.
 - [ ] Created an annotated tag with the stable version number. Include changes 
 from CHANGELOG.rst.

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up Python 3.8
@@ -36,7 +36,7 @@ jobs:
           - twine_check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up Python 3.8
@@ -58,18 +58,19 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
+          - "3.13-dev"
           - "pypy-3.9"
           - "pypy-3.10"
         os: ["ubuntu-latest"]
         include:
           - os: "macos-14"  # For m1 macos
             python-version: "3.12"
-          - os: "macos-latest"  # for x86 macos
+          - os: "macos-13"  # for x86 macos
             python-version: "3.8"
           - os: "windows-latest"
             python-version: "3.8"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
@@ -94,22 +95,25 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        distro: [ "ubuntu_latest" ]
-        arch: ["aarch64"]
+        python_version:
+          - "3.8"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: uraimo/run-on-arch-action@v2.2.0
+      - uses: uraimo/run-on-arch-action@v2.5.0
         name: Build & run test
         with:
-          arch: ${{ matrix.arch }}
-          distro: ${{ matrix.distro }}
-          install: |
-            apt-get update -q -y
-            apt-get install -q -y python3 python3-pip tox cmake git googletest
-          run: |
-            tox
+          arch: none
+          distro: none
+          base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64"
+          # versioningit needs an accessible git repository but the container
+          # is run as root, which is different from the repository user.
+          # use git config to override this.
+          run: |-
+            git config --global --add safe.directory $PWD
+            CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest
+            python${{matrix.python_version}} -m pytest tests
 
   # Test if the python-zlib-ng conda package can be build. Which is linked
   # dynamically to the conda zlib-ng package.
@@ -122,17 +126,21 @@ jobs:
         shell: bash -l {0}
     strategy:
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - "ubuntu-latest"
+          - "macos-13"
+          - "macos-14"
+          - "windows-latest"
         python_version: [ "python" ]
         include:
           - os: "ubuntu-latest"
             python_version: "pypy"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Install miniconda.
-        uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda.
+        uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda.
         with:
           channels: conda-forge,defaults
       - name: Install requirements (universal)
@@ -158,17 +166,19 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-          - macos-latest
+          - macos-13
+          - macos-14
           - windows-latest
         cibw_archs_linux: ["x86_64"]
         build_sdist: [true]
         include:
           - os: "ubuntu-latest"
             cibw_archs_linux: "aarch64"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
+          fetch-depth: 0  # Fetch everything to get accurately versioned tag.
       - uses: actions/setup-python@v2
         name: Install Python
       - name: Install cibuildwheel twine build
@@ -187,7 +197,8 @@ jobs:
       - name: Build wheels
         run: cibuildwheel --output-dir dist
         env:
-          CIBW_SKIP: "*-win32 *-manylinux_i686"  # Skip 32 bit
+          # Skip 32 bit, macosx_arm64 causes issues on cpython 3.8
+          CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64"
           CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
           CIBW_TEST_REQUIRES: "pytest"
           # Simple tests that requires the project to be build correctly

.gitignore

@@ -1,3 +1,5 @@
+src/zlib_ng/_version.py
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

CHANGELOG.rst

@@ -7,6 +7,14 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.5.0
+-----------------
++ Wheels are now build for MacOS arm64 architectures.
++ Fix a bug where READ and WRITE in zlib_ng.gzip_ng were inconsistent with the
+  values in gzip on Python 3.13
++ Small simplifications to the ``gzip_ng.compress`` and ``gzip_ng.decompress``
+  functions, which should lead to less overhead.
+
 version 0.4.3
 -----------------
 + Fix a bug where files larger than 4GB could not be decompressed.

MANIFEST.in

@@ -1 +1,10 @@
 graft src/zlib_ng/zlib-ng
+prune tests
+prune docs
+prune benchmark_scripts
+prune .github
+exclude tox.ini
+exclude requirements-docs.txt
+exclude codecov.yml
+exclude .readthedocs.yml
+exclude .git*

pyproject.toml

@@ -1,3 +1,10 @@
 [build-system]
-requires = ["setuptools>=51", "wheel"]
+requires = ["setuptools>=64", "versioningit>=1.1.0"]
 build-backend = "setuptools.build_meta"
+
+[tool.versioningit.vcs]
+method="git"
+default-tag = "v0.0.0"
+
+[tool.versioningit.write]
+file = "src/zlib_ng/_version.py"

setup.py

@@ -16,6 +16,8 @@
 from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
 
+import versioningit
+
 ZLIB_NG_SOURCE = os.path.join("src", "zlib_ng", "zlib-ng")
 
 SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or
@@ -124,7 +126,7 @@ def build_zlib_ng():
 
 setup(
     name="zlib-ng",
-    version="0.4.3",
+    version=versioningit.get_version(),
     description="Drop-in replacement for zlib and gzip modules using zlib-ng",
     author="Leiden University Medical Center",
     author_email="[email protected]",  # A placeholder for now

src/zlib_ng/__init__.py

@@ -5,4 +5,6 @@
 # This file is part of python-zlib-ng which is distributed under the
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
-__version__ = "0.4.3"
+from ._version import __version__
+
+__all__ = ["__version__"]

src/zlib_ng/_version.pyi

@@ -0,0 +1,8 @@
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+# Python Software Foundation; All Rights Reserved
+
+# This file is part of python-zlib-ng which is distributed under the
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
+
+__version__: int

src/zlib_ng/gzip_ng.py

@@ -22,6 +22,7 @@
 import gzip
 import io
 import os
+import shutil
 import struct
 import sys
 import time
@@ -41,7 +42,7 @@
 READ_BUFFER_SIZE = 512 * 1024
 
 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-READ, WRITE = 1, 2
+READ, WRITE = gzip.READ, gzip.WRITE
 
 BadGzipFile = gzip.BadGzipFile  # type: ignore
 
@@ -180,50 +181,27 @@ def write(self, data):
 _GzipNGReader = _GzipReader
 
 
-def _create_simple_gzip_header(compresslevel: int,
-                               mtime=None) -> bytes:
-    """
-    Write a simple gzip header with no extra fields.
-    :param compresslevel: Compresslevel used to determine the xfl bytes.
-    :param mtime: The mtime (must support conversion to a 32-bit integer).
-    :return: A bytes object representing the gzip header.
-    """
-    if mtime is None:
-        mtime = time.time()
-    if compresslevel == _COMPRESS_LEVEL_BEST:
-        xfl = 2
-    elif compresslevel == _COMPRESS_LEVEL_FAST:
-        xfl = 4
-    else:
-        xfl = 0
-    # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
-    # fields added to header), mtime, xfl and os (255 for unknown OS).
-    return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
-
-
 def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     """Compress data in one shot and return the compressed string.
     compresslevel sets the compression level in range of 0-9.
     mtime can be used to set the modification time. The modification time is
     set to the current time by default.
     """
-    if mtime == 0:
-        # Use zlib as it creates the header with 0 mtime by default.
-        # This is faster and with less overhead.
-        return zlib_ng.compress(data, level=compresslevel, wbits=31)
-    header = _create_simple_gzip_header(compresslevel, mtime)
-    trailer = struct.pack("<LL", zlib_ng.crc32(data), (len(data) & 0xffffffff))
-    # Wbits=-15 creates a raw deflate block.
-    return (header + zlib_ng.compress(data, level=compresslevel, wbits=-15) +
-            trailer)
+    # Wbits=31 automatically includes a gzip header and trailer.
+    gzip_data = zlib_ng.compress(data, level=compresslevel, wbits=31)
+    if mtime is None:
+        mtime = time.time()
+    # Reuse gzip header created by zlib_ng, replace mtime and OS byte for
+    # consistency.
+    header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255)
+    return header + gzip_data[10:]
 
 
 def decompress(data):
     """Decompress a gzip compressed string in one shot.
     Return the decompressed string.
     """
-    fp = io.BytesIO(data)
-    reader = _GzipReader(fp, max(len(data), 16))
+    reader = _GzipReader(data)
     return reader.readall()
 
 
@@ -325,11 +303,7 @@ def main():
     global READ_BUFFER_SIZE
     READ_BUFFER_SIZE = args.buffer_size
     try:
-        while True:
-            block = in_file.read(args.buffer_size)
-            if block == b"":
-                break
-            out_file.write(block)
+        shutil.copyfileobj(in_file, out_file, args.buffer_size)
     finally:
         if in_file is not sys.stdin.buffer:
             in_file.close()