From 8373d7dff85b18e0ce3cd5b630cc743dec3f82b7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Apr 2024 15:45:29 +0200 Subject: [PATCH 01/18] Update version number --- setup.py | 2 +- src/zlib_ng/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4c16c7f..8da9d85 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.4.3", + version="0.5.0-dev", description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index c1e6f1b..aa9ea12 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -5,4 +5,4 @@ # This file is part of python-zlib-ng which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__ = "0.4.3" +__version__ = "0.5.0-dev" From 15ca0b99dc428d3d6b0fab999caa315019952624 Mon Sep 17 00:00:00 2001 From: "Benjamin A. Beasley" Date: Wed, 15 May 2024 11:49:54 -0400 Subject: [PATCH 02/18] Use values of READ and WRITE from the gzip module These changed from integers to mode strings in Python 3.13, and we need to remain consistent with the standard library. Fixes #44. Reference: https://docs.python.org/3.13/library/gzip.html#gzip.GzipFile.mode --- src/zlib_ng/gzip_ng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index 33db4be..24b5512 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -41,7 +41,7 @@ READ_BUFFER_SIZE = 512 * 1024 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ, WRITE = gzip.READ, gzip.WRITE BadGzipFile = gzip.BadGzipFile # type: ignore From 04ff4988070e4cdf06f0e54d393395f54107b2b0 Mon Sep 17 00:00:00 2001 From: "Benjamin A. Beasley" Date: Wed, 15 May 2024 11:56:32 -0400 Subject: [PATCH 03/18] Add changelog entry for READ/WRITE fix --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 742b50f..5d2c128 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 0.5.0-dev +----------------- ++ Fix a bug where READ and WRITE in zlib_ng.gzip_ng were inconsistent with the + values in gzip on Python 3.13 + version 0.4.3 ----------------- + Fix a bug where files larger than 4GB could not be decompressed. From 36b722fb01d1dae4911c100d43f70f8a65f094c8 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Jul 2024 10:35:13 +0200 Subject: [PATCH 04/18] Explicitly test on x86-64 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51fa4ee..ac1bf14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: include: - os: "macos-14" # For m1 macos python-version: "3.12" - - os: "macos-latest" # for x86 macos + - os: "macos-13" # for x86 macos python-version: "3.8" - os: "windows-latest" python-version: "3.8" From c5c40521c30102113afd50683cd0c4d5cd5d39bb Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Jul 2024 10:40:44 +0200 Subject: [PATCH 05/18] Update conda incubator --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac1bf14..bca05eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,11 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: + - "ubuntu-latest" + - "macos-13" + - "macos-14" + - "windows-latest" python_version: [ "python" ] include: - os: "ubuntu-latest" @@ -132,7 +136,7 @@ jobs: with: submodules: recursive - name: Install miniconda. - uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda. + uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) From 5e5f1beb99dcb80b3dec38fcf50b49fb619fde4b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Jul 2024 12:57:44 +0200 Subject: [PATCH 06/18] Only build on MacOS x86 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bca05eb..52dd2c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,7 @@ jobs: matrix: os: - ubuntu-latest - - macos-latest + - macos-13 - windows-latest cibw_archs_linux: ["x86_64"] build_sdist: [true] From 708b60d6e46cdc04c95e615c099dae7c1f922139 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:38:47 +0200 Subject: [PATCH 07/18] Use automated versioning with versioningit --- .github/release_checklist.md | 2 -- .gitignore | 2 ++ pyproject.toml | 9 ++++++++- setup.py | 4 +++- src/zlib_ng/__init__.py | 2 +- src/zlib_ng/_version.pyi | 8 ++++++++ 6 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 src/zlib_ng/_version.pyi diff --git a/.github/release_checklist.md b/.github/release_checklist.md index 9abffc2..71b59ae 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -2,9 +2,7 @@ Release checklist - [ ] Check outstanding issues on JIRA and Github. - [ ] Check [latest documentation](https://python-zlib-ng.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - - [ ] Set version to a stable number. - [ ] Change current development version in `CHANGELOG.rst` to stable version. - - [ ] Change the version in `__init__.py` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. diff --git a/.gitignore b/.gitignore index b6e4761..671f3ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +src/zlib_ng/_version.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 62f5f82..28ae41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ [build-system] -requires = ["setuptools>=51", "wheel"] +requires = ["setuptools>=64", "versioningit>=1.1.0"] build-backend = "setuptools.build_meta" + +[tool.versioningit.vcs] +method="git" +default-tag = "v0.0.0" + +[tool.versioningit.write] +file = "src/zlib_ng/_version.py" diff --git a/setup.py b/setup.py index 8da9d85..95a6a44 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext +import versioningit + ZLIB_NG_SOURCE = os.path.join("src", "zlib_ng", "zlib-ng") SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or @@ -124,7 +126,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.5.0-dev", + version=versioningit.get_version(), description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index aa9ea12..6f20222 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -5,4 +5,4 @@ # This file is part of python-zlib-ng which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__ = "0.5.0-dev" +from ._version import __version__ diff --git a/src/zlib_ng/_version.pyi b/src/zlib_ng/_version.pyi new file mode 100644 index 0000000..6c8f915 --- /dev/null +++ b/src/zlib_ng/_version.pyi @@ -0,0 +1,8 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 +# Python Software Foundation; All Rights Reserved + +# This file is part of python-zlib-ng which is distributed under the +# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. + +__version__: int \ No newline at end of file From 7865bc7a9cba7acd685bcdd2009d63081ace029b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:40:07 +0200 Subject: [PATCH 08/18] Update github action checkout and fetch everything for deploy builds --- .github/workflows/ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52dd2c0..7382e1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 @@ -36,7 +36,7 @@ jobs: - twine_check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 @@ -69,7 +69,7 @@ jobs: - os: "windows-latest" python-version: "3.8" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -97,7 +97,7 @@ jobs: distro: [ "ubuntu_latest" ] arch: ["aarch64"] steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - uses: uraimo/run-on-arch-action@v2.2.0 @@ -132,7 +132,7 @@ jobs: - os: "ubuntu-latest" python_version: "pypy" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive - name: Install miniconda. @@ -170,9 +170,10 @@ jobs: - os: "ubuntu-latest" cibw_archs_linux: "aarch64" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 # Fetch everything to get accurately versioned tag. - uses: actions/setup-python@v2 name: Install Python - name: Install cibuildwheel twine build From 81029afe823f2d5bc2761773e75eb4ea068d07f4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 15:51:26 +0200 Subject: [PATCH 09/18] Update MANIFEST.in --- MANIFEST.in | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 49eb90b..6cde131 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,10 @@ graft src/zlib_ng/zlib-ng +prune tests +prune docs +prune benchmark_scripts +prune .github +exclude tox.ini +exclude requirements-docs.txt +exclude codecov.yml +exclude .readthedocs.yml +exclude .git* From 8f0cda431d602587bac32013fda304e149797345 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 16:00:15 +0200 Subject: [PATCH 10/18] List version as importable and used --- src/zlib_ng/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index 6f20222..8504ed8 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -6,3 +6,5 @@ # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. from ._version import __version__ + +__all__ = ["__version__"] From 56d1d3e51044c1f94660c1783a1076e4797412cf Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 16 Jul 2024 16:22:03 +0200 Subject: [PATCH 11/18] Eanble building wheels on MacOS arm64 architectures --- .github/workflows/ci.yml | 4 +++- CHANGELOG.rst | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7382e1d..cc76729 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,7 @@ jobs: os: - ubuntu-latest - macos-13 + - macos-14 - windows-latest cibw_archs_linux: ["x86_64"] build_sdist: [true] @@ -192,7 +193,8 @@ jobs: - name: Build wheels run: cibuildwheel --output-dir dist env: - CIBW_SKIP: "*-win32 *-manylinux_i686" # Skip 32 bit + # Skip 32 bit, macosx_arm64 causes issues on cpython 3.8 + CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_TEST_REQUIRES: "pytest" # Simple tests that requires the project to be build correctly diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5d2c128..e68a8e4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changelog version 0.5.0-dev ----------------- ++ Wheels are now build for MacOS arm64 architectures. + Fix a bug where READ and WRITE in zlib_ng.gzip_ng were inconsistent with the values in gzip on Python 3.13 From 017f7487ee497da27f881d53f5d915008c8c3b97 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 17 Jul 2024 09:06:55 +0200 Subject: [PATCH 12/18] Fix aarch64 test --- .github/workflows/ci.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc76729..5ad6c8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,22 +94,25 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - distro: [ "ubuntu_latest" ] - arch: ["aarch64"] + python_version: + - "3.8" steps: - uses: actions/checkout@v4 with: submodules: recursive - - uses: uraimo/run-on-arch-action@v2.2.0 + - uses: uraimo/run-on-arch-action@v2.5.0 name: Build & run test with: - arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} - install: | - apt-get update -q -y - apt-get install -q -y python3 python3-pip tox cmake git googletest - run: | - tox + arch: none + distro: none + base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64" + # versioningit needs an accessible git repository but the container + # is run as root, which is different from the repository user. + # use git config to override this. + run: |- + git config --global --add safe.directory $PWD + CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest + python${{matrix.python_version}} -m pytest tests # Test if the python-zlib-ng conda package can be build. Which is linked # dynamically to the conda zlib-ng package. From d7bd7140e4861ba1f2b0200b7b0842fc7c8bc0f3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 06:58:36 +0200 Subject: [PATCH 13/18] Replace compress function with simpler CPython 3.13 variant --- src/zlib_ng/gzip_ng.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index 24b5512..769698f 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -180,42 +180,20 @@ def write(self, data): _GzipNGReader = _GzipReader -def _create_simple_gzip_header(compresslevel: int, - mtime=None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" Date: Fri, 19 Jul 2024 07:07:49 +0200 Subject: [PATCH 14/18] Simplify decompress function --- src/zlib_ng/gzip_ng.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index 769698f..d98a49f 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -200,8 +200,7 @@ def decompress(data): """Decompress a gzip compressed string in one shot. Return the decompressed string. """ - fp = io.BytesIO(data) - reader = _GzipReader(fp, max(len(data), 16)) + reader = _GzipReader(data) return reader.readall() From 4edc533f7a675b2870b787f8b2ed8e206eaac2f1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 07:08:58 +0200 Subject: [PATCH 15/18] Use shutil.copyfileobj for fileobj copying --- src/zlib_ng/gzip_ng.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index d98a49f..f24bdbe 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -22,6 +22,7 @@ import gzip import io import os +import shutil import struct import sys import time @@ -302,11 +303,7 @@ def main(): global READ_BUFFER_SIZE READ_BUFFER_SIZE = args.buffer_size try: - while True: - block = in_file.read(args.buffer_size) - if block == b"": - break - out_file.write(block) + shutil.copyfileobj(in_file, out_file, args.buffer_size) finally: if in_file is not sys.stdin.buffer: in_file.close() From e918f3e611dd80a57a01833e34d4893b415a9116 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 19 Jul 2024 09:21:11 +0200 Subject: [PATCH 16/18] Update changelog with simplification changes --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e68a8e4..76712e3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ version 0.5.0-dev + Wheels are now build for MacOS arm64 architectures. + Fix a bug where READ and WRITE in zlib_ng.gzip_ng were inconsistent with the values in gzip on Python 3.13 ++ Small simplifications to the ``gzip_ng.compress`` and ``gzip_ng.decompress`` + functions, which should lead to less overhead. version 0.4.3 ----------------- From 48a8b7bae5e35e8322d88c89b409ccfd403452d9 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 14:17:07 +0200 Subject: [PATCH 17/18] Prepare release 0.5.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 76712e3..e38a722 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 0.5.0-dev +version 0.5.0 ----------------- + Wheels are now build for MacOS arm64 architectures. + Fix a bug where READ and WRITE in zlib_ng.gzip_ng were inconsistent with the From c63960d87d09a40a2383eb291afa1ec9b0eea040 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 9 Aug 2024 14:18:17 +0200 Subject: [PATCH 18/18] Also test on 3.13-dev --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad6c8d..e1eacc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: - "3.10" - "3.11" - "3.12" + - "3.13-dev" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"]