Merge remote-tracking branch 'upstream/v3' into user/tom/fix/v2-compat

Author: TomAugspurger

.github/workflows/gpu_test.yml

@@ -0,0 +1,66 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: GPU Test V3
+
+on:
+  push:
+    branches: [ v3 ]
+  pull_request:
+    branches: [ v3 ]
+  workflow_dispatch:
+
+env:
+  LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
+
+    runs-on: gpu-runner
+    strategy:
+      matrix:
+        python-version: ['3.11']
+        numpy-version: ['2.0']
+        dependency-set: ["minimal"]
+
+    steps:
+    - uses: actions/checkout@v4
+    # - name: cuda-toolkit
+    #   uses: Jimver/[email protected]
+    #   id: cuda-toolkit
+    #   with:
+    #     cuda: '12.4.1'
+    - name: Set up CUDA
+      run: |
+        wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb
+        sudo dpkg -i cuda-keyring_1.1-1_all.deb
+        sudo apt-get update
+        sudo apt-get -y install cuda-toolkit-12-6
+        echo "/usr/local/cuda/bin" >> $GITHUB_PATH
+    - name: GPU check
+      run: |
+        nvidia-smi
+        echo $PATH
+        echo $LD_LIBRARY_PATH
+        nvcc -V
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install Hatch and CuPy
+      run: |
+        python -m pip install --upgrade pip 
+        pip install hatch
+    - name: Set Up Hatch Env
+      run: |
+        hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
+        hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
+    - name: Run Tests
+      run: |
+        hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.9.0
+      - uses: pypa/gh-action-pypi-publish@v1.10.0
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.5.7'
+    rev: v0.6.3
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

pyproject.toml

@@ -75,6 +75,9 @@ jupyter = [
     'ipytree>=0.2.2',
     'ipywidgets>=8.0.0',
 ]
+gpu = [
+    "cupy-cuda12x",
+]
 docs = [
     'sphinx',
     'sphinx-autobuild>=2021.3.14',
@@ -121,7 +124,7 @@ build.hooks.vcs.version-file = "src/zarr/_version.py"
 [tool.hatch.envs.test]
 dependencies = [
     "numpy~={matrix:numpy}",
-    "universal_pathlib"
+    "universal_pathlib",
 ]
 features = ["test", "extra"]
 
@@ -135,8 +138,34 @@ python = ["3.10", "3.11", "3.12"]
 numpy = ["1.24", "1.26", "2.0"]
 features = ["optional"]
 
+[[tool.hatch.envs.test.matrix]]
+python = ["3.10", "3.11", "3.12"]
+numpy = ["1.24", "1.26", "2.0"]
+features = ["gpu"]
+
 [tool.hatch.envs.test.scripts]
 run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests"
+run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
+run = "run-coverage --no-cov"
+run-verbose = "run-coverage --verbose"
+run-mypy = "mypy src"
+run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
+list-env = "pip list"
+
+[tool.hatch.envs.gputest]
+dependencies = [
+    "numpy~={matrix:numpy}",
+    "universal_pathlib",
+]
+features = ["test", "extra", "gpu"]
+
+[[tool.hatch.envs.gputest.matrix]]
+python = ["3.10", "3.11", "3.12"]
+numpy = ["1.24", "1.26", "2.0"]
+version = ["minimal"]
+
+[tool.hatch.envs.gputest.scripts]
+run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
@@ -169,6 +198,7 @@ extend-exclude = [
     "buck-out",
     "build",
     "dist",
+    "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6
     "venv",
     "docs",
     "src/zarr/v2/",
@@ -224,4 +254,8 @@ filterwarnings = [
     "error:::zarr.*",
     "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
     "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
+    "ignore:Creating a zarr.buffer.gpu.*:UserWarning",
+]
+markers = [
+    "gpu: mark a test as requiring CuPy and GPU"
 ]

src/zarr/codecs/blosc.py

@@ -10,7 +10,8 @@
 
 from zarr.abc.codec import BytesBytesCodec
 from zarr.core.array_spec import ArraySpec
-from zarr.core.buffer import Buffer, as_numpy_array_wrapper
+from zarr.core.buffer import Buffer
+from zarr.core.buffer.cpu import as_numpy_array_wrapper
 from zarr.core.common import JSON, parse_enum, parse_named_configuration, to_thread
 from zarr.registry import register_codec
 

src/zarr/codecs/crc32c_.py

@@ -38,6 +38,7 @@ async def _decode_single(
         crc32_bytes = data[-4:]
         inner_bytes = data[:-4]
 
+        # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved
         computed_checksum = np.uint32(crc32c(cast(typing_extensions.Buffer, inner_bytes))).tobytes()
         stored_checksum = bytes(crc32_bytes)
         if computed_checksum != stored_checksum:

src/zarr/codecs/gzip.py

@@ -7,7 +7,8 @@
 
 from zarr.abc.codec import BytesBytesCodec
 from zarr.core.array_spec import ArraySpec
-from zarr.core.buffer import Buffer, as_numpy_array_wrapper
+from zarr.core.buffer import Buffer
+from zarr.core.buffer.cpu import as_numpy_array_wrapper
 from zarr.core.common import JSON, parse_named_configuration, to_thread
 from zarr.registry import register_codec
 

src/zarr/codecs/sharding.py

@@ -101,7 +101,7 @@ class _ShardIndex(NamedTuple):
 
     @property
     def chunks_per_shard(self) -> ChunkCoords:
-        result = tuple(self.offsets_and_lengths[:-1])
+        result = tuple(self.offsets_and_lengths.shape[0:-1])
         # The cast is required until https://github.com/numpy/numpy/pull/27211 is merged
         return cast(ChunkCoords, result)
 

src/zarr/codecs/zstd.py

@@ -9,7 +9,8 @@
 
 from zarr.abc.codec import BytesBytesCodec
 from zarr.core.array_spec import ArraySpec
-from zarr.core.buffer import Buffer, as_numpy_array_wrapper
+from zarr.core.buffer import Buffer
+from zarr.core.buffer.cpu import as_numpy_array_wrapper
 from zarr.core.common import JSON, parse_named_configuration, to_thread
 from zarr.registry import register_codec
 

src/zarr/core/array.py

@@ -513,15 +513,24 @@ async def _set_selection(
 
         # check value shape
         if np.isscalar(value):
-            value = np.asanyarray(value, dtype=self.metadata.dtype)
+            array_like = prototype.buffer.create_zero_length().as_array_like()
+            if isinstance(array_like, np._typing._SupportsArrayFunc):
+                # TODO: need to handle array types that don't support __array_function__
+                # like PyTorch and JAX
+                array_like_ = cast(np._typing._SupportsArrayFunc, array_like)
+            value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_)
         else:
             if not hasattr(value, "shape"):
                 value = np.asarray(value, self.metadata.dtype)
             # assert (
             #     value.shape == indexer.shape
             # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}"
             if not hasattr(value, "dtype") or value.dtype.name != self.metadata.dtype.name:
-                value = np.array(value, dtype=self.metadata.dtype, order="A")
+                if hasattr(value, "astype"):
+                    # Handle things that are already NDArrayLike more efficiently
+                    value = value.astype(dtype=self.metadata.dtype, order="A")
+                else:
+                    value = np.array(value, dtype=self.metadata.dtype, order="A")
         value = cast(NDArrayLike, value)
         # We accept any ndarray like object from the user and convert it
         # to a NDBuffer (or subclass). From this point onwards, we only pass

src/zarr/core/buffer/__init__.py

@@ -0,0 +1,19 @@
+from zarr.core.buffer.core import (
+    ArrayLike,
+    Buffer,
+    BufferPrototype,
+    NDArrayLike,
+    NDBuffer,
+    default_buffer_prototype,
+)
+from zarr.core.buffer.cpu import numpy_buffer_prototype
+
+__all__ = [
+    "ArrayLike",
+    "Buffer",
+    "NDArrayLike",
+    "NDBuffer",
+    "BufferPrototype",
+    "default_buffer_prototype",
+    "numpy_buffer_prototype",
+]