try to get the upstream-dev CI to complete again (#8823)

* install upstream-dev h5py and enable h5netcdf again

* try installing nightly `pyarrow` [skip-ci]

* use `micromamba` to speed up the force-removal of packages

* add a missing trailing backslash [skip-ci]

* revert the enabling of upstream-dev h5netcdf [skip-ci]

* also build `numexpr`

* replace the removed `numpy.array_api` with `array-api-strict`

* don't install upstream-dev `h5py`

* remove `numexpr` from the environment instead [skip-ci]

* ignore the missing typing in `array_api_strict`

* restore the old `numpy.array_api` import for `numpy<2`

* ignore the redefinition of `Array`

Author: keewis

ci/install-upstream-wheels.sh

@@ -4,10 +4,12 @@
 micromamba install "cython>=0.29.20" py-cpuinfo
 # temporarily (?) remove numbagg and numba
 micromamba remove -y numba numbagg
+# temporarily remove numexpr
+micromamba remove -y numexpr
 # temporarily remove backends
-micromamba remove -y cf_units h5py hdf5 netcdf4
+micromamba remove -y cf_units hdf5 h5py netcdf4
 # forcibly remove packages to avoid artifacts
-conda uninstall -y --force \
+micromamba remove -y --force \
     numpy \
     scipy \
     pandas \
@@ -30,8 +32,17 @@ python -m pip install \
     scipy \
     matplotlib \
     pandas
+# for some reason pandas depends on pyarrow already.
+# Remove once a `pyarrow` version compiled with `numpy>=2.0` is on `conda-forge`
+python -m pip install \
+    -i https://pypi.fury.io/arrow-nightlies/ \
+    --prefer-binary \
+    --no-deps \
+    --pre \
+    --upgrade \
+    pyarrow
 # without build isolation for packages compiling against numpy
-# TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime
+# TODO: remove once there are `numpy>=2.0` builds for these
 python -m pip install \
     --no-deps \
     --upgrade \

ci/requirements/all-but-dask.yml

@@ -5,6 +5,7 @@ channels:
 dependencies:
   - black
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/environment-3.12.yml

@@ -4,6 +4,7 @@ channels:
   - nodefaults
 dependencies:
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/environment-windows-3.12.yml

@@ -2,6 +2,7 @@ name: xarray-tests
 channels:
   - conda-forge
 dependencies:
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/environment-windows.yml

@@ -2,6 +2,7 @@ name: xarray-tests
 channels:
   - conda-forge
 dependencies:
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/environment.yml

@@ -4,6 +4,7 @@ channels:
   - nodefaults
 dependencies:
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/min-all-deps.yml

@@ -8,6 +8,7 @@ dependencies:
   # When upgrading python, numpy, or pandas, must also change
   # doc/user-guide/installing.rst, doc/user-guide/plotting.rst and setup.py.
   - python=3.9
+  - array-api-strict=1.0  # dependency for testing the array api compat
   - boto3=1.24
   - bottleneck=1.3
   - cartopy=0.21

pyproject.toml

@@ -138,6 +138,7 @@ module = [
   "toolz.*",
   "zarr.*",
   "numpy.exceptions.*", # remove once support for `numpy<2.0` has been dropped
+  "array_api_strict.*",
 ]
 
 # Gradually we want to add more modules to this list, ratcheting up our total

xarray/tests/test_array_api.py

@@ -1,18 +1,25 @@
 from __future__ import annotations
 
-import warnings
-
 import pytest
 
 import xarray as xr
 from xarray.testing import assert_equal
 
 np = pytest.importorskip("numpy", minversion="1.22")
 
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore")
-    import numpy.array_api as xp  # isort:skip
-    from numpy.array_api._array_object import Array  # isort:skip
+try:
+    import warnings
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+
+        import numpy.array_api as xp
+        from numpy.array_api._array_object import Array
+except ImportError:
+    # for `numpy>=2.0`
+    xp = pytest.importorskip("array_api_strict")
+
+    from array_api_strict._array_object import Array  # type: ignore[no-redef]
 
 
 @pytest.fixture