From 9f5a11021f071a082b126057967d1f713259effe Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 22 May 2024 20:45:38 +0530 Subject: [PATCH 01/61] Add CI job to test out-of-tree Pyodide builds --- .github/workflows/emscripten.yml | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/emscripten.yml diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 0000000000..230ce56c33 --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,79 @@ +# Attributed to NumPy https://github.com/numpy/numpy/pull/25894 +# https://github.com/numpy/numpy/blob/d2d2c25fa81b47810f5cbd85ea6485eb3a3ffec3/.github/workflows/emscripten.yml +# + +name: Pyodide wheel + +on: + # TODO: refine after this is ready to merge + [push, pull_request, workflow_dispatch] + +env: + FORCE_COLOR: 3 + PYODIDE_VERSION: 0.25.1 + # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. + # The appropriate versions can be found in the Pyodide repodata.json + # "info" field, or in Makefile.envs: + # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 + PYTHON_VERSION: 3.11.3 + EMSCRIPTEN_VERSION: 3.1.46 + NODE_VERSION: 18 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + build_wasm_emscripten: + name: Build and test Zarr for Pyodide + runs-on: ubuntu-22.04 + # To enable this workflow on a fork, comment out: + # FIXME: uncomment after this is ready to merge + # if: github.repository == 'zarr-developers/zarr-python' + steps: + - name: Checkout Zarr repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up Emscripten toolchain + uses: mymindstorm/setup-emsdk@v14 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pyodide-build + run: python -m pip install "pyodide-build==${{ env.PYODIDE_VERSION }}" + + - name: Build Zarr for Pyodide + run: | + pyodide build + + - name: Run Zarr tests for Pyodide + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + python -m pip install dist/*.whl + python -m pip install pytest pytest-cov + python -m pytest -v --cov=zarr --cov-config=pyproject.toml zarr + + - name: Upload Pyodide wheel artifact for debugging + # FIXME: Remove after this is ready to merge + uses: actions/upload-artifact@v4 + with: + name: zarr-pyodide-wheel + path: dist/*.whl + + From 29282fc9455df6543e6fd601fb1ffddd22a39722 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 24 May 2024 00:47:02 +0530 Subject: [PATCH 02/61] Add `[msgpack]` dependency for `numcodecs` --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 947bec9369..e995d0b971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,11 @@ requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.24', - 'fasteners', - 'numcodecs>=0.10.0', - 'crc32c', - 'zstandard', + 'fasteners; sys_platform != "emscripten"', + # 'numcodecs[msgpack]>=0.10.0; sys_platform != "emscripten"', # does not currently work + 'numcodecs[msgpack]>=0.10.0', # this works + 'crc32c', # not available in Pyodide, have to see if we can make it optional for Pyodide + 'zstandard', # not available in Pyodide, have to see if we can make it optional for Pyodide 'typing_extensions', 'donfig' ] From d465742a193e145340e3afe91a7e0bd5b3893419 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 27 May 2024 23:26:24 +0530 Subject: [PATCH 03/61] Bump to Pyodide 0.26.0, update comments --- .github/workflows/emscripten.yml | 9 +++------ pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 230ce56c33..22437d05c7 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -1,6 +1,5 @@ # Attributed to NumPy https://github.com/numpy/numpy/pull/25894 # https://github.com/numpy/numpy/blob/d2d2c25fa81b47810f5cbd85ea6485eb3a3ffec3/.github/workflows/emscripten.yml -# name: Pyodide wheel @@ -10,13 +9,13 @@ on: env: FORCE_COLOR: 3 - PYODIDE_VERSION: 0.25.1 + PYODIDE_VERSION: 0.26.0 # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 - PYTHON_VERSION: 3.11.3 - EMSCRIPTEN_VERSION: 3.1.46 + PYTHON_VERSION: 3.12.1 + EMSCRIPTEN_VERSION: 3.1.58 NODE_VERSION: 18 concurrency: @@ -75,5 +74,3 @@ jobs: with: name: zarr-pyodide-wheel path: dist/*.whl - - diff --git a/pyproject.toml b/pyproject.toml index e995d0b971..e7e89c5670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,8 +27,8 @@ dependencies = [ 'fasteners; sys_platform != "emscripten"', # 'numcodecs[msgpack]>=0.10.0; sys_platform != "emscripten"', # does not currently work 'numcodecs[msgpack]>=0.10.0', # this works - 'crc32c', # not available in Pyodide, have to see if we can make it optional for Pyodide - 'zstandard', # not available in Pyodide, have to see if we can make it optional for Pyodide + 'crc32c', + 'zstandard', 'typing_extensions', 'donfig' ] From cdf0bb205cd962ab518b220ab9fc2f9962ceeae6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:06:22 +0530 Subject: [PATCH 04/61] Try to run tests without async --- .github/workflows/emscripten.yml | 2 +- pyproject.toml | 2 +- tests/v3/_shared.py | 16 ++++++++++++++++ tests/v3/test_buffer.py | 5 +++-- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/v3/_shared.py diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 22437d05c7..88fc6ff7e6 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -66,7 +66,7 @@ jobs: source .venv-pyodide/bin/activate python -m pip install dist/*.whl python -m pip install pytest pytest-cov - python -m pytest -v --cov=zarr --cov-config=pyproject.toml zarr + python -m pytest -v --cov=zarr --cov-config=pyproject.toml - name: Upload Pyodide wheel artifact for debugging # FIXME: Remove after this is ready to merge diff --git a/pyproject.toml b/pyproject.toml index e7e89c5670..da57fdbea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,7 +249,7 @@ minversion = "7" testpaths = ["tests"] log_cli_level = "INFO" xfail_strict = true -asyncio_mode = "auto" +# asyncio_mode = "auto" doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS", diff --git a/tests/v3/_shared.py b/tests/v3/_shared.py new file mode 100644 index 0000000000..d853a02675 --- /dev/null +++ b/tests/v3/_shared.py @@ -0,0 +1,16 @@ +# A common file that can be used to add constants, functions, +# convenience classes, etc. that are shared across multiple tests + +import platform +import sys + +import pytest + +IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] + + +def asyncio_tests_wrapper(func): + if IS_WASM: + return func + else: + return pytest.mark.asyncio(func) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 2f58d116fe..07ef96bf6e 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -5,11 +5,12 @@ import numpy as np import numpy.typing as npt -import pytest from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer +from ._shared import asyncio_tests_wrapper + if TYPE_CHECKING: from typing_extensions import Self @@ -45,7 +46,7 @@ def test_nd_array_like(xp): assert isinstance(ary, NDArrayLike) -@pytest.mark.asyncio +@asyncio_tests_wrapper async def test_async_array_factory(store_path): expect = np.zeros((9, 9), dtype="uint16", order="F") a = await AsyncArray.create( From dfe032115bdac8112ef60f9187bfd3c653da4a6e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:51:53 +0530 Subject: [PATCH 05/61] Move shared file to rootdir, outside v2 and v3 --- tests/{v3 => }/_shared.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{v3 => }/_shared.py (100%) diff --git a/tests/v3/_shared.py b/tests/_shared.py similarity index 100% rename from tests/v3/_shared.py rename to tests/_shared.py From b100ec9d0f7e1473349f5b1e157ff166efe01e0c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:52:27 +0530 Subject: [PATCH 06/61] Move `fasteners` import inside ThreadSynchronizer --- src/zarr/v2/sync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zarr/v2/sync.py b/src/zarr/v2/sync.py index 49684a51ee..e8dff71e32 100644 --- a/src/zarr/v2/sync.py +++ b/src/zarr/v2/sync.py @@ -2,8 +2,6 @@ from collections import defaultdict from threading import Lock -import fasteners - class ThreadSynchronizer: """Provides synchronization using thread locks.""" @@ -42,6 +40,7 @@ def __init__(self, path): def __getitem__(self, item): path = os.path.join(self.path, item) + import fasteners lock = fasteners.InterProcessLock(path) return lock From b0dddca9d6d302e100f22106634cc58fe11f7621 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:02:53 +0530 Subject: [PATCH 07/61] Make the tests directory importable, fix `_shared` --- tests/__init__.py | 0 tests/_shared.py | 1 + tests/v3/test_buffer.py | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/_shared.py b/tests/_shared.py index d853a02675..969936db01 100644 --- a/tests/_shared.py +++ b/tests/_shared.py @@ -1,5 +1,6 @@ # A common file that can be used to add constants, functions, # convenience classes, etc. that are shared across multiple tests +# similar to tests/v2/util.py, but can be used for both v2 and v3 import platform import sys diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 07ef96bf6e..2e9dcde2a5 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -9,7 +9,7 @@ from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer -from ._shared import asyncio_tests_wrapper +from .._shared import asyncio_tests_wrapper if TYPE_CHECKING: from typing_extensions import Self From d227728f23ee33dd9264490281f67f6123434f2f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:03:10 +0530 Subject: [PATCH 08/61] Import list of greetings from `numcodecs` --- tests/v2/test_core.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index f053725b95..75222893bb 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -27,7 +27,14 @@ Zlib, ) from numcodecs.compat import ensure_bytes, ensure_ndarray -from numcodecs.tests.common import greetings + +try: + from numcodecs.tests.common import greetings +except ModuleNotFoundError: + greetings = ['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!', + 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', + 'こんにちは世界', '世界，你好！', 'Helló, világ!', 'Zdravo svete!', + 'เฮลโลเวิลด์'] from numpy.testing import assert_array_almost_equal, assert_array_equal import zarr.v2 From fdb2bef749d9ad3be7d0396264ce5d31a6924c02 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:10:21 +0530 Subject: [PATCH 09/61] Skip some tests that use threading --- tests/v2/test_sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index ea6fd0523d..8ff6ab3819 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -8,6 +8,7 @@ import numpy as np from numpy.testing import assert_array_equal +import pytest from zarr.v2.attrs import Attributes from zarr.v2.core import Array @@ -20,7 +21,10 @@ from .test_core import TestArray from .test_hierarchy import TestGroup +from tests._shared import IS_WASM + +@pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") class TestAttributesWithThreadSynchronizer(TestAttributes): def init_attributes(self, store, read_only=False, cache=True): key = ".zattrs" @@ -30,6 +34,7 @@ def init_attributes(self, store, read_only=False, cache=True): ) +@pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") class TestAttributesProcessSynchronizer(TestAttributes): def init_attributes(self, store, read_only=False, cache=True): key = ".zattrs" From 621077a1f7e7ea057b62870209cca625574d469f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:11:04 +0530 Subject: [PATCH 10/61] Skip some tests that use `fcntl` --- tests/v2/test_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 8ff6ab3819..9b9ddd3618 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -153,6 +153,7 @@ def test_hexdigest(self): assert "05b0663ffe1785f38d3a459dec17e57a18f254af" == z.hexdigest() +@pytest.mark.skipif(IS_WASM, reason="fcntl not available in WASM") class TestArrayWithProcessSynchronizer(TestArray, MixinArraySyncTests): def create_array(self, read_only=False, **kwargs): path = tempfile.mkdtemp() @@ -291,6 +292,7 @@ def test_synchronizer_property(self): assert isinstance(g.synchronizer, ThreadSynchronizer) +@pytest.mark.skipif(IS_WASM, reason="fcntl not available in WASM") class TestGroupWithProcessSynchronizer(TestGroup, MixinGroupSyncTests): def create_store(self): path = tempfile.mkdtemp() From 7ae9a975bf42e13a327732ef52b8275ea314a7d0 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:14:47 +0530 Subject: [PATCH 11/61] Skip tests that require `dbm` --- tests/v2/test_core.py | 4 ++++ tests/v2/test_hierarchy.py | 4 ++++ tests/v2/test_storage.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 75222893bb..976dbccfc9 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -65,6 +65,8 @@ from zarr.v2.util import buffer_size from .util import abs_container, skip_test_env_var, have_fsspec, mktemp +from tests._shared import IS_WASM + # noinspection PyMethodMayBeStatic @@ -1992,6 +1994,7 @@ def create_store(self): return store +@pytest.mark.skipif(IS_WASM, reason="no dbm support in WASM") class TestArrayWithDBMStore(TestArray): def create_store(self): path = mktemp(suffix=".anydbm") @@ -2003,6 +2006,7 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(IS_WASM, reason="no dbm support in WASM") @pytest.mark.skip(reason="can't get bsddb3 to work on CI right now") class TestArrayWithDBMStoreBerkeleyDB(TestArray): def create_store(self): diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 23c5a56edf..31553242f9 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -45,6 +45,8 @@ from zarr.v2.util import InfoReporter from .util import skip_test_env_var, have_fsspec, abs_container, mktemp +from tests._shared import IS_WASM + # noinspection PyStatementEffect @@ -1122,6 +1124,7 @@ def test_move(self): pass +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestGroupWithDBMStore(TestGroup): @staticmethod def create_store(): @@ -1131,6 +1134,7 @@ def create_store(): return store, None +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestGroupWithDBMStoreBerkeleyDB(TestGroup): @staticmethod def create_store(): diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 17b80e6a5c..6e49c09cf2 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -59,6 +59,7 @@ from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps +from tests._shared import IS_WASM @contextmanager def does_not_raise(): @@ -1765,6 +1766,7 @@ def test_store_and_retrieve_ndarray(self): assert np.array_equiv(y, x) +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestDBMStore(StoreTests): def create_store(self, dimension_separator=None): path = mktemp(suffix=".anydbm") @@ -1780,6 +1782,7 @@ def test_context_manager(self): assert 2 == len(store) +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestDBMStoreDumb(TestDBMStore): def create_store(self, **kwargs): path = mktemp(suffix=".dumbdbm") From 22eb6daaa47fbed0ef6772a42d8d523051daa904 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:37:11 +0530 Subject: [PATCH 12/61] Move `IS_WASM` logic to internal `zarr` API --- src/zarr/testing/utils.py | 6 ++++++ tests/__init__.py | 0 tests/_shared.py | 17 ----------------- tests/v2/test_core.py | 2 +- tests/v2/test_hierarchy.py | 2 +- tests/v2/test_storage.py | 2 +- tests/v2/test_sync.py | 2 +- 7 files changed, 10 insertions(+), 21 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/_shared.py diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 04b05d1b1c..2d45d34579 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,5 +1,8 @@ from __future__ import annotations +import platform +import sys + from zarr.buffer import Buffer from zarr.common import BytesLike @@ -16,3 +19,6 @@ def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | N if isinstance(b2, Buffer): b2 = b2.to_bytes() assert b1 == b2 + + +IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/_shared.py b/tests/_shared.py deleted file mode 100644 index 969936db01..0000000000 --- a/tests/_shared.py +++ /dev/null @@ -1,17 +0,0 @@ -# A common file that can be used to add constants, functions, -# convenience classes, etc. that are shared across multiple tests -# similar to tests/v2/util.py, but can be used for both v2 and v3 - -import platform -import sys - -import pytest - -IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] - - -def asyncio_tests_wrapper(func): - if IS_WASM: - return func - else: - return pytest.mark.asyncio(func) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 976dbccfc9..9da81cb787 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -65,7 +65,7 @@ from zarr.v2.util import buffer_size from .util import abs_container, skip_test_env_var, have_fsspec, mktemp -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM # noinspection PyMethodMayBeStatic diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 31553242f9..4936bf1b2f 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -45,7 +45,7 @@ from zarr.v2.util import InfoReporter from .util import skip_test_env_var, have_fsspec, abs_container, mktemp -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM # noinspection PyStatementEffect diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 6e49c09cf2..47db11950b 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -59,7 +59,7 @@ from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM @contextmanager def does_not_raise(): diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 9b9ddd3618..5065d066aa 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -21,7 +21,7 @@ from .test_core import TestArray from .test_hierarchy import TestGroup -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM @pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") From 683694773e758b3f7b0c919f3ae39bfd1e4dea51 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:37:34 +0530 Subject: [PATCH 13/61] Skip a few tests trying to import `multiprocessing` --- tests/v2/test_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 5065d066aa..8a86f40d6f 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -101,6 +101,7 @@ def test_parallel_append(self): pool.terminate() +@pytest.mark.skipif(IS_WASM, reason="no multiprocessing support in WASM") class TestArrayWithThreadSynchronizer(TestArray, MixinArraySyncTests): def create_array(self, read_only=False, **kwargs): store = KVStore(dict()) @@ -265,6 +266,7 @@ def test_parallel_require_group(self): pool.terminate() +@pytest.mark.skipif(IS_WASM, reason="no multiprocessing support in WASM") class TestGroupWithThreadSynchronizer(TestGroup, MixinGroupSyncTests): def create_group( self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None From fe3bf274de04489c3f20fff28184bbd60794b6b5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:41:08 +0530 Subject: [PATCH 14/61] Skip tests that use async and threading code This is because they test async code and also use threads. See the following issues: https://github.com/pyodide/pyodide/issues/2221 https://github.com/pyodide/pyodide/issues/237 --- tests/v3/test_codecs.py | 8 +++++++- tests/v3/test_group.py | 4 ++++ tests/v3/test_sync.py | 6 ++++++ tests/v3/test_v2.py | 2 ++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index a595b12494..744699e07a 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -25,7 +25,7 @@ from zarr.config import config from zarr.indexing import morton_order_iter from zarr.store import MemoryStore, StorePath -from zarr.testing.utils import assert_bytes_equal +from zarr.testing.utils import IS_WASM, assert_bytes_equal @dataclass(frozen=True) @@ -65,6 +65,7 @@ def order_from_dim(order: Literal["F", "C"], ndim: int) -> tuple[int, ...]: return tuple(range(ndim)) +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -95,6 +96,7 @@ def test_sharding( assert np.array_equal(sample_data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -128,6 +130,7 @@ def test_sharding_partial( assert np.array_equal(sample_data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_read( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -155,6 +158,7 @@ def test_sharding_partial_read( assert np.all(read_data == 1) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_overwrite( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -191,6 +195,7 @@ def test_sharding_partial_overwrite( assert np.array_equal(data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize( "outer_index_location", ["start", "end"], @@ -298,6 +303,7 @@ async def test_order( assert_bytes_equal(await (store / "order/0.0").get(), z._store["0.0"]) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 36b82f413c..4798a0ffdf 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -6,6 +6,7 @@ from zarr.buffer import Buffer from zarr.store.core import make_store_path from zarr.sync import sync +from zarr.testing.utils import IS_WASM if TYPE_CHECKING: from zarr.common import ZarrFormat @@ -18,6 +19,7 @@ from zarr.store import StorePath +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") # todo: put RemoteStore in here @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) def test_group_children(store: MemoryStore | LocalStore) -> None: @@ -55,6 +57,7 @@ def test_group_children(store: MemoryStore | LocalStore) -> None: assert sorted(dict(members_observed)) == sorted(members_expected) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("store", (("local", "memory")), indirect=["store"]) def test_group(store: MemoryStore | LocalStore) -> None: store_path = StorePath(store) @@ -94,6 +97,7 @@ def test_group(store: MemoryStore | LocalStore) -> None: assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) @pytest.mark.parametrize("exists_ok", (True, False)) def test_group_create(store: MemoryStore | LocalStore, exists_ok: bool) -> None: diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 5b953573d8..2dbc8559b6 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -6,6 +6,7 @@ import pytest from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync +from zarr.testing.utils import IS_WASM @pytest.fixture(params=[True, False]) @@ -31,12 +32,14 @@ def test_get_lock() -> None: assert lock is lock2 +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(return_value="foo") assert sync(foo(), loop=sync_loop) == "foo" foo.assert_awaited_once() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(side_effect=ValueError("foo-bar")) with pytest.raises(ValueError, match="foo-bar"): @@ -44,6 +47,7 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo.assert_awaited_once() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_timeout() -> None: duration = 0.002 @@ -54,6 +58,7 @@ async def foo() -> None: sync(foo(), timeout=duration / 2) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: def foo() -> str: return "foo" @@ -97,6 +102,7 @@ def test_sync_raises_if_loop_is_invalid_type() -> None: foo.assert_not_awaited() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_mixin(sync_loop) -> None: class AsyncFoo: def __init__(self) -> None: diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index 2a38dc8fdc..d884cebc16 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -6,6 +6,7 @@ from zarr.abc.store import Store from zarr.array import Array from zarr.store import MemoryStore, StorePath +from zarr.testing.utils import IS_WASM @pytest.fixture @@ -13,6 +14,7 @@ def store() -> Iterator[Store]: yield StorePath(MemoryStore()) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_simple(store: Store): data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) From 08997ec474905903cec7eb24c9de8d1bceb04416 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:51:18 +0530 Subject: [PATCH 15/61] Improve `asyncio_tests_wrapper`, fix test imports --- tests/v3/test_buffer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 2e9dcde2a5..7af2915724 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -5,16 +5,21 @@ import numpy as np import numpy.typing as npt +import pytest from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer - -from .._shared import asyncio_tests_wrapper +from zarr.testing.utils import IS_WASM if TYPE_CHECKING: from typing_extensions import Self +# Helper function to skip async tests on WASM platforms +def asyncio_tests_wrapper(func): + return func if IS_WASM else pytest.mark.asyncio(func) + + class MyNDArrayLike(np.ndarray): """An example of a ndarray-like class""" From 9bfc860c2f84b0667172154cea95400626dd0dda Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:07:15 +0530 Subject: [PATCH 16/61] Skip entire `test_codecs.py` file --- tests/v3/test_codecs.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index 744699e07a..c44451a07b 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -27,6 +27,11 @@ from zarr.store import MemoryStore, StorePath from zarr.testing.utils import IS_WASM, assert_bytes_equal +# Skip entire file if running on WASM platforms, see +# 1. https://github.com/pyodide/pyodide/issues/2221 +# 2. https://github.com/pyodide/pyodide/issues/237 +pytestmark = pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") + @dataclass(frozen=True) class _AsyncArrayProxy: @@ -65,7 +70,6 @@ def order_from_dim(order: Literal["F", "C"], ndim: int) -> tuple[int, ...]: return tuple(range(ndim)) -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -96,7 +100,6 @@ def test_sharding( assert np.array_equal(sample_data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -130,7 +133,6 @@ def test_sharding_partial( assert np.array_equal(sample_data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_read( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -158,7 +160,6 @@ def test_sharding_partial_read( assert np.all(read_data == 1) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_overwrite( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -195,7 +196,6 @@ def test_sharding_partial_overwrite( assert np.array_equal(data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize( "outer_index_location", ["start", "end"], @@ -303,7 +303,6 @@ async def test_order( assert_bytes_equal(await (store / "order/0.0").get(), z._store["0.0"]) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) From 9bcb350b18ea534d51d6c96ba50d5c6989439431 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:10:06 +0530 Subject: [PATCH 17/61] Skip yet another test that requires threads --- tests/v3/test_codecs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index c44451a07b..6a486b3b76 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -411,6 +411,7 @@ async def test_transpose( assert await (store / "transpose/0.0").get() == await (store / "transpose_zarr/0.0").get() +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_transpose_invalid( store: Store, ): From 9985abb34fe5e309a43b37af8b8d7a85c8a26029 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:10:29 +0530 Subject: [PATCH 18/61] xfail test where array's fill values are different --- tests/v2/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 9da81cb787..40e678bb97 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -1706,6 +1706,7 @@ def create_store(self): store = N5Store(path) return store + @pytest.mark.xfail(reason="Can't get this to pass under WASM right now") def test_array_0d(self): # test behaviour for array with 0 dimensions From 7ea12ef6acdeafb4ae6c5effd363b8e615df6cc2 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:08:23 +0530 Subject: [PATCH 19/61] xfail test because Emscripten FS --- tests/v2/test_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 47db11950b..b5895d7dab 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -939,6 +939,7 @@ def create_store(self, normalize_keys=False, dimension_separator=".", **kwargs): ) return store + @pytest.mark.xfail(reason="Emscripten filesystem handles umasks differently") def test_filesystem_path(self): # test behaviour with path that does not exist path = "data/store" From a6565ded85d4acc1e39b44a59f38da08af763baf Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:16:54 +0530 Subject: [PATCH 20/61] Skip last test that tries to run threads --- tests/v3/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 2dbc8559b6..1030b3152e 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -18,6 +18,7 @@ def sync_loop(request) -> asyncio.AbstractEventLoop | None: return None +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_get_loop() -> None: # test that calling _get_loop() twice returns the same loop loop = _get_loop() From 85f621cb8c9e199df9dedc37b5e21e501287d36e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:20:39 +0530 Subject: [PATCH 21/61] Another test that tries to run threads --- tests/v3/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 1030b3152e..03acbb044f 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -68,6 +68,7 @@ def foo() -> str: sync(foo(), loop=sync_loop) +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_closed() -> None: loop = _get_loop() From 1a64255fb996bce57f4dafdbbc030f8268e3f7ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:21:33 +0530 Subject: [PATCH 22/61] xfail another array's differing `fill_values` test --- tests/v2/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 40e678bb97..72f465df70 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -983,6 +983,7 @@ def test_0len_dim_2d(self): z.store.close() # noinspection PyStatementEffect + @pytest.mark.xfail(reason="Can't get this to pass under WASM right now") def test_array_0d(self): # test behaviour for array with 0 dimensions From c8cb38bb896e841bb34a065e64d168e86d666738 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:27:17 +0530 Subject: [PATCH 23/61] Skip entire sync file under WASM, no threading --- tests/v3/test_sync.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 03acbb044f..69cbf815bf 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -8,6 +8,8 @@ from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync from zarr.testing.utils import IS_WASM +pytestmark = pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") + @pytest.fixture(params=[True, False]) def sync_loop(request) -> asyncio.AbstractEventLoop | None: @@ -18,7 +20,6 @@ def sync_loop(request) -> asyncio.AbstractEventLoop | None: return None -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_get_loop() -> None: # test that calling _get_loop() twice returns the same loop loop = _get_loop() @@ -33,14 +34,12 @@ def test_get_lock() -> None: assert lock is lock2 -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(return_value="foo") assert sync(foo(), loop=sync_loop) == "foo" foo.assert_awaited_once() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(side_effect=ValueError("foo-bar")) with pytest.raises(ValueError, match="foo-bar"): @@ -48,7 +47,6 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo.assert_awaited_once() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_timeout() -> None: duration = 0.002 @@ -59,7 +57,6 @@ async def foo() -> None: sync(foo(), timeout=duration / 2) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: def foo() -> str: return "foo" @@ -68,7 +65,6 @@ def foo() -> str: sync(foo(), loop=sync_loop) -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_closed() -> None: loop = _get_loop() @@ -104,7 +100,6 @@ def test_sync_raises_if_loop_is_invalid_type() -> None: foo.assert_not_awaited() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_mixin(sync_loop) -> None: class AsyncFoo: def __init__(self) -> None: From eb36d40077b63c0958a20cb84a7be1ec74a3b214 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 21:46:32 +0530 Subject: [PATCH 24/61] Restore pytest config options, remove when needed --- .github/workflows/emscripten.yml | 2 ++ pyproject.toml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 88fc6ff7e6..c672c8f32d 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -62,6 +62,8 @@ jobs: - name: Run Zarr tests for Pyodide run: | + # Avoid missing asyncio plugin error from pytest, unavailable in Pyodide + if grep -q 'asyncio_mode = "auto"' "pyproject.toml"; then sed '/asyncio_mode = "auto"/d' "pyproject.toml" > temp && mv temp "pyproject.toml"; fi pyodide venv .venv-pyodide source .venv-pyodide/bin/activate python -m pip install dist/*.whl diff --git a/pyproject.toml b/pyproject.toml index da57fdbea6..48eb5b1544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,7 +249,8 @@ minversion = "7" testpaths = ["tests"] log_cli_level = "INFO" xfail_strict = true -# asyncio_mode = "auto" +# Doesn't work under WASM, remove when running Pyodide test suite +asyncio_mode = "auto" doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS", From b3a5b8aa66aaed5ad8aae7b71f9a79fd90475aef Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:22:34 +0530 Subject: [PATCH 25/61] Bump Emscripten, Pyodide xbuildenv, Node.js versions --- .github/workflows/emscripten.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index c672c8f32d..74d4b1377c 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -9,14 +9,14 @@ on: env: FORCE_COLOR: 3 - PYODIDE_VERSION: 0.26.0 + PYODIDE_VERSION: "https://github.com/pyodide/pyodide-build-environment-nightly/releases/download/20250523-emscripten_4.0.9/xbuildenv.tar.bz2" # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 - PYTHON_VERSION: 3.12.1 - EMSCRIPTEN_VERSION: 3.1.58 - NODE_VERSION: 18 + PYTHON_VERSION: 3.13 # any 3.13.x version works + EMSCRIPTEN_VERSION: 4.0.9 + NODE_VERSION: 22 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From 42d2792db5dbd7112197fd524965c81b13d27fb0 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:23:12 +0530 Subject: [PATCH 26/61] Running on `ubuntu-latest` should be fine --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 74d4b1377c..651356ca70 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -28,7 +28,7 @@ permissions: jobs: build_wasm_emscripten: name: Build and test Zarr for Pyodide - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest # To enable this workflow on a fork, comment out: # FIXME: uncomment after this is ready to merge # if: github.repository == 'zarr-developers/zarr-python' From 27068e2b88dbebbef843ea2fcb62546687257616 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:23:35 +0530 Subject: [PATCH 27/61] Don't persist credentials with git clone --- .github/workflows/emscripten.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 651356ca70..baf11ac822 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -35,6 +35,8 @@ jobs: steps: - name: Checkout Zarr repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python ${{ env.PYTHON_VERSION }} id: setup-python From aff9b18fa84dd4e5477a5c754737fed45bb49f1d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:23:45 +0530 Subject: [PATCH 28/61] Don't pin the version of `pyodide-build` --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index baf11ac822..8a989dad45 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -56,7 +56,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} - name: Install pyodide-build - run: python -m pip install "pyodide-build==${{ env.PYODIDE_VERSION }}" + run: python -m pip install pyodide-build - name: Build Zarr for Pyodide run: | From bb2c13665e311fd5461eb1fef3fdfd7c4b449a80 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:23:56 +0530 Subject: [PATCH 29/61] Use same xbuildenv for building and testing --- .github/workflows/emscripten.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 8a989dad45..5c9a0d9dcd 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -60,6 +60,7 @@ jobs: - name: Build Zarr for Pyodide run: | + pyodide xbuildenv install --url ${{ env.PYODIDE_VERSION }} pyodide build - name: Run Zarr tests for Pyodide From 710195a6b02c8ec3ce72dec0ad9e91bb9b7cab8a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:24:12 +0530 Subject: [PATCH 30/61] Temporarily build numcodecs for WASM as well --- .github/workflows/emscripten.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 5c9a0d9dcd..d5484ca956 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -63,12 +63,35 @@ jobs: pyodide xbuildenv install --url ${{ env.PYODIDE_VERSION }} pyodide build + ### (Temporarily) build numcodecs as well, as we have an older version in the Pyodide distribution (v0.13.1) + + - name: Clone numcodecs repository + uses: actions/checkout@v4 + with: + # See https://github.com/zarr-developers/numcodecs/pull/529 + repository: agriyakhetarpal/numcodecs + ref: setup-emscripten-ci + path: numcodecs-wasm + submodules: recursive + persist-credentials: false + + - name: Build numcodecs for WASM + run: pyodide build + working-directory: numcodecs-wasm + + ### Back to Zarr repository to run tests + - name: Run Zarr tests for Pyodide run: | - # Avoid missing asyncio plugin error from pytest, unavailable in Pyodide - if grep -q 'asyncio_mode = "auto"' "pyproject.toml"; then sed '/asyncio_mode = "auto"/d' "pyproject.toml" > temp && mv temp "pyproject.toml"; fi + # Set up Pyodide virtual environment and activate it pyodide venv .venv-pyodide source .venv-pyodide/bin/activate + + # Avoid missing asyncio plugin error from pytest, unavailable in Pyodide + if grep -q 'asyncio_mode = "auto"' "pyproject.toml"; then sed '/asyncio_mode = "auto"/d' "pyproject.toml" > temp && mv temp "pyproject.toml"; fi + + # Install numcodecs + python -m pip install numcodecs-wasm/dist/*.whl python -m pip install dist/*.whl python -m pip install pytest pytest-cov python -m pytest -v --cov=zarr --cov-config=pyproject.toml From 07e5bc9a2758b2e436754688f8ae2674373262a7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 26 May 2025 17:33:40 +0530 Subject: [PATCH 31/61] Use Pyodide 0.28.0a2 for now --- .github/workflows/emscripten.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index d5484ca956..66f5f57828 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -9,7 +9,7 @@ on: env: FORCE_COLOR: 3 - PYODIDE_VERSION: "https://github.com/pyodide/pyodide-build-environment-nightly/releases/download/20250523-emscripten_4.0.9/xbuildenv.tar.bz2" + PYODIDE_VERSION: 0.28.0a2 # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: @@ -60,7 +60,7 @@ jobs: - name: Build Zarr for Pyodide run: | - pyodide xbuildenv install --url ${{ env.PYODIDE_VERSION }} + pyodide xbuildenv install ${{ env.PYODIDE_VERSION }} pyodide build ### (Temporarily) build numcodecs as well, as we have an older version in the Pyodide distribution (v0.13.1) From 403fbb0f374a4476f63ba414f56399e6514d74fa Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 27 May 2025 16:16:54 +0530 Subject: [PATCH 32/61] Use `fetch-depth: 0` to bring correct versions --- .github/workflows/emscripten.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 66f5f57828..d15d9d698b 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -36,6 +36,7 @@ jobs: - name: Checkout Zarr repository uses: actions/checkout@v4 with: + fetch-depth: 0 persist-credentials: false - name: Set up Python ${{ env.PYTHON_VERSION }} @@ -74,6 +75,7 @@ jobs: path: numcodecs-wasm submodules: recursive persist-credentials: false + fetch-depth: 0 - name: Build numcodecs for WASM run: pyodide build From 24dbc779d6ef1657e1c64736ec9a82ce66eecb16 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:36:06 +0530 Subject: [PATCH 33/61] Skip `test_multiprocessing` for WASM --- tests/test_array.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_array.py b/tests/test_array.py index a6bcd17c4b..ddbaddd7e8 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -49,6 +49,7 @@ from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath +from zarr.testing.utils import is_wasm if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike @@ -1677,6 +1678,10 @@ def _index_array(arr: Array, index: Any) -> Any: return arr[index] +@pytest.mark.skipif( + is_wasm(), + reason = "can't start new processes in Pyodide", +) @pytest.mark.parametrize( "method", [ From ecea615ed70310b230722dccec2c9dda833345ae Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:43:29 +0530 Subject: [PATCH 34/61] Skip all sync tests --- tests/test_sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index 13b475f8da..6519e06e97 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -16,7 +16,12 @@ sync, ) from zarr.storage import MemoryStore +from zarr.testing.utils import is_wasm +pytestmark = pytest.mark.skipif( + is_wasm(), + reason = "can't start new threads in Pyodide/WASM, so the synchronous API doesn't work", +) @pytest.fixture(params=[True, False]) def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None: From d919bd78c45bba6cffca714872c829c69a1000ab Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:43:44 +0530 Subject: [PATCH 35/61] xfail `test_array_roundtrip` for now --- tests/test_properties.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_properties.py b/tests/test_properties.py index d48dfe2fef..5188b5de71 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -8,6 +8,7 @@ from numpy.testing import assert_array_equal from zarr.core.buffer import default_buffer_prototype +from zarr.testing.utils import is_wasm pytest.importorskip("hypothesis") @@ -76,6 +77,7 @@ def deep_equal(a: Any, b: Any) -> bool: return a == b +@pytest.mark.xfail(is_wasm(), reason="Hypothesis deadline being exceeded in Pyodide/WASM") @given(data=st.data(), zarr_format=zarr_formats) def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) From 0bb7d475236e9151765ccd2af93d01863101bd1b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:44:17 +0530 Subject: [PATCH 36/61] Skip `test_group_members_performance[memory]` for now --- tests/test_group.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_group.py b/tests/test_group.py index b4dace2568..8e20593f04 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -42,6 +42,7 @@ from zarr.storage._common import make_store_path from zarr.storage._utils import _join_paths, normalize_path from zarr.testing.store import LatencyStore +from zarr.testing.utils import is_wasm from .conftest import meta_from_array, parse_store @@ -1969,7 +1970,13 @@ async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) - raise ValueError(f"Invalid impl: {impl}") -@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize( + "store", + [ + pytest.param("memory", marks=pytest.mark.skipif(is_wasm(), reason="performance is marginally worse for Pyodide/WASM")), + ], + indirect=True, +) def test_group_members_performance(store: Store) -> None: """ Test that the execution time of Group.members is less than the number of members times the From fb59ebad060d2513547aa50810cf746121ea2989 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:46:03 +0530 Subject: [PATCH 37/61] Set concurrency and max workers as 1 --- src/zarr/core/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 2a10943d80..1095480105 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig +from zarr.testing.utils import is_wasm if TYPE_CHECKING: from donfig.config_obj import ConfigSet @@ -106,8 +107,8 @@ def enable_gpu(self) -> ConfigSet: ], }, }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, + "async": {"concurrency": 1 if is_wasm() else 10, "timeout": None}, + "threading": {"max_workers": None if is_wasm() else 1}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", From 4c6bed6bf53cb4b647612b6ed37a9e8e8f7d3985 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 00:45:35 +0530 Subject: [PATCH 38/61] Update `zarr.config` tests to match --- tests/test_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 2cbf172752..fa7acb7dcf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -43,6 +43,7 @@ TestBuffer, TestNDArrayLike, ) +from zarr.testing.utils import is_wasm def test_config_defaults_set() -> None: @@ -82,8 +83,8 @@ def test_config_defaults_set() -> None: ], }, }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, + "async": {"concurrency": 1 if is_wasm() else 10, "timeout": None}, + "threading": {"max_workers": 1 if is_wasm() else None}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", From 6754131e5e697da1365b5a0e97d9b53e0bfff969 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 01:23:43 +0530 Subject: [PATCH 39/61] Move WASM check to resolve circular import --- src/zarr/constants.py | 4 ++++ src/zarr/core/config.py | 7 ++++--- src/zarr/core/sync.py | 40 +++++++++++++++++++++++++++++++++++++++ src/zarr/testing/utils.py | 7 ------- tests/test_array.py | 6 +++--- tests/test_config.py | 6 +++--- tests/test_group.py | 9 +++++++-- tests/test_properties.py | 4 ++-- tests/test_sync.py | 7 ++++--- 9 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 src/zarr/constants.py diff --git a/src/zarr/constants.py b/src/zarr/constants.py new file mode 100644 index 0000000000..27f20caab7 --- /dev/null +++ b/src/zarr/constants.py @@ -0,0 +1,4 @@ +import platform +import sys + +IS_WASM: bool = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 1095480105..d5c21c1a42 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -32,7 +32,8 @@ from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig -from zarr.testing.utils import is_wasm + +from zarr.constants import IS_WASM if TYPE_CHECKING: from donfig.config_obj import ConfigSet @@ -107,8 +108,8 @@ def enable_gpu(self) -> ConfigSet: ], }, }, - "async": {"concurrency": 1 if is_wasm() else 10, "timeout": None}, - "threading": {"max_workers": None if is_wasm() else 1}, + "async": {"concurrency": 1 if IS_WASM else 10, "timeout": None}, + "threading": {"max_workers": None if IS_WASM else 1}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index d9b4839e8e..98d7abace2 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -10,6 +10,7 @@ from typing_extensions import ParamSpec +from zarr.constants import IS_WASM from zarr.core.config import config if TYPE_CHECKING: @@ -35,6 +36,38 @@ class SyncError(Exception): pass +# TODO: not sure if there is a better place to put this function, so I've kept it here for now. +def _make_shutdown_asyncgens_noop_for_pyodide() -> None: + """ + Patch Pyodide's WebLoop to fix interoperability with pytest-asyncio. + + WebLoop.shutdown_asyncgens() raises NotImplementedError, which causes + pytest-asyncio to issue warnings during test cleanup and potentially + cause resource leaks and this makes tests hang. This is a bit of a + hack, but it allows us to run tests that use pytest-asyncio. + """ + try: + if not IS_WASM and "pyodide" not in sys.modules: + return + + import pyodide.webloop + + if hasattr(pyodide.webloop.WebLoop, 'shutdown_asyncgens'): + async def no_op_shutdown_asyncgens(self) -> None: # noqa: ANN001 + return + + pyodide.webloop.WebLoop.shutdown_asyncgens = no_op_shutdown_asyncgens + logger.debug("Patched WebLoop.shutdown_asyncgens for pytest-asyncio compatibility") + + # If patching fails for any reason, we log it, but we won't want to crash Zarr + except Exception as e: + msg = f"Could not patch WebLoop for pytest compatibility: {e}" + logger.debug(msg) + + +if IS_WASM: + _make_shutdown_asyncgens_noop_for_pyodide() + def _get_lock() -> threading.Lock: """Allocate or return a threading lock. @@ -170,6 +203,13 @@ def _get_loop() -> asyncio.AbstractEventLoop: The loop will be running on a separate thread. """ + # In WASM environments, we can't create threads, so raise an error + if IS_WASM: + raise RuntimeError( + "Thread-based event loop not available in WASM environment. " + "Use zarr.api.asynchronous or ensure sync() handles WASM case." + ) + if loop[0] is None: with _get_lock(): # repeat the check just in case the loop got filled between the diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index e2d90bcc57..afc15d742c 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -50,10 +50,3 @@ def gpu_test(func: T) -> T: ) ), ) - - -def is_wasm() -> bool: - import platform - import sys - - return sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/tests/test_array.py b/tests/test_array.py index ddbaddd7e8..a2cb8e2f85 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -27,6 +27,7 @@ VLenUTF8Codec, ZstdCodec, ) +from zarr.constants import IS_WASM from zarr.core._info import ArrayInfo from zarr.core.array import ( CompressorsLike, @@ -49,7 +50,6 @@ from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.storage import LocalStore, MemoryStore, StorePath -from zarr.testing.utils import is_wasm if TYPE_CHECKING: from zarr.core.array_spec import ArrayConfigLike @@ -1679,8 +1679,8 @@ def _index_array(arr: Array, index: Any) -> Any: @pytest.mark.skipif( - is_wasm(), - reason = "can't start new processes in Pyodide", + IS_WASM, + reason="can't start new processes in Pyodide", ) @pytest.mark.parametrize( "method", diff --git a/tests/test_config.py b/tests/test_config.py index fa7acb7dcf..06f68f813d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,6 +19,7 @@ GzipCodec, ShardingCodec, ) +from zarr.constants import IS_WASM from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer @@ -43,7 +44,6 @@ TestBuffer, TestNDArrayLike, ) -from zarr.testing.utils import is_wasm def test_config_defaults_set() -> None: @@ -83,8 +83,8 @@ def test_config_defaults_set() -> None: ], }, }, - "async": {"concurrency": 1 if is_wasm() else 10, "timeout": None}, - "threading": {"max_workers": 1 if is_wasm() else None}, + "async": {"concurrency": 1 if IS_WASM else 10, "timeout": None}, + "threading": {"max_workers": 1 if IS_WASM else None}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", diff --git a/tests/test_group.py b/tests/test_group.py index 8e20593f04..378ac2cb0b 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -19,6 +19,7 @@ import zarr.storage from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store +from zarr.constants import IS_WASM from zarr.core import sync_group from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype @@ -42,7 +43,6 @@ from zarr.storage._common import make_store_path from zarr.storage._utils import _join_paths, normalize_path from zarr.testing.store import LatencyStore -from zarr.testing.utils import is_wasm from .conftest import meta_from_array, parse_store @@ -1973,7 +1973,12 @@ async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) - @pytest.mark.parametrize( "store", [ - pytest.param("memory", marks=pytest.mark.skipif(is_wasm(), reason="performance is marginally worse for Pyodide/WASM")), + pytest.param( + "memory", + marks=pytest.mark.skipif( + IS_WASM, reason="performance is marginally worse for Pyodide/WASM" + ), + ), ], indirect=True, ) diff --git a/tests/test_properties.py b/tests/test_properties.py index 5188b5de71..680118d5bb 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -7,8 +7,8 @@ import pytest from numpy.testing import assert_array_equal +from zarr.constants import IS_WASM from zarr.core.buffer import default_buffer_prototype -from zarr.testing.utils import is_wasm pytest.importorskip("hypothesis") @@ -77,7 +77,7 @@ def deep_equal(a: Any, b: Any) -> bool: return a == b -@pytest.mark.xfail(is_wasm(), reason="Hypothesis deadline being exceeded in Pyodide/WASM") +@pytest.mark.xfail(IS_WASM, reason="Hypothesis deadline being exceeded in Pyodide/WASM") @given(data=st.data(), zarr_format=zarr_formats) def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) diff --git a/tests/test_sync.py b/tests/test_sync.py index 6519e06e97..ba17d01b31 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -5,6 +5,7 @@ import pytest import zarr +from zarr.constants import IS_WASM from zarr.core.sync import ( SyncError, SyncMixin, @@ -16,13 +17,13 @@ sync, ) from zarr.storage import MemoryStore -from zarr.testing.utils import is_wasm pytestmark = pytest.mark.skipif( - is_wasm(), - reason = "can't start new threads in Pyodide/WASM, so the synchronous API doesn't work", + IS_WASM, + reason="can't start new threads in Pyodide/WASM, so the synchronous API doesn't work", ) + @pytest.fixture(params=[True, False]) def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None: if request.param is True: From f426ed78e1154c7c107ec4d6048e66c5a70c85ac Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 01:52:40 +0530 Subject: [PATCH 40/61] Mark Blosc `test_typesize` as a known failure case --- tests/test_codecs/test_blosc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py index 6e6e9df383..3cc58f9d36 100644 --- a/tests/test_codecs/test_blosc.py +++ b/tests/test_codecs/test_blosc.py @@ -8,6 +8,7 @@ import zarr from zarr.abc.store import Store from zarr.codecs import BloscCodec +from zarr.constants import IS_WASM from zarr.core.buffer import default_buffer_prototype from zarr.storage import StorePath @@ -58,6 +59,7 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: assert blosc_configuration_json["shuffle"] == "shuffle" +@pytest.mark.xfail(IS_WASM, reason="Blosc size mismatch, known failure case for Pyodide/WASM") async def test_typesize() -> None: a = np.arange(1000000, dtype=np.uint64) codecs = [zarr.codecs.BytesCodec(), zarr.codecs.BloscCodec()] From 405d24706f072a5cb6de2e4a7446ef6a02377f51 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 01:55:03 +0530 Subject: [PATCH 41/61] Skip `async.concurrency` config override test case --- tests/test_config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 06f68f813d..4460be8311 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -115,7 +115,14 @@ def test_config_defaults_set() -> None: @pytest.mark.parametrize( ("key", "old_val", "new_val"), - [("array.order", "C", "F"), ("async.concurrency", 10, 20), ("json_indent", 2, 0)], + [ + ("array.order", "C", "F"), + pytest.param( + "async.concurrency", 10, 20, + marks=pytest.mark.skipif(IS_WASM, reason="Concurrency is fixed to 1 in WASM") + ), + ("json_indent", 2, 0), + ] ) def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: assert config.get(key) == old_val From e93073a42c240d0cf0629a3488d89e0c69623ded Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 02:17:56 +0530 Subject: [PATCH 42/61] Mark some indexing tests as flaky on WASM --- tests/test_properties.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_properties.py b/tests/test_properties.py index 680118d5bb..070bce77f3 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -104,6 +104,7 @@ def test_array_creates_implicit_groups(array): # this decorator removes timeout; not ideal but it should avoid intermittent CI failures +@pytest.mark.skipif(IS_WASM, reason="Unreliable test on Pyodide/WASM due to Hypothesis") @settings(deadline=None) @given(data=st.data()) def test_basic_indexing(data: st.DataObject) -> None: @@ -119,6 +120,7 @@ def test_basic_indexing(data: st.DataObject) -> None: assert_array_equal(nparray, zarray[:]) +@pytest.mark.skipif(IS_WASM, reason="Unreliable test on Pyodide/WASM due to Hypothesis") @given(data=st.data()) def test_oindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. @@ -140,6 +142,7 @@ def test_oindex(data: st.DataObject) -> None: assert_array_equal(nparray, zarray[:]) +@pytest.mark.skipif(IS_WASM, reason="Unreliable test on Pyodide/WASM due to Hypothesis") @given(data=st.data()) def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. From d86295313e507fe175169ab4eb9574b9674af91f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 02:18:05 +0530 Subject: [PATCH 43/61] Oops, fix a config test --- src/zarr/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index d5c21c1a42..54b4efa098 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -109,7 +109,7 @@ def enable_gpu(self) -> ConfigSet: }, }, "async": {"concurrency": 1 if IS_WASM else 10, "timeout": None}, - "threading": {"max_workers": None if IS_WASM else 1}, + "threading": {"max_workers": 1 if IS_WASM else None}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", From cbd1d4dfb930703131aaf5049423b5ac58fbfad7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 02:20:31 +0530 Subject: [PATCH 44/61] Fix another config test --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 4460be8311..99a9185ec8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -107,7 +107,7 @@ def test_config_defaults_set() -> None: } ] assert config.get("array.order") == "C" - assert config.get("async.concurrency") == 10 + assert config.get("async.concurrency") == 1 if IS_WASM else 10 assert config.get("async.timeout") is None assert config.get("codec_pipeline.batch_size") == 1 assert config.get("json_indent") == 2 From 3e8bdefb15f5404a323fc8c161fcf048ae990fda Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 27 May 2025 16:17:31 +0530 Subject: [PATCH 45/61] Hook into Pyodide WebLoop --- src/zarr/core/sync.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 98d7abace2..33ade629d3 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -4,13 +4,14 @@ import atexit import logging import os +import sys import threading from concurrent.futures import ThreadPoolExecutor, wait from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec -from zarr.constants import IS_WASM +from zarr._constants import IS_WASM from zarr.core.config import config if TYPE_CHECKING: @@ -36,6 +37,7 @@ class SyncError(Exception): pass + # TODO: not sure if there is a better place to put this function, so I've kept it here for now. def _make_shutdown_asyncgens_noop_for_pyodide() -> None: """ @@ -43,8 +45,13 @@ def _make_shutdown_asyncgens_noop_for_pyodide() -> None: WebLoop.shutdown_asyncgens() raises NotImplementedError, which causes pytest-asyncio to issue warnings during test cleanup and potentially - cause resource leaks and this makes tests hang. This is a bit of a + cause resource leaks that make tests hang. This is a bit of a hack, but it allows us to run tests that use pytest-asyncio. + + This is necessary because pytest-asyncio tries to clean up async generators + when tearing down test event loops, but Pyodide's WebLoop doesn't support + this as it integrates with the browser's event loo rather than managing + its own lifecycle. """ try: if not IS_WASM and "pyodide" not in sys.modules: @@ -52,8 +59,9 @@ def _make_shutdown_asyncgens_noop_for_pyodide() -> None: import pyodide.webloop - if hasattr(pyodide.webloop.WebLoop, 'shutdown_asyncgens'): - async def no_op_shutdown_asyncgens(self) -> None: # noqa: ANN001 + if hasattr(pyodide.webloop.WebLoop, "shutdown_asyncgens"): + + async def no_op_shutdown_asyncgens(self) -> None: # type: ignore[no-untyped-def] # noqa: ANN001 return pyodide.webloop.WebLoop.shutdown_asyncgens = no_op_shutdown_asyncgens @@ -166,6 +174,17 @@ def sync( -------- >>> sync(async_function(), existing_loop) """ + # WASM environments (like Pyodide) cannot start new threads, so we need to handle + # coroutines differently. We integrate with the existing Pyodide WebLoop which + # schedules tasks on the browser's event loop using setTimeout(): + # https://developer.mozilla.org/en-US/docs/Web/API/setTimeout + if IS_WASM: + current_loop = asyncio.get_running_loop() + return current_loop.run_until_complete(coro) + + # This code path is the original thread-based implementation + # for non-WASM environments; it creates a dedicated I/O thread + # with its own event loop. if loop is None: # NB: if the loop is not running *yet*, it is OK to submit work # and we will wait for it @@ -203,7 +222,6 @@ def _get_loop() -> asyncio.AbstractEventLoop: The loop will be running on a separate thread. """ - # In WASM environments, we can't create threads, so raise an error if IS_WASM: raise RuntimeError( "Thread-based event loop not available in WASM environment. " From 4ee492ee4f95ad0716b3c03641fabcf00054b039 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 04:12:13 +0530 Subject: [PATCH 46/61] Move `zarr.constants` to `zarr._constants` --- src/zarr/_constants.py | 9 +++++++++ src/zarr/codecs/zstd.py | 14 ++++++-------- src/zarr/constants.py | 4 ---- src/zarr/core/config.py | 2 +- tests/test_array.py | 2 +- tests/test_codecs/test_blosc.py | 2 +- tests/test_config.py | 10 ++++++---- tests/test_group.py | 2 +- tests/test_properties.py | 2 +- tests/test_sync.py | 2 +- 10 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 src/zarr/_constants.py delete mode 100644 src/zarr/constants.py diff --git a/src/zarr/_constants.py b/src/zarr/_constants.py new file mode 100644 index 0000000000..63ca615e99 --- /dev/null +++ b/src/zarr/_constants.py @@ -0,0 +1,9 @@ +# This file only exists to not incur circular import issues +# TODO: find a better location for this or keep it here + +from __future__ import annotations + +import platform +import sys + +IS_WASM: bool = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index b4a4a13c29..f8b032bd48 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -5,9 +5,7 @@ from functools import cached_property from typing import TYPE_CHECKING -import numcodecs from numcodecs.zstd import Zstd -from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper @@ -44,12 +42,12 @@ class ZstdCodec(BytesBytesCodec): def __init__(self, *, level: int = 0, checksum: bool = False) -> None: # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec - _numcodecs_version = Version(numcodecs.__version__) - if _numcodecs_version < Version("0.13.0"): - raise RuntimeError( - "numcodecs version >= 0.13.0 is required to use the zstd codec. " - f"Version {_numcodecs_version} is currently installed." - ) + # _numcodecs_version = Version(numcodecs.__version__) + # if _numcodecs_version < Version("0.13.0"): + # raise RuntimeError( + # "numcodecs version >= 0.13.0 is required to use the zstd codec. " + # f"Version {_numcodecs_version} is currently installed." + # ) level_parsed = parse_zstd_level(level) checksum_parsed = parse_checksum(checksum) diff --git a/src/zarr/constants.py b/src/zarr/constants.py deleted file mode 100644 index 27f20caab7..0000000000 --- a/src/zarr/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -import platform -import sys - -IS_WASM: bool = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 54b4efa098..e01819feb3 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -33,7 +33,7 @@ from donfig import Config as DConfig -from zarr.constants import IS_WASM +from zarr._constants import IS_WASM if TYPE_CHECKING: from donfig.config_obj import ConfigSet diff --git a/tests/test_array.py b/tests/test_array.py index a2cb8e2f85..c6e696d154 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -18,6 +18,7 @@ import zarr.api.asynchronous import zarr.api.synchronous as sync_api from zarr import Array, AsyncArray, Group +from zarr._constants import IS_WASM from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, @@ -27,7 +28,6 @@ VLenUTF8Codec, ZstdCodec, ) -from zarr.constants import IS_WASM from zarr.core._info import ArrayInfo from zarr.core.array import ( CompressorsLike, diff --git a/tests/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py index 3cc58f9d36..4140d9154f 100644 --- a/tests/test_codecs/test_blosc.py +++ b/tests/test_codecs/test_blosc.py @@ -6,9 +6,9 @@ from packaging.version import Version import zarr +from zarr._constants import IS_WASM from zarr.abc.store import Store from zarr.codecs import BloscCodec -from zarr.constants import IS_WASM from zarr.core.buffer import default_buffer_prototype from zarr.storage import StorePath diff --git a/tests/test_config.py b/tests/test_config.py index 99a9185ec8..ebfc13b781 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -10,6 +10,7 @@ import zarr import zarr.api from zarr import zeros +from zarr._constants import IS_WASM from zarr.abc.codec import CodecPipeline from zarr.abc.store import ByteSetter, Store from zarr.codecs import ( @@ -19,7 +20,6 @@ GzipCodec, ShardingCodec, ) -from zarr.constants import IS_WASM from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer @@ -118,11 +118,13 @@ def test_config_defaults_set() -> None: [ ("array.order", "C", "F"), pytest.param( - "async.concurrency", 10, 20, - marks=pytest.mark.skipif(IS_WASM, reason="Concurrency is fixed to 1 in WASM") + "async.concurrency", + 10, + 20, + marks=pytest.mark.skipif(IS_WASM, reason="Concurrency is fixed to 1 in WASM"), ), ("json_indent", 2, 0), - ] + ], ) def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: assert config.get(key) == old_val diff --git a/tests/test_group.py b/tests/test_group.py index 378ac2cb0b..794ea1b33f 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -18,8 +18,8 @@ import zarr.api.synchronous import zarr.storage from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr._constants import IS_WASM from zarr.abc.store import Store -from zarr.constants import IS_WASM from zarr.core import sync_group from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype diff --git a/tests/test_properties.py b/tests/test_properties.py index 070bce77f3..83ea82d5e4 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -7,7 +7,7 @@ import pytest from numpy.testing import assert_array_equal -from zarr.constants import IS_WASM +from zarr._constants import IS_WASM from zarr.core.buffer import default_buffer_prototype pytest.importorskip("hypothesis") diff --git a/tests/test_sync.py b/tests/test_sync.py index ba17d01b31..84824839b5 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -5,7 +5,7 @@ import pytest import zarr -from zarr.constants import IS_WASM +from zarr._constants import IS_WASM from zarr.core.sync import ( SyncError, SyncMixin, From d94970e6fca7bddd4d06c36d5c2168a23729f771 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 04:17:02 +0530 Subject: [PATCH 47/61] Bump to Pyodide 0.28.0a3 --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index d15d9d698b..3d74723b6c 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -9,7 +9,7 @@ on: env: FORCE_COLOR: 3 - PYODIDE_VERSION: 0.28.0a2 + PYODIDE_VERSION: 0.28.0a3 # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: From cd3424ce79a05d84a761390859a47c605d35d68f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 04:20:42 +0530 Subject: [PATCH 48/61] Fix typo --- src/zarr/core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 33ade629d3..96ca6c43b5 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -50,7 +50,7 @@ def _make_shutdown_asyncgens_noop_for_pyodide() -> None: This is necessary because pytest-asyncio tries to clean up async generators when tearing down test event loops, but Pyodide's WebLoop doesn't support - this as it integrates with the browser's event loo rather than managing + this as it integrates with the browser's event loop rather than managing its own lifecycle. """ try: From 5044a2270bac2255a2e1196a4c46938c13665e53 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:30:43 +0530 Subject: [PATCH 49/61] `asyncio_mode = "auto"` works now, clean it up --- .github/workflows/emscripten.yml | 3 --- pyproject.toml | 1 - 2 files changed, 4 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 3d74723b6c..c7cd39fafe 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -89,9 +89,6 @@ jobs: pyodide venv .venv-pyodide source .venv-pyodide/bin/activate - # Avoid missing asyncio plugin error from pytest, unavailable in Pyodide - if grep -q 'asyncio_mode = "auto"' "pyproject.toml"; then sed '/asyncio_mode = "auto"/d' "pyproject.toml" > temp && mv temp "pyproject.toml"; fi - # Install numcodecs python -m pip install numcodecs-wasm/dist/*.whl python -m pip install dist/*.whl diff --git a/pyproject.toml b/pyproject.toml index 6ea9fba117..a43e51abd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -391,7 +391,6 @@ minversion = "7" testpaths = ["tests", "docs/user-guide"] log_cli_level = "INFO" xfail_strict = true -# Doesn't work under WASM, remove when running Pyodide test suite asyncio_mode = "auto" doctest_optionflags = [ "NORMALIZE_WHITESPACE", From e131867f49f8080fc192a7815dfef13794d71727 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:32:48 +0530 Subject: [PATCH 50/61] Restore `test_group_members_performance` --- tests/test_group.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/test_group.py b/tests/test_group.py index 794ea1b33f..bb67ed3938 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1970,18 +1970,7 @@ async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) - raise ValueError(f"Invalid impl: {impl}") -@pytest.mark.parametrize( - "store", - [ - pytest.param( - "memory", - marks=pytest.mark.skipif( - IS_WASM, reason="performance is marginally worse for Pyodide/WASM" - ), - ), - ], - indirect=True, -) +@pytest.mark.parametrize("store", ["memory"], indirect=True) def test_group_members_performance(store: Store) -> None: """ Test that the execution time of Group.members is less than the number of members times the @@ -1989,6 +1978,12 @@ def test_group_members_performance(store: Store) -> None: """ get_latency = 0.1 + # Performance for Pyodide is only marginally worse than for Python, + # usually in the 1.012~1.018 range for a latency of 0.1). So we add + # some lenience for the WASM case that should be sufficient for + # unpredictable ephemeral environments like CI. + lenience = 1.10 if IS_WASM else 1.00 + # use the input store to create some groups group_create = zarr.group(store=store) num_groups = 10 @@ -2009,7 +2004,7 @@ def test_group_members_performance(store: Store) -> None: _ = group_read.members() elapsed = time.time() - start - assert elapsed < (num_groups * get_latency) + assert elapsed < (num_groups * get_latency) * lenience @pytest.mark.parametrize("store", ["memory"], indirect=True) From c7c22dc5a7b7c9cefd974ce218014d9ed33facc1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:33:53 +0530 Subject: [PATCH 51/61] Disable SIMD when building numcodecs --- .github/workflows/emscripten.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index c7cd39fafe..e838e8d7f0 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -23,7 +23,7 @@ concurrency: cancel-in-progress: true permissions: - contents: read # to fetch code (actions/checkout) + contents: read jobs: build_wasm_emscripten: @@ -76,6 +76,9 @@ jobs: submodules: recursive persist-credentials: false fetch-depth: 0 + env: + DISABLE_NUMCODECS_AVX2: 1 + DISABLE_NUMCODECS_SSE2: 1 - name: Build numcodecs for WASM run: pyodide build From e62c9338904c0e466a7aa3b05d561dcb44665ea8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:39:24 +0530 Subject: [PATCH 52/61] Oops, disable AVX2, SSE2 at the right place --- .github/workflows/emscripten.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index e838e8d7f0..9a293827a5 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -76,13 +76,13 @@ jobs: submodules: recursive persist-credentials: false fetch-depth: 0 - env: - DISABLE_NUMCODECS_AVX2: 1 - DISABLE_NUMCODECS_SSE2: 1 - name: Build numcodecs for WASM run: pyodide build working-directory: numcodecs-wasm + env: + DISABLE_NUMCODECS_AVX2: 1 + DISABLE_NUMCODECS_SSE2: 1 ### Back to Zarr repository to run tests From d3bcf56f828893139bd7ce950ded8edbaf2d71c9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:48:07 +0530 Subject: [PATCH 53/61] Debug improper numcodecs version --- .github/workflows/emscripten.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 9a293827a5..d1cd711820 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -37,6 +37,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + fetch-tags: true persist-credentials: false - name: Set up Python ${{ env.PYTHON_VERSION }} @@ -76,6 +77,7 @@ jobs: submodules: recursive persist-credentials: false fetch-depth: 0 + fetch-tags: true - name: Build numcodecs for WASM run: pyodide build From e4b73796d4cffbf243adeed67f728179ba4205da Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 05:54:13 +0530 Subject: [PATCH 54/61] Debug numcodecs version again --- .github/workflows/emscripten.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index d1cd711820..82d0406a0a 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -38,7 +38,6 @@ jobs: with: fetch-depth: 0 fetch-tags: true - persist-credentials: false - name: Set up Python ${{ env.PYTHON_VERSION }} id: setup-python @@ -75,7 +74,6 @@ jobs: ref: setup-emscripten-ci path: numcodecs-wasm submodules: recursive - persist-credentials: false fetch-depth: 0 fetch-tags: true From 35cecc181fd1e1bcfa707453cba88de27189f756 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:03:55 +0530 Subject: [PATCH 55/61] Fetch tags manually for now --- .github/workflows/emscripten.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 82d0406a0a..7dc2510df8 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -77,6 +77,11 @@ jobs: fetch-depth: 0 fetch-tags: true + # For some reason fetch-depth: 0 and fetch-tags: true aren't working... + - name: Manually fetch tags for numcodecs + working-directory: numcodecs-wasm + run: git fetch --tags + - name: Build numcodecs for WASM run: pyodide build working-directory: numcodecs-wasm From ceb70c7ee3f3911e147163739756de2187d9cfbc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:16:28 +0530 Subject: [PATCH 56/61] Force Zarr to install --- .github/workflows/emscripten.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 7dc2510df8..21018a6fcc 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -99,13 +99,12 @@ jobs: # Install numcodecs python -m pip install numcodecs-wasm/dist/*.whl - python -m pip install dist/*.whl - python -m pip install pytest pytest-cov - python -m pytest -v --cov=zarr --cov-config=pyproject.toml - - name: Upload Pyodide wheel artifact for debugging - # FIXME: Remove after this is ready to merge - uses: actions/upload-artifact@v4 - with: - name: zarr-pyodide-wheel - path: dist/*.whl + # Install Zarr without dependencies until we can figure out the + # numcodecs wheel versioning issue + python -m pip install dist/*.whl --no-deps + # pip install $(ls dist/*.whl)"[test]" + pip install coverage pytest pytest-asyncio pytest-cov pytest-accept rich mypy hypothesis + + python -m pytest tests -v --cov=zarr --cov-config=pyproject.toml + From 1ea992f9a20239c91c467f24a06f93373ba2a976 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:27:25 +0530 Subject: [PATCH 57/61] Install `numcodecs` with `crc32c` --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 21018a6fcc..fc8e8ae0c3 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -98,7 +98,7 @@ jobs: source .venv-pyodide/bin/activate # Install numcodecs - python -m pip install numcodecs-wasm/dist/*.whl + python -m pip install numcodecs-wasm/dist/*.whl"[crc32c]" # Install Zarr without dependencies until we can figure out the # numcodecs wheel versioning issue From f51ddd10044cdf7935dcd1d3c3991b4f7f0e01c7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:27:42 +0530 Subject: [PATCH 58/61] Install the rest of the missing dependencies --- .github/workflows/emscripten.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index fc8e8ae0c3..de7e26ff09 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -103,8 +103,11 @@ jobs: # Install Zarr without dependencies until we can figure out the # numcodecs wheel versioning issue python -m pip install dist/*.whl --no-deps + pip install "packaging>=22.0" "numpy>=1.25" "typing_extensions>=4.9" "donfig>=0.8" + + # Install test dependencies # pip install $(ls dist/*.whl)"[test]" - pip install coverage pytest pytest-asyncio pytest-cov pytest-accept rich mypy hypothesis + pip install "coverage" "pytest" "pytest-asyncio" 'pytest-cov" "pytest-accept" "rich" "mypy" "hypothesis" python -m pytest tests -v --cov=zarr --cov-config=pyproject.toml From 0ec47b994771155e30eb2adfa47a4936c237238d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:35:35 +0530 Subject: [PATCH 59/61] Escape wheel filename correctly --- .github/workflows/emscripten.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index de7e26ff09..2e9dc8fafa 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -98,11 +98,11 @@ jobs: source .venv-pyodide/bin/activate # Install numcodecs - python -m pip install numcodecs-wasm/dist/*.whl"[crc32c]" + pip install $(ls numcodecs-wasm/dist/*.whl)"[crc32c]" # Install Zarr without dependencies until we can figure out the # numcodecs wheel versioning issue - python -m pip install dist/*.whl --no-deps + pip install dist/*.whl --no-deps pip install "packaging>=22.0" "numpy>=1.25" "typing_extensions>=4.9" "donfig>=0.8" # Install test dependencies From e1617c0edf0a27ef28c2554d9968fabe9d9bd4b3 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 06:48:59 +0530 Subject: [PATCH 60/61] Remove extra install line --- .github/workflows/emscripten.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 2e9dc8fafa..4c04f5470b 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -106,7 +106,6 @@ jobs: pip install "packaging>=22.0" "numpy>=1.25" "typing_extensions>=4.9" "donfig>=0.8" # Install test dependencies - # pip install $(ls dist/*.whl)"[test]" pip install "coverage" "pytest" "pytest-asyncio" 'pytest-cov" "pytest-accept" "rich" "mypy" "hypothesis" python -m pytest tests -v --cov=zarr --cov-config=pyproject.toml From 23e34bff9596a975f6af128548de9e37ffce25be Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 28 May 2025 07:06:54 +0530 Subject: [PATCH 61/61] Fix misquoted end --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 4c04f5470b..42620cef1e 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -106,7 +106,7 @@ jobs: pip install "packaging>=22.0" "numpy>=1.25" "typing_extensions>=4.9" "donfig>=0.8" # Install test dependencies - pip install "coverage" "pytest" "pytest-asyncio" 'pytest-cov" "pytest-accept" "rich" "mypy" "hypothesis" + pip install "coverage" "pytest" "pytest-asyncio" "pytest-cov" "pytest-accept" "rich" "mypy" "hypothesis" python -m pytest tests -v --cov=zarr --cov-config=pyproject.toml