From a513b4941038f38886c4b0103625ce0170e9d190 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 26 Jul 2023 12:13:51 +0200 Subject: [PATCH 1/9] Added support of python 3.11 --- .github/workflows/build-sphinx.yml | 2 +- .github/workflows/conda-package.yml | 8 ++++---- .github/workflows/generate_coverage.yaml | 2 +- .github/workflows/pre-commit.yml | 2 +- setup.py | 1 + 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-sphinx.yml b/.github/workflows/build-sphinx.yml index e1ad17217cc8..4a33a47bd371 100644 --- a/.github/workflows/build-sphinx.yml +++ b/.github/workflows/build-sphinx.yml @@ -16,7 +16,7 @@ jobs: shell: bash -l {0} env: - python-ver: '3.9' + python-ver: '3.11' CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' steps: diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 2f0956f8cc4a..41b3dc913f14 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: - python: ['3.8', '3.9', '3.10'] + python: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-20.04, windows-latest] runs-on: ${{ matrix.os }} @@ -114,7 +114,7 @@ jobs: strategy: matrix: - python: ['3.8', '3.9', '3.10'] + python: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-20.04, ubuntu-latest] experimental: [false] @@ -215,7 +215,7 @@ jobs: strategy: matrix: - python: ['3.8', '3.9', '3.10'] + python: ['3.8', '3.9', '3.10', '3.11'] experimental: [false] continue-on-error: ${{ matrix.experimental }} @@ -346,7 +346,7 @@ jobs: strategy: matrix: - python: ['3.8', '3.9', '3.10'] + python: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-20.04, windows-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index 413835d336cf..d3a217a03f81 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -14,7 +14,7 @@ jobs: shell: bash -l {0} env: - python-ver: '3.10' + python-ver: '3.11' CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' steps: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 673f6c172430..18668bf10c45 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -19,5 +19,5 @@ jobs: - uses: actions/checkout@v3.5.2 - uses: actions/setup-python@v4.6.1 with: - python-version: '3.10' + python-version: '3.11' - uses: pre-commit/action@v3.0.0 diff --git a/setup.py b/setup.py index 9b56902f3592..b2e1dd24ad62 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 +Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython Topic :: Software Development Topic :: Scientific/Engineering From fedffe21d22c248bff2b992a7216cfaf6aae1596 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 26 Jul 2023 14:15:41 +0200 Subject: [PATCH 2/9] Pinned python back to 3.9 in build sphinx --- .github/workflows/build-sphinx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sphinx.yml b/.github/workflows/build-sphinx.yml index 4a33a47bd371..e1ad17217cc8 100644 --- a/.github/workflows/build-sphinx.yml +++ b/.github/workflows/build-sphinx.yml @@ -16,7 +16,7 @@ jobs: shell: bash -l {0} env: - python-ver: '3.11' + python-ver: '3.9' CHANNELS: '-c dppy/label/dev -c intel -c conda-forge --override-channels' steps: From 24e4eb548bcd8f1a74e0d357fe17f5c25b11c983 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 26 Jul 2023 17:12:07 +0200 Subject: [PATCH 3/9] The change to help icpx to find the libraries in --- conda-recipe/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index b25baed8ecd1..015a25bbbbd2 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${BUILD_PREFIX}/lib" + # Intel LLVM must cooperate with compiler and sysroot from conda echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icpx_for_conda.cfg export ICPXCFG="$(pwd)/icpx_for_conda.cfg" From 0a0827247b1db67aa4a2ccdbd79e2f12c880a520 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 1 Aug 2023 14:08:17 -0500 Subject: [PATCH 4/9] Updated FFT tests to work with numpy from conda-forge --- tests/helper.py | 22 ++++++++++++++++++++++ tests/test_fft.py | 16 +++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/tests/helper.py b/tests/helper.py index 6971feecb534..42ada2949833 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -1,10 +1,32 @@ from sys import platform import dpctl +import numpy +from numpy.testing import assert_allclose, assert_array_equal import dpnp +def assert_dtype_allclose(dpnp_arr, numpy_arr): + """ + Assert DPNP and NumPy array based on maximum dtype resolution of input arrays + for floating and complex types. + For other dtypes the assertion is based on exact matching of the arrays. + + """ + + is_inexact = lambda x: dpnp.issubdtype(x.dtype, dpnp.inexact) + if is_inexact(dpnp_arr) or is_inexact(numpy_arr): + tol = 8 * max( + numpy.finfo(dpnp_arr.dtype).resolution, + numpy.finfo(numpy_arr.dtype).resolution, + ) + assert_allclose(dpnp_arr.asnumpy(), numpy_arr, atol=tol, rtol=tol) + else: + assert_array_equal(dpnp_arr.asnumpy(), numpy_arr) + assert dpnp_arr.dtype == numpy_arr.dtype + + def get_complex_dtypes(device=None): """ Build a list of complex types supported by DPNP based on device capabilities. diff --git a/tests/test_fft.py b/tests/test_fft.py index 2be8e0880c3a..7cf72cefd39b 100644 --- a/tests/test_fft.py +++ b/tests/test_fft.py @@ -1,6 +1,8 @@ import numpy import pytest +from .helper import assert_dtype_allclose + import dpnp @@ -14,10 +16,9 @@ def test_fft(type, norm): dpnp_data = dpnp.array(data) np_res = numpy.fft.fft(data, norm=norm) - dpnp_res = dpnp.asnumpy(dpnp.fft.fft(dpnp_data, norm=norm)) + dpnp_res = dpnp.fft.fft(dpnp_data, norm=norm) - numpy.testing.assert_allclose(dpnp_res, np_res, rtol=1e-4, atol=1e-7) - assert dpnp_res.dtype == np_res.dtype + assert_dtype_allclose(dpnp_res, np_res) @pytest.mark.parametrize( @@ -32,8 +33,7 @@ def test_fft_ndim(type, shape, norm): np_res = numpy.fft.fft(np_data, norm=norm) dpnp_res = dpnp.fft.fft(dpnp_data, norm=norm) - numpy.testing.assert_allclose(dpnp_res, np_res, rtol=1e-4, atol=1e-7) - assert dpnp_res.dtype == np_res.dtype + assert_dtype_allclose(dpnp_res, np_res) @pytest.mark.parametrize( @@ -50,8 +50,7 @@ def test_fft_ifft(type, shape, norm): np_res = numpy.fft.ifft(np_data, norm=norm) dpnp_res = dpnp.fft.ifft(dpnp_data, norm=norm) - numpy.testing.assert_allclose(dpnp_res, np_res, rtol=1e-4, atol=1e-7) - assert dpnp_res.dtype == np_res.dtype + assert_dtype_allclose(dpnp_res, np_res) @pytest.mark.parametrize("type", ["float32", "float64", "int32", "int64"]) @@ -65,5 +64,4 @@ def test_fft_rfft(type, shape): np_res = numpy.fft.rfft(np_data) dpnp_res = dpnp.fft.rfft(dpnp_data) - numpy.testing.assert_allclose(dpnp_res, np_res, rtol=1e-4, atol=1e-7) - assert dpnp_res.dtype == np_res.dtype + assert_dtype_allclose(dpnp_res, np_res) From 5042d1c4f6862e4461ec7511d9f7569248625ca3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 2 Aug 2023 07:58:38 -0500 Subject: [PATCH 5/9] Update tan test to use assert_allclose --- tests/helper.py | 3 ++- tests/test_umath.py | 2 +- tests/third_party/cupy/sorting_tests/test_sort.py | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/helper.py b/tests/helper.py index 42ada2949833..8c315debc1b7 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -2,10 +2,11 @@ import dpctl import numpy -from numpy.testing import assert_allclose, assert_array_equal import dpnp +from numpy.testing import assert_allclose, assert_array_equal + def assert_dtype_allclose(dpnp_arr, numpy_arr): """ diff --git a/tests/test_umath.py b/tests/test_umath.py index 7d8bb4acfe55..96ff2a849c88 100644 --- a/tests/test_umath.py +++ b/tests/test_umath.py @@ -480,7 +480,7 @@ def test_tan(self): np_array = numpy.array(array_data, dtype=numpy.float64) expected = numpy.tan(np_array, out=out) - assert_array_equal(expected, result) + assert_allclose(expected, result) @pytest.mark.parametrize( "dtype", diff --git a/tests/third_party/cupy/sorting_tests/test_sort.py b/tests/third_party/cupy/sorting_tests/test_sort.py index d9fb94fc3e02..21e0418a65ec 100644 --- a/tests/third_party/cupy/sorting_tests/test_sort.py +++ b/tests/third_party/cupy/sorting_tests/test_sort.py @@ -317,7 +317,11 @@ def test_argsort_zero_dim(self, xp, dtype): @testing.numpy_cupy_array_equal() def test_argsort_one_dim(self, xp, dtype): a = testing.shaped_random((10,), xp, dtype) - return self.argsort(a) + res = self.argsort(a) + print() + print(xp) + print(a, res) + return res @testing.for_all_dtypes() @testing.numpy_cupy_array_equal() From 262460b1d28ca1288fca1ff7941a51fb44607b21 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 3 Aug 2023 07:04:06 -0500 Subject: [PATCH 6/9] Updated remaing FFT tests --- tests/helper.py | 3 +-- tests/test_fft.py | 4 +--- tests/test_sycl_queue.py | 5 ++--- tests/third_party/cupy/fft_tests/test_fft.py | 16 +++++++++++++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/helper.py b/tests/helper.py index 8c315debc1b7..42ada2949833 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -2,11 +2,10 @@ import dpctl import numpy +from numpy.testing import assert_allclose, assert_array_equal import dpnp -from numpy.testing import assert_allclose, assert_array_equal - def assert_dtype_allclose(dpnp_arr, numpy_arr): """ diff --git a/tests/test_fft.py b/tests/test_fft.py index dc282e35d0ac..ae6835b86eda 100644 --- a/tests/test_fft.py +++ b/tests/test_fft.py @@ -1,11 +1,9 @@ import numpy import pytest -from .helper import assert_dtype_allclose - import dpnp -from .helper import has_support_aspect64 +from .helper import assert_dtype_allclose, has_support_aspect64 pytestmark = pytest.mark.skipif( not has_support_aspect64(), reason="Aborted on Iris Xe: SAT-6028" diff --git a/tests/test_sycl_queue.py b/tests/test_sycl_queue.py index 0de3a1b0ba9a..7dc15a6bccfb 100644 --- a/tests/test_sycl_queue.py +++ b/tests/test_sycl_queue.py @@ -6,7 +6,7 @@ import dpnp -from .helper import get_all_dtypes, is_win_platform +from .helper import assert_dtype_allclose, get_all_dtypes, is_win_platform list_of_backend_str = [ "host", @@ -670,8 +670,7 @@ def test_fft_rfft(type, shape, device): np_res = numpy.fft.rfft(np_data) dpnp_res = dpnp.fft.rfft(dpnp_data) - numpy.testing.assert_allclose(dpnp_res, np_res, rtol=1e-4, atol=1e-7) - assert dpnp_res.dtype == np_res.dtype + assert_dtype_allclose(dpnp_res, np_res) expected_queue = dpnp_data.get_array().sycl_queue result_queue = dpnp_res.get_array().sycl_queue diff --git a/tests/third_party/cupy/fft_tests/test_fft.py b/tests/third_party/cupy/fft_tests/test_fft.py index 9214a8515e50..0ddd3cc56c17 100644 --- a/tests/third_party/cupy/fft_tests/test_fft.py +++ b/tests/third_party/cupy/fft_tests/test_fft.py @@ -23,7 +23,11 @@ class TestFft(unittest.TestCase): @testing.for_all_dtypes() @testing.numpy_cupy_allclose( - rtol=1e-4, atol=1e-7, accept_error=ValueError, contiguous_check=False + rtol=1e-4, + atol=1e-7, + accept_error=ValueError, + contiguous_check=False, + type_check=False, ) def test_fft(self, xp, dtype): a = testing.shaped_random(self.shape, xp, dtype) @@ -33,7 +37,11 @@ def test_fft(self, xp, dtype): @testing.for_all_dtypes() @testing.numpy_cupy_allclose( - rtol=1e-4, atol=1e-7, accept_error=ValueError, contiguous_check=False + rtol=1e-4, + atol=1e-7, + accept_error=ValueError, + contiguous_check=False, + type_check=False, ) def test_ifft(self, xp, dtype): a = testing.shaped_random(self.shape, xp, dtype) @@ -154,7 +162,9 @@ def test_ifftn(self, xp, dtype): @testing.gpu class TestRfft(unittest.TestCase): @testing.for_all_dtypes(no_complex=True) - @testing.numpy_cupy_allclose(rtol=1e-4, atol=1e-7, contiguous_check=False) + @testing.numpy_cupy_allclose( + rtol=1e-4, atol=1e-7, contiguous_check=False, type_check=False + ) def test_rfft(self, xp, dtype): a = testing.shaped_random(self.shape, xp, dtype) out = xp.fft.rfft(a, n=self.n, norm=self.norm) From 31acc30a80fdd3c1f65aeac29fb63dd1e26ffeb8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 3 Aug 2023 09:01:45 -0500 Subject: [PATCH 7/9] Updated test with dpnp.dot() --- .github/workflows/conda-package.yml | 4 ++-- tests/test_dot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 41b3dc913f14..7227a704a84d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -199,7 +199,7 @@ jobs: # TODO: run the whole scope once the issues on CPU are resolved - name: Run tests run: | - python -m pytest -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} + python -m pytest -s -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} working-directory: ${{ env.tests-path }} test_windows: @@ -336,7 +336,7 @@ jobs: # TODO: run the whole scope once the issues on CPU are resolved - name: Run tests run: | - python -m pytest -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} + python -m pytest -s -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} working-directory: ${{ env.tests-path }} upload: diff --git a/tests/test_dot.py b/tests/test_dot.py index 7b4cdd02c928..f27e3a58059d 100644 --- a/tests/test_dot.py +++ b/tests/test_dot.py @@ -4,7 +4,7 @@ import dpnp as inp -from .helper import get_all_dtypes +from .helper import assert_dtype_allclose, get_all_dtypes @pytest.mark.parametrize("type", get_all_dtypes(no_bool=True, no_complex=True)) @@ -31,7 +31,7 @@ def test_dot_arange(type): result = inp.dot(ia, ib) expected = numpy.dot(a, b) - assert_allclose(expected, result) + assert_dtype_allclose(result, expected) @pytest.mark.parametrize("type", get_all_dtypes(no_bool=True, no_complex=True)) From e6fb914df4e2f255e4e3d9476e8800706ccbcc26 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 3 Aug 2023 14:18:53 -0500 Subject: [PATCH 8/9] Updated argsort test --- .github/workflows/conda-package.yml | 4 ++-- tests/third_party/cupy/sorting_tests/test_sort.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 7227a704a84d..41b3dc913f14 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -199,7 +199,7 @@ jobs: # TODO: run the whole scope once the issues on CPU are resolved - name: Run tests run: | - python -m pytest -s -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} + python -m pytest -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} working-directory: ${{ env.tests-path }} test_windows: @@ -336,7 +336,7 @@ jobs: # TODO: run the whole scope once the issues on CPU are resolved - name: Run tests run: | - python -m pytest -s -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} + python -m pytest -q -ra --disable-warnings -vv ${{ env.TEST_SCOPE }} working-directory: ${{ env.tests-path }} upload: diff --git a/tests/third_party/cupy/sorting_tests/test_sort.py b/tests/third_party/cupy/sorting_tests/test_sort.py index 21e0418a65ec..7ae7911f90ee 100644 --- a/tests/third_party/cupy/sorting_tests/test_sort.py +++ b/tests/third_party/cupy/sorting_tests/test_sort.py @@ -318,10 +318,7 @@ def test_argsort_zero_dim(self, xp, dtype): def test_argsort_one_dim(self, xp, dtype): a = testing.shaped_random((10,), xp, dtype) res = self.argsort(a) - print() - print(xp) - print(a, res) - return res + return a[res] @testing.for_all_dtypes() @testing.numpy_cupy_array_equal() From c56675dff36f4c4cf53265bde38a5ad1f358cd86 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 4 Aug 2023 10:07:50 -0500 Subject: [PATCH 9/9] Reduced array size for single precision dtype in dot() test --- tests/test_dot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_dot.py b/tests/test_dot.py index f27e3a58059d..80da5090e1b7 100644 --- a/tests/test_dot.py +++ b/tests/test_dot.py @@ -4,7 +4,7 @@ import dpnp as inp -from .helper import assert_dtype_allclose, get_all_dtypes +from .helper import get_all_dtypes @pytest.mark.parametrize("type", get_all_dtypes(no_bool=True, no_complex=True)) @@ -20,18 +20,18 @@ def test_dot_ones(type): assert_array_equal(expected, result) -@pytest.mark.parametrize("type", get_all_dtypes(no_bool=True, no_complex=True)) -def test_dot_arange(type): +@pytest.mark.parametrize("dtype", get_all_dtypes(no_bool=True, no_complex=True)) +def test_dot_arange(dtype): n = 10**2 - m = 10**3 - a = numpy.hstack((numpy.arange(n, dtype=type),) * m) + m = 10**3 if dtype is not inp.float32 else 10**2 + a = numpy.hstack((numpy.arange(n, dtype=dtype),) * m) b = numpy.flipud(a) ia = inp.array(a) ib = inp.array(b) result = inp.dot(ia, ib) expected = numpy.dot(a, b) - assert_dtype_allclose(result, expected) + assert_allclose(expected, result) @pytest.mark.parametrize("type", get_all_dtypes(no_bool=True, no_complex=True))