Merge pull request #505 from pydata/andfoy-support_free_threading

This is an extension of #504 to fix some issues with tests.  The tests for aarch64 are currently commented out, because CI (via qemu) gets stalled.  However, my arm64 box (MacMini M4) does not have a single issue, so I suppose it is an issue with qemu.

This overrides #504.

@andfoy @rgommers

Author: FrancescAlted

.github/workflows/build.yml

@@ -6,37 +6,56 @@ permissions:
   contents: read
 
 env:
-  CIBW_BEFORE_BUILD: pip install setuptools oldest-supported-numpy
+  CIBW_BEFORE_BUILD: pip install setuptools oldest-supported-numpy pytest
+  CIBW_BEFORE_TEST: pip install pytest
   CIBW_BUILD_VERBOSITY: 1
-  CIBW_TEST_COMMAND: python -c "import sys, numexpr; sys.exit(0 if numexpr.test().wasSuccessful() else 1)"
-  CIBW_TEST_SKIP: "*macosx*arm64*"
+  CIBW_TEST_COMMAND: pytest --pyargs numexpr
+  # Testing on aarch64 takes too long, as it is currently emulated on GitHub Actions
+  CIBW_TEST_SKIP: "*linux*aarch64*"
   # Building for musllinux and aarch64 takes way too much time.
   # Moreover, NumPy is not providing musllinux for x86_64 either, so it's not worth it.
   CIBW_SKIP: "*musllinux*aarch64* *musllinux*x86_64*"
 
 jobs:
   build_wheels:
-    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} - ${{ matrix.p_ver }}
-    runs-on: ${{ matrix.os }}
+    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
+    runs-on: ${{ matrix.runs-on || matrix.os }}
     permissions:
       contents: write
     env:
-      CIBW_BUILD: ${{ matrix.cibw_build }}
+      CIBW_BUILD: ${{ matrix.cibw_pattern }}
       CIBW_ARCHS_LINUX: ${{ matrix.arch }}
       CIBW_ARCHS_MACOS: "x86_64 arm64"
+      CIBW_ENABLE: cpython-freethreading
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        arch: [x86_64, aarch64]
-        cibw_build: ["cp3{10,11,12,13}-*"]
-        p_ver: ["3.10-3.13"]
-        exclude:
-          - os: windows-latest
-            arch: aarch64
-          # cibuild is already in charge to build aarch64 (see CIBW_ARCHS_MACOS)
-          - os: macos-latest
+        include:
+          # Linux x86_64 builds
+          - os: ubuntu-latest
+            arch: x86_64
+            cibw_pattern: "cp3{10,11,12,13,13t}-manylinux*"
+            artifact_name: "linux-x86_64"
+
+          # Linux ARM64 builds (native runners)
+          - os: ubuntu-latest
             arch: aarch64
+            cibw_pattern: "cp3{10,11,12,13,13t}-manylinux*"
+            artifact_name: "linux-aarch64"
+            # Don't use native runners for now (looks like wait times are too long)
+            #runs-on: ["ubuntu-latest", "arm64"]
 
+          # Windows builds
+          - os: windows-latest
+            arch: x86_64
+            cibw_pattern: "cp3{10,11,12,13,13t}-win*"
+            artifact_name: "windows-x86_64"
+
+          # macOS builds (universal2)
+          - os: macos-latest
+            arch: x86_64
+            cibw_pattern: "cp3{10,11,12,13,13t}-macosx*"
+            artifact_name: "macos-universal2"
     steps:
       - uses: actions/checkout@v3
 
@@ -45,17 +64,22 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Install cibuildwheel
+      - name: Setup free-threading variables
+        if: ${{ endsWith(matrix.cibw_build, 't-*') }}
+        shell: bash -l {0}
         run: |
-          python -m pip install cibuildwheel
+          echo "CIBW_BEFORE_BUILD=pip install setuptools numpy" >> "$GITHUB_ENV"
+          echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV"
+          echo "CIBW_TEST_COMMAND=pytest --parallel-threads=4 --pyargs numexpr" >> "$GITHUB_ENV"
 
-      - uses: docker/setup-qemu-action@v2
-        if: ${{ matrix.arch == 'aarch64' }}
-        name: Set up QEMU
+      - name: Set up QEMU
+        if: matrix.arch == 'aarch64'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
 
       - name: Build wheels
-        run: |
-          python -m cibuildwheel --output-dir wheelhouse
+        uses: pypa/[email protected]
 
       - name: Make sdist
         if: ${{ matrix.os == 'windows-latest' }}
@@ -65,6 +89,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
+          name: ${{ matrix.artifact_name }}
           path: ./wheelhouse/*
 
       - name: Upload to GitHub Release

.gitignore

@@ -7,6 +7,7 @@ artifact/
 numexpr.egg-info/
 *.pyc
 *.swp
+*.so
 *~
 doc/_build
 site.cfg

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: debug-statements
+
+# Too many things to fix, let's just ignore it for now
+#-   repo: https://github.com/pycqa/flake8
+#    rev: 7.0.0
+#    hooks:
+#    -   id: flake8
+#
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+
+# Too many things to fix, let's just ignore it for now
+#-   repo: https://github.com/pre-commit/mirrors-mypy
+#    rev: v1.8.0
+#    hooks:
+#    -   id: mypy
+#        exclude: ^(docs/|setup.py)

.readthedocs.yaml

@@ -19,4 +19,4 @@ sphinx:
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
-  - requirements: doc/requirements.txt
+  - requirements: doc/requirements.txt

AUTHORS.txt

@@ -23,7 +23,7 @@ Google Inc. contributed bug fixes.
 
 David Cox improved readability of the Readme.
 
-Robert A. McLeod contributed bug fixes and ported the documentation to 
+Robert A. McLeod contributed bug fixes and ported the documentation to
 numexpr.readthedocs.io. He has served as the maintainer of the package
 since 2016 to 2023.
 

README.rst

@@ -159,6 +159,24 @@ Usage
   array([ True, False, False], dtype=bool)
 
 
+Free-threading support
+----------------------
+Starting on CPython 3.13 onwards there is a new distribution that disables the
+Global Interpreter Lock (GIL) altogether, thus increasing the performance yields
+under multi-threaded conditions on a single interpreter, as opposed to having to use
+multiprocessing.
+
+Whilst numexpr has been demonstrated to work under free-threaded
+CPython, considerations need to be taken when using numexpr native parallel
+implementation vs using Python threads directly in order to prevent oversubscription,
+we recommend either using the main CPython interpreter thread to spawn multiple C threads
+using the parallel numexpr API, or spawning multiple CPython threads that do not use
+the parallel API.
+
+For more information about free-threaded CPython, we recommend visiting the following
+`community Wiki <https://py-free-threading.github.io/>`
+
+
 Documentation
 -------------
 

bench/boolean_timing.py

@@ -9,8 +9,10 @@
 ####################################################################
 
 from __future__ import print_function
+
 import sys
 import timeit
+
 import numpy
 
 array_size = 5_000_000

bench/issue-36.py

@@ -2,10 +2,14 @@
 # performs better than the serial code.  See issue #36 for details.
 
 from __future__ import print_function
+
+from time import time
+
 import numpy as np
-import numexpr as ne
 from numpy.testing import assert_array_equal
-from time import time
+
+import numexpr as ne
+
 
 def bench(N):
     print("*** array length:", N)
@@ -31,4 +35,3 @@ def bench(N):
     ne.set_num_threads(2)
     for N in range(10, 20):
         bench(2**N)
-

bench/issue-47.py

@@ -1,4 +1,5 @@
 import numpy
+
 import numexpr
 
 numexpr.set_num_threads(8)

bench/large_array_vs_numpy.py

@@ -31,10 +31,12 @@
 import os
 
 os.environ["NUMEXPR_NUM_THREADS"] = "16"
+import threading
+import timeit
+
 import numpy as np
+
 import numexpr as ne
-import timeit
-import threading
 
 array_size = 10**8
 num_runs = 10

bench/multidim.py

@@ -12,9 +12,12 @@
 # Based on a script provided by Andrew Collette.
 
 from __future__ import print_function
+
+import time
+
 import numpy as np
+
 import numexpr as nx
-import time
 
 test_shapes = [
     (100*100*100),
@@ -90,5 +93,3 @@ def test_func(a, b, c):
     print("Simple: ", (stop1-start1)/nruns)
     print("Numexpr: ", (stop2-start2)/nruns)
     print("Chunked: ", (stop3-start3)/nruns)
-
-

bench/poly.py

@@ -17,11 +17,13 @@
 #######################################################################
 
 from __future__ import print_function
+
 import sys
 from time import time
+
 import numpy as np
-import numexpr as ne
 
+import numexpr as ne
 
 #expr = ".25*x**3 + .75*x**2 - 1.5*x - 2"  # the polynomial to compute
 expr = "((.25*x + .75)*x - 1.5)*x - 2"  # a computer-friendly polynomial

bench/timing.py

@@ -9,7 +9,10 @@
 ####################################################################
 
 from __future__ import print_function
-import timeit, numpy
+
+import timeit
+
+import numpy
 
 array_size = 5e6
 iterations = 2

bench/unaligned-simple.py

@@ -13,8 +13,11 @@
 """
 
 from __future__ import print_function
+
 from timeit import Timer
+
 import numpy as np
+
 import numexpr as ne
 
 niter = 10

bench/varying-expr.py

@@ -13,9 +13,12 @@
 # the latency of numexpr when working with small arrays.
 
 from __future__ import print_function
+
 import sys
 from time import time
+
 import numpy as np
+
 import numexpr as ne
 
 N = 100

bench/vml_timing.py

@@ -9,9 +9,12 @@
 ####################################################################
 
 from __future__ import print_function
+
 import sys
 import timeit
+
 import numpy
+
 import numexpr
 
 array_size = 5_000_000

bench/vml_timing2.py

@@ -4,11 +4,14 @@
 # https://github.com/pydata/numexpr/wiki/NumexprMKL
 
 from __future__ import print_function
+
 import datetime
 import sys
+from time import time
+
 import numpy as np
+
 import numexpr as ne
-from time import time
 
 N = int(2**26)
 

bench/vml_timing3.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
+from timeit import default_timer as timer
+
 import numpy as np
+
 import numexpr as ne
-from timeit import default_timer as timer
 
 x = np.ones(100000)
 scaler = -1J

doc/api.rst

@@ -3,11 +3,11 @@ NumExpr API
 
 .. automodule:: numexpr
    :members: evaluate, re_evaluate, disassemble, NumExpr, get_vml_version, set_vml_accuracy_mode, set_vml_num_threads, set_num_threads, detect_number_of_cores, detect_number_of_threads
-   
+
 .. py:attribute:: ncores
 
     The number of (virtual) cores detected.
-                  
+
 .. py:attribute:: nthreads
 
     The number of threads currently in-use.
@@ -18,11 +18,11 @@ NumExpr API
 
 .. py:attribute:: version
 
-    The version of NumExpr.      
-                  
-    
+    The version of NumExpr.
+
+
 Tests submodule
 ---------------
 
 .. automodule:: numexpr.tests
-   :members: test, print_versions
+   :members: test, print_versions

doc/index.rst

@@ -25,4 +25,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-