diff --git a/.github/workflows/buildwheel.yml b/.github/workflows/buildwheel.yml index 61b2d0b3..108c2ff8 100644 --- a/.github/workflows/buildwheel.yml +++ b/.github/workflows/buildwheel.yml @@ -10,8 +10,7 @@ jobs: strategy: fail-fast: false matrix: - # os: [ubuntu-20.04, windows-2019, macos-12] - os: [ubuntu-20.04, macos-12] + os: [ubuntu-20.04, windows-2019, macos-12] steps: - uses: actions/checkout@v3 @@ -20,22 +19,38 @@ jobs: with: python-version: '3.10' + - uses: msys2/setup-msys2@v2 + with: + msystem: mingw64 + # path-type inherit is used so that when cibuildwheel calls msys2 to + # run bin/cibw_prepare_python_windows.sh the virtual environment + # created by cibuildwheel will be available within msys2. The + # msys2/setup-msys2 README warns that using inherit here can be + # problematic in some situations. Maybe there is a better way to do + # this. + path-type: inherit + if: ${{ matrix.os == 'windows-2019' }} + - name: Build wheels uses: pypa/cibuildwheel@v2.11.2 env: CIBW_BUILD: cp39-* cp310-* cp311-* - #CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*" - CIBW_SKIP: "*-win32 *-musllinux_*" + CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*" + #CIBW_SKIP: "*-win32 *-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_I686_IMAGE: manylinux2014 CIBW_BEFORE_ALL_LINUX: bin/cibw_before_build_linux.sh CIBW_BEFORE_ALL_MACOS: bin/cibw_before_build_macosx.sh - CIBW_BEFORE_BUILD: pip install numpy cython + CIBW_BEFORE_ALL_WINDOWS: msys2 -c bin/cibw_before_build_windows.sh + CIBW_BEFORE_BUILD_WINDOWS: msys2 -c bin/cibw_prepare_python_windows.sh + CIBW_BEFORE_BUILD: pip install numpy cython delvewheel CIBW_ENVIRONMENT: > C_INCLUDE_PATH=$(pwd)/.local/include/ LIBRARY_PATH=$(pwd)/.local/lib/ LD_LIBRARY_PATH=$(pwd)/.local/lib:$LD_LIBRARY_PATH - CIBW_TEST_COMMAND: python -c 'import flint; print(str(flint.fmpz(2)))' + PYTHON_FLINT_MINGW64=true + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bin\repair_mingw_wheel.bat {dest_dir} {wheel} + CIBW_TEST_COMMAND: python -c "import flint; print(str(flint.fmpz(2)))" - uses: actions/upload-artifact@v3 with: @@ -48,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-12] + os: [ubuntu-20.04, windows-2019, macos-12] python-version: ['3.9', '3.10', '3.11'] steps: @@ -60,7 +75,5 @@ jobs: with: name: artifact path: wheelhouse - - run: python -m venv venv - - run: venv/bin/pip install -U pip - - run: venv/bin/pip install --find-links wheelhouse python_flint - - run: venv/bin/python test/test.py + - run: pip install --find-links wheelhouse python_flint + - run: python test/test.py diff --git a/.gitignore b/.gitignore index 2cc28b3f..d746d05a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ __pycache__ MANIFEST .eggs .local +*.egg-info diff --git a/bin/build_mingw64_wheel.sh b/bin/build_mingw64_wheel.sh new file mode 100755 index 00000000..7c239a09 --- /dev/null +++ b/bin/build_mingw64_wheel.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# make_wheels.sh +# +# Build relocatable Windows wheel for python_flint using the mingw-w64 +# toolchain in the MSYS2 enironment. +# +# - First install Python +# +# https://www.python.org/ftp/python/3.10.8/python-3.10.8-amd64.exe +# +# - Then checkout the code: +# +# $ git clone https://github.com/fredrik-johansson/python-flint/issues/1 +# +# - Then install msys2 +# +# https://repo.msys2.org/distrib/x86_64/msys2-x86_64-20221028.exe +# +# - Then open msys2, cd into the checked out repo. Make sure setup.py says +# +# libraries = ["arb", "flint", "mpfr", "gmp"] +# +# - Set the environment variable to the directory containing the installed +# Python that we want to build a wheel for i.e. the one installed from +# python.org. If python was on PATH then it would be +# +# PYTHONDIR=`dirname $(which python)` +# PYTHONVER=3.10 +# +# - Then run this script. + +set -o errexit + +# +# In CI this environment variable needs to be set to the directory containing +# the python.org installation of Python. If Python is installed in msys2 then +# it is also necesary to set this environment variable so that it picks up the +# right installation of Python i.e. the one that we want to build a wheel for. +# +if [[ -z "$PYTHONDIR" ]]; then + PYTHONDIR=`dirname $(which python)` +fi +PYTHON=$PYTHONDIR/python +VER="${PYTHONVER//./}" + +WHEELNAME=python_flint-0.3.0-cp$VER-cp$VER-win_amd64.whl + +$PYTHON -c 'print("hello world")' + +echo $PYTHONDIR +ls $PYTHONDIR +ls $PYTHONDIR/libs + +# Install the mingw-w64 toolchain +pacman -S --noconfirm mingw-w64-x86_64-gcc m4 make mingw-w64-x86_64-tools-git + +# This takes ~30mins +#bin/build_dependencies_unix.sh + +# Add the libpython$VER.a file to Python installation +cd $PYTHONDIR + gendef python$VER.dll + dlltool --dllname python$VER.dll --def python$VER.def --output-lib libpython$VER.a + mv libpython$VER.a libs +cd - + +# Make a virtual environment to install the build dependencies +$PYTHON -m venv .local/venv +source .local/venv/Scripts/activate +pip install numpy cython wheel delvewheel psutil + +# Pass this flag to setup.py +export PYTHON_FLINT_MINGW64_TMP=true + +# Build the wheel +C_INCLUDE_PATH=.local/include/ LIBRARY_PATH=.local/lib/ python setup.py build_ext -cmingw32 -f bdist_wheel + +# delvewheel requires DLLs created by mingw64 to be stripped +# +# https://github.com/scipy/scipy/blob/main/tools/wheels/repair_windows.sh +strip .local/bin/*.dll .local/lib/*.dll + +# Unpack the wheel and strip the .pyd DLL inside +cd dist +wheel unpack $WHEELNAME +rm $WHEELNAME +strip python_flint-*/flint/*.pyd +wheel pack python_flint-* +cd .. + +# Make the wheel relocatable +delvewheel repair dist/python_flint-0.3.0-cp$VER-cp$VER-win_amd64.whl \ + --add-path .local/bin:.local/lib/ + +# Make a virtual enironment to test the wheel +$PYTHON -m venv test_env +source test_env/Scripts/activate +pip install wheelhouse/$WHEELNAME +python -c 'import flint; print(flint.fmpz(2) + 2)' # 2 + 2 = 4? diff --git a/bin/cibw.bat b/bin/cibw.bat new file mode 100644 index 00000000..d43abf6c --- /dev/null +++ b/bin/cibw.bat @@ -0,0 +1,32 @@ +rem +rem This bat file can be used to test cibuildwheel locally on Windows. The +rem cibw_*_windows.sh files have lines to set the PATH for working locally but +rem those are commented out because they are not needed in CI. To use this +rem script +rem +rem 1. Uncomment those lines +rem 2. > pip install cibuildwheel +rem 3. > bin\cibw.bat +rem +rem The variables defined below should match those that are set in CI except +rem that C:\msys64\usr\bin\bash should just be msys2 -c in CI. +rem +rem It is also worth commenting out the line to build GMP etc after you have +rem built those once because that is by far the slowest step. +rem + +rem +rem If this script is run repeatedly then it would fail because of any leftover +rem wheels from a previous run so we delete them here. +rem +del /q wheelhouse\* + +set CIBW_BUILD=cp39-* cp310-* cp311-* +set CIBW_SKIP=*-win32 *-manylinux_i686 *-musllinux_* +set CIBW_BEFORE_ALL_WINDOWS=C:\msys64\usr\bin\bash bin/cibw_before_build_windows.sh +set CIBW_BEFORE_BUILD_WINDOWS=C:\msys64\usr\bin\bash bin/cibw_prepare_python_windows.sh +set CIBW_ENVIRONMENT=PYTHON_FLINT_MINGW64=true +set CIBW_REPAIR_WHEEL_COMMAND_WINDOWS=bin\repair_mingw_wheel.bat {dest_dir} {wheel} +set CIBW_TEST_COMMAND=python -c "import flint; print(str(flint.fmpz(2)))" + +cibuildwheel --platform windows diff --git a/bin/cibw_before_build_windows.sh b/bin/cibw_before_build_windows.sh new file mode 100755 index 00000000..ea2f8a89 --- /dev/null +++ b/bin/cibw_before_build_windows.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o errexit + +# Uncomment this to run cibuildwheel locally on Windows: +# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin + +# +# Make a setup.cfg to specify compiling with mingw64 (even though it says +# mingw32...) +# +echo '[build]' > setup.cfg +echo 'compiler = mingw32' >> setup.cfg +cat setup.cfg + +# Install the mingw-w64 toolchain +pacman -S --noconfirm mingw-w64-x86_64-gcc m4 make mingw-w64-x86_64-tools-git + +# This takes ~30mins +bin/build_dependencies_unix.sh diff --git a/bin/cibw_prepare_python_windows.sh b/bin/cibw_prepare_python_windows.sh new file mode 100644 index 00000000..d0410b35 --- /dev/null +++ b/bin/cibw_prepare_python_windows.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -o errexit + +# Uncomment this to run cibuildwheel locally on Windows: +# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin + +# VER should be set be e.g. 310 for Python 3.10 +VER=`python -c 'import sys; print("%s%s" % sys.version_info[:2])'` +echo VER=${VER} + +################################################### +# Find parent Python installation from the venv # +################################################### + +which python +PYTHONBIN=`dirname $(which python)` +PYTHONDIR=`dirname $PYTHONBIN` +cfgfile=$PYTHONDIR/pyvenv.cfg +homeline=`grep home $cfgfile` +homepath=${homeline#*=} + +echo --------------------------------------------------- +echo $homepath +echo --------------------------------------------------- + +################################################### +# Find pythonXX.dll and make a .a library # +################################################### + +cd $homepath +gendef python${VER}.dll +dlltool --dllname python${VER}.dll \ + --def python${VER}.def \ + --output-lib libpython${VER}.a + +mv libpython${VER}.a libs + +################################################### +# Install build dependencies # +################################################### + +pip install Cython numpy delvewheel diff --git a/bin/repair_mingw_wheel.bat b/bin/repair_mingw_wheel.bat new file mode 100644 index 00000000..e88d68ba --- /dev/null +++ b/bin/repair_mingw_wheel.bat @@ -0,0 +1,13 @@ +rem +rem This batch file serves the purpose of taking Windows style path arguments, +rem converting them to environment variables and calling msys2. This is needed +rem because otherwise in CI msys2 -c will mangle the paths turning e.g. C:\a\b +rem into C:ab. +rem + +set tempfile=tmpfile.deleteme +set WHEELHOUSE=%1 +set WHEELNAME=%2 + +msys2 -c bin/repair_mingw_wheel.sh +rem C:\msys64\usr\bin\bash bin/repair_mingw_wheel.sh diff --git a/bin/repair_mingw_wheel.sh b/bin/repair_mingw_wheel.sh new file mode 100755 index 00000000..c4827867 --- /dev/null +++ b/bin/repair_mingw_wheel.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Repair Windows wheels. See e.g.: +# +# https://github.com/scipy/scipy/blob/main/tools/wheels/repair_windows.sh + +set -o errexit + +# Uncomment this to run cibuildwheel locally on Windows: +# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin + +# We cannot use ordinary command line arguments in CI because msys2 -c mangles +# them. Instead we have a batch file to receive the arguments and convert them +# into environment variables before calling this script. When running locally +# this script could be run directly giving the parameters as command line +# arguments instead. + +if [[ -z "${WHEELHOUSE}" ]]; then + WHEELNAME=$1 +fi +if [[ -z "${WHEELNAME}" ]]; then + WHEELHOUSE=$2 +fi + +echo WHEELHOUSE=$WHEELHOUSE +echo WHEELNAME=$WHEELNAME + +wheeldir=$(dirname $WHEELNAME) +echo $wheeldir + +# delvewheel requires DLLs created by mingw64 to be stripped. This strips the +# DLLs for GMP etc that will have been build previously. +strip .local/bin/*.dll .local/lib/*.dll + +# Make sure to leave the wheel in the same directory +wheeldir=$(dirname $WHEELNAME) +pushd $wheeldir + # Unpack the wheel and strip any .pyd DLLs inside + wheel unpack $WHEELNAME + rm $WHEELNAME + strip python_flint-*/flint/*.pyd + wheel pack python_flint-* +popd + +# Make the wheel relocatable. This will fail with an error message about +# --no-mangle if strip has not been applied to all mingw64-created .dll and +# .pyd files that are needed for the wheel. +delvewheel repair $WHEELNAME \ + -w $WHEELHOUSE \ + --add-path .local/bin:.local/lib/ diff --git a/setup.py b/setup.py index 126003fd..1ebb8a11 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ import sys import os +from subprocess import check_call from distutils.core import setup from distutils.extension import Extension @@ -9,7 +10,27 @@ from distutils.sysconfig import get_config_vars if sys.platform == 'win32': - libraries = ["arb", "flint", "mpir", "mpfr", "pthreads"] + # + # This is used in CI to build wheels with mingw64 + # + if os.getenv('PYTHON_FLINT_MINGW64'): + libraries = ["arb", "flint", "mpfr", "gmp"] + includedir = os.path.join(os.path.dirname(__file__), '.local', 'include') + librarydir1 = os.path.join(os.path.dirname(__file__), '.local', 'bin') + librarydir2 = os.path.join(os.path.dirname(__file__), '.local', 'lib') + librarydirs = [librarydir1, librarydir2] + default_include_dirs += [includedir] + default_lib_dirs += librarydirs + # Add gcc to the PATH in GitHub Actions when this setup.py is called by + # cibuildwheel. + os.environ['PATH'] += r';C:\msys64\mingw64\bin' + elif os.getenv('PYTHON_FLINT_MINGW64_TMP'): + # This would be used to build under Windows against these libraries if + # they have been installed somewhere other than .local + libraries = ["arb", "flint", "mpfr", "gmp"] + else: + # For the MSVC toolchain link with mpir instead of gmp + libraries = ["arb", "flint", "mpir", "mpfr", "pthreads"] else: libraries = ["arb", "flint"] (opt,) = get_config_vars('OPT')