diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be33471a..18f996e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: [3.6, 3.7, 3.8, 3.9, pypy3] platform: [ - { os: "macOS-latest", python-architecture: "x64", rust-target: "x86_64-apple-darwin" }, + { os: "macos-latest", python-architecture: "x64", rust-target: "x86_64-apple-darwin" }, { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" }, { os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" }, { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" }, @@ -44,6 +44,10 @@ jobs: profile: minimal default: true + - name: Install Rust aarch64-apple-darwin target + if: matrix.platform.os == 'macos-latest' + run: rustup target add aarch64-apple-darwin + - name: Install test dependencies run: pip install --upgrade tox setuptools @@ -74,6 +78,24 @@ jobs: tox -c $example_dir -e py done + - name: Test macOS universal2 + if: matrix.platform.os == 'macos-latest' + shell: bash + env: + DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer + MACOSX_DEPLOYMENT_TARGET: '10.9' + ARCHFLAGS: -arch x86_64 -arch arm64 + PYO3_CROSS_LIB_DIR: /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib + run: | + cd examples/namespace_package + pip install wheel + python setup.py bdist_wheel + ls -l dist/ + pip install --force-reinstall dist/namespace_package*_universal2.whl + cd - + python -c "from namespace_package import rust; assert rust.rust_func() == 14" + python -c "from namespace_package import python; assert python.python_func() == 15" + test-abi3: runs-on: ${{ matrix.os }} strategy: @@ -92,6 +114,10 @@ jobs: toolchain: stable override: true + - name: Install Rust aarch64-apple-darwin target + if: matrix.os == 'macos-latest' + run: rustup target add aarch64-apple-darwin + - name: Build package run: pip install -e . diff --git a/CHANGELOG.md b/CHANGELOG.md index cc050c4e..517eeb48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Added - Support building x86-64 wheel on arm64 macOS machine. [#114](https://github.com/PyO3/setuptools-rust/pull/114) +- Add macOS universal2 wheel building support. [#115](https://github.com/PyO3/setuptools-rust/pull/115) ### Changed - Respect `PYO3_PYTHON` and `PYTHON_SYS_EXECUTABLE` environment variables if set. [#96](https://github.com/PyO3/setuptools-rust/pull/96) diff --git a/README.md b/README.md index 116516c4..1500e428 100644 --- a/README.md +++ b/README.md @@ -133,12 +133,29 @@ It is possible to use any of the `manylinux` docker images: `manylinux1`, `manyl You can define rust extension with RustExtension class: -RustExtension(name, path, args=None, features=None, -rust\_version=None, quiet=False, debug=False) +```python +RustExtension( + name, + path="Cargo.toml", + args=None, + features=None, + rustc_flags=None, + rust_version=None, + quiet=False, + debug=None, + binding=Binding.PyO3, + strip=Strip.No, + script=False, + native=False, + optional=False, + py_limited_api=False, +) +``` The class for creating rust extensions. - - param str name + - param str `name` + the full name of the extension, including any packages -- ie. *not* a filename or pathname, but Python dotted name. It is possible to specify multiple binaries, if extension uses @@ -147,51 +164,71 @@ The class for creating rust extensions. binaries and values are full name of the executable inside python package. - - param str path + - param str `path` + path to the Cargo.toml manifest file - - param \[str\] args + - param \[str\] `args` + a list of extra argumenents to be passed to cargo. - - param \[str\] features + - param \[str\] `features` + a list of features to also build - - param \[str\] rustc\_flags + - param \[str\] `rustc_flags` + A list of arguments to pass to rustc, e.g. cargo rustc --features \ \ -- \ - - param str rust\_version + - param str `rust_version` + sematic version of rust compiler version -- for example *\>1.14,\<1.16*, default is None - - param bool quiet - Does not echo cargo's output. default is False + - param bool `quiet` - - param bool debug - Controls whether --debug or --release is passed to cargo. If set + Does not echo cargo's output. default is `False` + + - param bool `debug` + + Controls whether `--debug` or `--release` is passed to cargo. If set to None then build type is auto-detect. Inplace build is debug - build otherwise release. Default: None + build otherwise release. Default: `None` + + - param int `binding` + + Controls which python binding is in use. + * `Binding.PyO3` uses PyO3 + * `Binding.RustCPython` uses rust-cpython + * `Binding.NoBinding` uses no binding. + * `Binding.Exec` build executable. - - param int binding - Controls which python binding is in use. Binding.PyO3 uses PyO3 - Binding.RustCPython uses rust-cpython Binding.NoBinding uses no - binding. Binding.Exec build executable. + - param int `strip` - - param int strip Strip symbols from final file. Does nothing for debug build. - Strip.No - do not strip symbols (default) Strip.Debug - strip - debug symbols Strip.All - strip all symbols + * `Strip.No` - do not strip symbols (default) + * `Strip.Debug` - strip debug symbols + * `Strip.All` - strip all symbols - - param bool script - Generate console script for executable if Binding.Exec is used. + - param bool `script` + + Generate console script for executable if `Binding.Exec` is used. + + - param bool `native` - - param bool native Build extension or executable with "-C target-cpu=native" - - param bool optional + - param bool `optional` + if it is true, a build failure in the extension will not abort the build process, but instead simply not install the failing extension. + - param bool `py_limited_api` + + Same as `py_limited_api` on `setuptools.Extension`. Note that if you + set this to True, your extension must pass the appropriate feature + flags to pyo3 (ensuring that `abi3` feature is enabled). ## Commands diff --git a/examples/namespace_package/tox.ini b/examples/namespace_package/tox.ini index 5e9f9b33..281e7b14 100644 --- a/examples/namespace_package/tox.ini +++ b/examples/namespace_package/tox.ini @@ -8,3 +8,4 @@ deps = setuptools-rust @ file://{toxinidir}/../../ pytest commands = pytest {posargs} +passenv = * diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index 23045b76..fd96c951 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -62,7 +62,39 @@ def finalize_options(self): ("inplace", "inplace"), ) + def get_target_triple(self): + # If we are on a 64-bit machine, but running a 32-bit Python, then + # we'll target a 32-bit Rust build. + # Automatic target detection can be overridden via the CARGO_BUILD_TARGET + # environment variable. + if os.getenv("CARGO_BUILD_TARGET"): + return os.environ["CARGO_BUILD_TARGET"] + elif self.plat_name == "win32": + return "i686-pc-windows-msvc" + elif self.plat_name == "win-amd64": + return "x86_64-pc-windows-msvc" + elif self.plat_name.startswith("macosx-") and platform.machine() == "x86_64": + # x86_64 or arm64 macOS targeting x86_64 + return "x86_64-apple-darwin" + def run_for_extension(self, ext: RustExtension): + arch_flags = os.getenv("ARCHFLAGS") + universal2 = False + if self.plat_name.startswith("macosx-") and arch_flags: + universal2 = "x86_64" in arch_flags and "arm64" in arch_flags + if universal2: + arm64_dylib_paths = self.build_extension(ext, "aarch64-apple-darwin") + x86_64_dylib_paths = self.build_extension(ext, "x86_64-apple-darwin") + dylib_paths = [] + for (target_fname, arm64_dylib), (_, x86_64_dylib) in zip(arm64_dylib_paths, x86_64_dylib_paths): + fat_dylib_path = arm64_dylib.replace("aarch64-apple-darwin/", "") + self.create_universal2_binary(fat_dylib_path, [arm64_dylib, x86_64_dylib]) + dylib_paths.append((target_fname, fat_dylib_path)) + else: + dylib_paths = self.build_extension(ext) + self.install_extension(ext, dylib_paths) + + def build_extension(self, ext: RustExtension, target_triple=None): executable = ext.binding == Binding.Exec rust_target_info = get_rust_target_info() @@ -84,22 +116,8 @@ def run_for_extension(self, ext: RustExtension): ) rustflags = "" - # If we are on a 64-bit machine, but running a 32-bit Python, then - # we'll target a 32-bit Rust build. - # Automatic target detection can be overridden via the CARGO_BUILD_TARGET - # environment variable. - target_triple = None + target_triple = target_triple or self.get_target_triple() target_args = [] - if os.getenv("CARGO_BUILD_TARGET"): - target_triple = os.environ["CARGO_BUILD_TARGET"] - elif self.plat_name == "win32": - target_triple = "i686-pc-windows-msvc" - elif self.plat_name == "win-amd64": - target_triple = "x86_64-pc-windows-msvc" - elif self.plat_name.startswith("macosx-") and platform.machine() == "x86_64": - # x86_64 or arm64 macOS targeting x86_64 - target_triple = "x86_64-apple-darwin" - if target_triple is not None: target_args = ["--target", target_triple] @@ -264,7 +282,14 @@ def run_for_extension(self, ext: RustExtension): raise DistutilsExecError( f"Rust build failed; unable to find any {wildcard_so} in {artifactsdir}" ) + return dylib_paths + def install_extension(self, ext: RustExtension, dylib_paths): + executable = ext.binding == Binding.Exec + debug_build = ext.debug if ext.debug is not None else self.inplace + debug_build = self.debug if self.debug is not None else debug_build + if self.release: + debug_build = False # Ask build_ext where the shared library would go if it had built it, # then copy it there. build_ext = self.get_finalized_command("build_ext") @@ -301,7 +326,7 @@ def run_for_extension(self, ext: RustExtension): args.insert(0, "strip") args.append(ext_path) try: - output = subprocess.check_output(args, env=env) + output = subprocess.check_output(args) except subprocess.CalledProcessError: pass @@ -323,3 +348,31 @@ def get_dylib_ext_path(self, ext, target_fname): return build_ext.get_ext_fullpath(target_fname) finally: del build_ext.ext_map[modpath] + + @staticmethod + def create_universal2_binary(output_path, input_paths): + # Try lipo first + command = ["lipo", "-create", "-output", output_path, *input_paths] + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + output = e.output + if isinstance(output, bytes): + output = e.output.decode("latin-1").strip() + raise CompileError( + "lipo failed with code: %d\n%s" % (e.returncode, output) + ) + except OSError: + # lipo not found, try using the fat-macho library + try: + from fat_macho import FatWriter + except ImportError: + raise DistutilsExecError( + "failed to locate `lipo` or import `fat_macho.FatWriter`. " + "Try installing with `pip install fat-macho` " + ) + fat = FatWriter() + for input_path in input_paths: + with open(input_path, "rb") as f: + fat.add(f.read()) + fat.write_to(output_path) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index 4232961b..6289e522 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -1,20 +1,16 @@ -from abc import ABC, abstractmethod +import os from distutils import log -from distutils.cmd import Command from distutils.command.check import check from distutils.command.clean import clean -from distutils.errors import DistutilsPlatformError -from setuptools.command.install import install + from setuptools.command.build_ext import build_ext +from setuptools.command.install import install try: from wheel.bdist_wheel import bdist_wheel except ImportError: bdist_wheel = None -from .extension import RustExtension -from .utils import get_rust_version - def add_rust_extension(dist): build_ext_base_class = dist.cmdclass.get('build_ext', build_ext) @@ -114,6 +110,25 @@ def finalize_options(self): self.distribution.entry_points["console_scripts"] = ep_scripts bdist_wheel_base_class.finalize_options(self) + + def get_tag(self): + python, abi, plat = super().get_tag() + arch_flags = os.getenv("ARCHFLAGS") + universal2 = False + if self.plat_name.startswith("macosx-") and arch_flags: + universal2 = "x86_64" in arch_flags and "arm64" in arch_flags + if universal2 and plat.startswith("macosx_"): + from wheel.macosx_libfile import calculate_macosx_platform_tag + + macos_target = os.getenv("MACOSX_DEPLOYMENT_TARGET") + if macos_target is None: + # Example: macosx_11_0_arm64 + macos_target = '.'.join(plat.split("_")[1:3]) + plat = calculate_macosx_platform_tag( + self.bdist_dir, + "macosx-{}-universal2".format(macos_target) + ) + return python, abi, plat dist.cmdclass["bdist_wheel"] = bdist_wheel_rust_extension