From fb1763af5521d990319d96c86e908ea2c591d412 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 22 Apr 2025 17:14:12 +0200 Subject: [PATCH 1/5] Use `sys.base_exec_prefix` for `Python_ROOT_DIR` `Python_ROOT_DIR` is used for the find hints for - `Python_EXECUTABLE`: Unnecessary because we provide `Python_EXECUTABLE` as a cache variable and that has precedence - `python-config`: should point to the target's variable. Generally this is a shell script which would be executable during cross-compilation (windows case unknown) - `Python_LIBRARY`: should point to the target's library which would be on the `crossenv` uses a fake environment so that `sys.base_*_prefix` variables point to the target's system. Presumably PEP720 would work the same. Signed-off-by: Cristian Le --- src/scikit_build_core/builder/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scikit_build_core/builder/builder.py b/src/scikit_build_core/builder/builder.py index 7079fd50..5fcb5a5a 100644 --- a/src/scikit_build_core/builder/builder.py +++ b/src/scikit_build_core/builder/builder.py @@ -240,7 +240,7 @@ def configure( # Modern Find Python for prefix in ("Python", "Python3"): cache_config[f"{prefix}_EXECUTABLE"] = Path(sys.executable) - cache_config[f"{prefix}_ROOT_DIR"] = Path(sys.prefix) + cache_config[f"{prefix}_ROOT_DIR"] = Path(sys.base_exec_prefix) cache_config[f"{prefix}_INCLUDE_DIR"] = python_include_dir cache_config[f"{prefix}_FIND_REGISTRY"] = "NEVER" # FindPython may break if this is set - only useful on Windows From fb97299380132a7ac82274f4fd2499a2527a3709 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Fri, 11 Apr 2025 18:47:41 +0200 Subject: [PATCH 2/5] Add cross-compilation inputs Signed-off-by: Cristian Le --- README.md | 3 +++ src/scikit_build_core/builder/builder.py | 3 +++ src/scikit_build_core/resources/scikit-build.schema.json | 4 ++++ src/scikit_build_core/settings/skbuild_model.py | 5 +++++ 4 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 30c48fc6..ac0a2614 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,9 @@ cmake.source-dir = "." # DEPRECATED in 0.10; use build.targets instead. cmake.targets = "" +# The CMAKE_TOOLCHAIN_FILE used for cross-compilation. +cmake.toolchain-file = "" + # The versions of Ninja to allow. If Ninja is not present on the system or does # not pass this specifier, it will be downloaded via PyPI if possible. An empty # string will disable this check. diff --git a/src/scikit_build_core/builder/builder.py b/src/scikit_build_core/builder/builder.py index 5fcb5a5a..cede912c 100644 --- a/src/scikit_build_core/builder/builder.py +++ b/src/scikit_build_core/builder/builder.py @@ -231,6 +231,9 @@ def configure( python_include_dir = get_python_include_dir() numpy_include_dir = get_numpy_include_dir() + if self.settings.cmake.toolchain_file: + cache_config["CMAKE_TOOLCHAIN_FILE"] = self.settings.cmake.toolchain_file + # Classic Find Python cache_config["PYTHON_EXECUTABLE"] = Path(sys.executable) cache_config["PYTHON_INCLUDE_DIR"] = python_include_dir diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json index e95d65c7..94340727 100644 --- a/src/scikit_build_core/resources/scikit-build.schema.json +++ b/src/scikit_build_core/resources/scikit-build.schema.json @@ -102,6 +102,10 @@ }, "description": "DEPRECATED in 0.10; use build.targets instead.", "deprecated": true + }, + "toolchain-file": { + "type": "string", + "description": "The CMAKE_TOOLCHAIN_FILE used for cross-compilation." } } }, diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py index d25a4a84..2f75ff51 100644 --- a/src/scikit_build_core/settings/skbuild_model.py +++ b/src/scikit_build_core/settings/skbuild_model.py @@ -111,6 +111,11 @@ class CMakeSettings: DEPRECATED in 0.10; use build.targets instead. """ + toolchain_file: Optional[Path] = None + """ + The CMAKE_TOOLCHAIN_FILE used for cross-compilation. + """ + @dataclasses.dataclass class SearchSettings: From 1ba62be7d423c5236a8c31c82a7a0bf47acaf855 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 22 Apr 2025 19:49:01 +0200 Subject: [PATCH 3/5] Add a setting to manually specify wheel tags Signed-off-by: Cristian Le --- README.md | 6 ++++++ src/scikit_build_core/build/wheel.py | 9 +++++++-- .../resources/scikit-build.schema.json | 10 ++++++++++ src/scikit_build_core/settings/skbuild_model.py | 7 +++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ac0a2614..6a4d23b2 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,12 @@ wheel.exclude = [] # The build tag to use for the wheel. If empty, no build tag is used. wheel.build-tag = "" +# Manually specify the wheel tags to use, ignoring other inputs such as +# ``wheel.py-api``. Each tag must be of the format +# {interpreter}-{abi}-{platform}. If not specified, these tags are automatically +# calculated. +wheel.tags = [] + # If CMake is less than this value, backport a copy of FindPython. Set to 0 # disable this, or the empty string. backport.find-python = "3.26.1" diff --git a/src/scikit_build_core/build/wheel.py b/src/scikit_build_core/build/wheel.py index 47e223bb..9e122ba4 100644 --- a/src/scikit_build_core/build/wheel.py +++ b/src/scikit_build_core/build/wheel.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Literal from packaging.requirements import Requirement +from packaging.tags import Tag from packaging.utils import canonicalize_name from .._compat import tomllib @@ -260,6 +261,10 @@ def _build_wheel_impl_impl( f"{{red}}({state})", ) + override_wheel_tags = None + if settings.wheel.tags: + override_wheel_tags = {Tag(*tag.split("-")) for tag in settings.wheel.tags} + with tempfile.TemporaryDirectory() as tmpdir: build_tmp_folder = Path(tmpdir) wheel_dir = build_tmp_folder / "wheel" @@ -364,7 +369,7 @@ def _build_wheel_impl_impl( wheel = WheelWriter( metadata, Path(metadata_directory), - tags.as_tags_set(), + override_wheel_tags or tags.as_tags_set(), WheelMetadata( root_is_purelib=targetlib == "purelib", build_tag=settings.wheel.build_tag, @@ -480,7 +485,7 @@ def _build_wheel_impl_impl( with WheelWriter( metadata, Path(wheel_directory), - tags.as_tags_set(), + override_wheel_tags or tags.as_tags_set(), WheelMetadata( root_is_purelib=targetlib == "purelib", build_tag=settings.wheel.build_tag, diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json index 94340727..6fb73f89 100644 --- a/src/scikit_build_core/resources/scikit-build.schema.json +++ b/src/scikit_build_core/resources/scikit-build.schema.json @@ -243,6 +243,13 @@ "type": "string", "default": "", "description": "The build tag to use for the wheel. If empty, no build tag is used." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Manually specify the wheel tags to use, ignoring other inputs such as ``wheel.py-api``. Each tag must be of the format {interpreter}-{abi}-{platform}. If not specified, these tags are automatically calculated." } } }, @@ -570,6 +577,9 @@ }, "exclude": { "$ref": "#/$defs/inherit" + }, + "tags": { + "$ref": "#/$defs/inherit" } } }, diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py index 2f75ff51..45c0294b 100644 --- a/src/scikit_build_core/settings/skbuild_model.py +++ b/src/scikit_build_core/settings/skbuild_model.py @@ -259,6 +259,13 @@ class WheelSettings: The build tag to use for the wheel. If empty, no build tag is used. """ + tags: List[str] = dataclasses.field(default_factory=list) + """ + Manually specify the wheel tags to use, ignoring other inputs such as + ``wheel.py-api``. Each tag must be of the format {interpreter}-{abi}-{platform}. + If not specified, these tags are automatically calculated. + """ + @dataclasses.dataclass class BackportSettings: From e7d80bc70a2bc09940172c716d092d5d2884c691 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 22 Apr 2025 19:58:43 +0200 Subject: [PATCH 4/5] Add an option to avoid passing the python hints Signed-off-by: Cristian Le --- README.md | 5 +++ docs/reference/configs.md | 22 +++++++++++ src/scikit_build_core/builder/builder.py | 39 ++++++++++--------- .../resources/scikit-build.schema.json | 5 +++ .../settings/skbuild_model.py | 7 ++++ 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6a4d23b2..badc6f41 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,11 @@ cmake.targets = "" # The CMAKE_TOOLCHAIN_FILE used for cross-compilation. cmake.toolchain-file = "" +# Do not pass the current environment's python hints such as +# ``Python_EXECUTABLE``. Primarily used for cross-compilation where the +# CMAKE_TOOLCHAIN_FILE should handle it instead. +cmake.python-hints = true + # The versions of Ninja to allow. If Ninja is not present on the system or does # not pass this specifier, it will be downloaded via PyPI if possible. An empty # string will disable this check. diff --git a/docs/reference/configs.md b/docs/reference/configs.md index 96ea5bd4..8211206a 100644 --- a/docs/reference/configs.md +++ b/docs/reference/configs.md @@ -176,6 +176,14 @@ print(mk_skbuild_docs()) DEPRECATED in 0.8; use version instead. ``` +```{eval-rst} +.. confval:: cmake.python-hints + :type: ``bool`` + :default: true + + Do not pass the current environment's python hints such as ``Python_EXECUTABLE``. Primarily used for cross-compilation where the CMAKE_TOOLCHAIN_FILE should handle it instead. +``` + ```{eval-rst} .. confval:: cmake.source-dir :type: ``Path`` @@ -191,6 +199,13 @@ print(mk_skbuild_docs()) DEPRECATED in 0.10; use build.targets instead. ``` +```{eval-rst} +.. confval:: cmake.toolchain-file + :type: ``Path`` + + The CMAKE_TOOLCHAIN_FILE used for cross-compilation. +``` + ```{eval-rst} .. confval:: cmake.verbose :type: ``bool`` @@ -440,4 +455,11 @@ print(mk_skbuild_docs()) The Python tags. The default (empty string) will use the default Python version. You can also set this to "cp38" to enable the CPython 3.8+ Stable ABI / Limited API (only on CPython and if the version is sufficient, otherwise this has no effect). Or you can set it to "py3" or "py2.py3" to ignore Python ABI compatibility. The ABI tag is inferred from this tag. ``` +```{eval-rst} +.. confval:: wheel.tags + :type: ``list[str]`` + + Manually specify the wheel tags to use, ignoring other inputs such as ``wheel.py-api``. Each tag must be of the format {interpreter}-{abi}-{platform}. If not specified, these tags are automatically calculated. +``` + diff --git a/src/scikit_build_core/builder/builder.py b/src/scikit_build_core/builder/builder.py index cede912c..28fa4da7 100644 --- a/src/scikit_build_core/builder/builder.py +++ b/src/scikit_build_core/builder/builder.py @@ -234,25 +234,26 @@ def configure( if self.settings.cmake.toolchain_file: cache_config["CMAKE_TOOLCHAIN_FILE"] = self.settings.cmake.toolchain_file - # Classic Find Python - cache_config["PYTHON_EXECUTABLE"] = Path(sys.executable) - cache_config["PYTHON_INCLUDE_DIR"] = python_include_dir - if python_library: - cache_config["PYTHON_LIBRARY"] = python_library - - # Modern Find Python - for prefix in ("Python", "Python3"): - cache_config[f"{prefix}_EXECUTABLE"] = Path(sys.executable) - cache_config[f"{prefix}_ROOT_DIR"] = Path(sys.base_exec_prefix) - cache_config[f"{prefix}_INCLUDE_DIR"] = python_include_dir - cache_config[f"{prefix}_FIND_REGISTRY"] = "NEVER" - # FindPython may break if this is set - only useful on Windows - if python_library and sysconfig.get_platform().startswith("win"): - cache_config[f"{prefix}_LIBRARY"] = python_library - if python_sabi_library and sysconfig.get_platform().startswith("win"): - cache_config[f"{prefix}_SABI_LIBRARY"] = python_sabi_library - if numpy_include_dir: - cache_config[f"{prefix}_NumPy_INCLUDE_DIR"] = numpy_include_dir + if self.settings.cmake.python_hints: + # Classic Find Python + cache_config["PYTHON_EXECUTABLE"] = Path(sys.executable) + cache_config["PYTHON_INCLUDE_DIR"] = python_include_dir + if python_library: + cache_config["PYTHON_LIBRARY"] = python_library + + # Modern Find Python + for prefix in ("Python", "Python3"): + cache_config[f"{prefix}_EXECUTABLE"] = Path(sys.executable) + cache_config[f"{prefix}_ROOT_DIR"] = Path(sys.base_exec_prefix) + cache_config[f"{prefix}_INCLUDE_DIR"] = python_include_dir + cache_config[f"{prefix}_FIND_REGISTRY"] = "NEVER" + # FindPython may break if this is set - only useful on Windows + if python_library and sysconfig.get_platform().startswith("win"): + cache_config[f"{prefix}_LIBRARY"] = python_library + if python_sabi_library and sysconfig.get_platform().startswith("win"): + cache_config[f"{prefix}_SABI_LIBRARY"] = python_sabi_library + if numpy_include_dir: + cache_config[f"{prefix}_NumPy_INCLUDE_DIR"] = numpy_include_dir cache_config["SKBUILD_SOABI"] = get_soabi(self.config.env, abi3=limited_api) diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json index 6fb73f89..78c10c74 100644 --- a/src/scikit_build_core/resources/scikit-build.schema.json +++ b/src/scikit_build_core/resources/scikit-build.schema.json @@ -106,6 +106,11 @@ "toolchain-file": { "type": "string", "description": "The CMAKE_TOOLCHAIN_FILE used for cross-compilation." + }, + "python-hints": { + "type": "boolean", + "default": true, + "description": "Do not pass the current environment's python hints such as ``Python_EXECUTABLE``. Primarily used for cross-compilation where the CMAKE_TOOLCHAIN_FILE should handle it instead." } } }, diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py index 45c0294b..598473f5 100644 --- a/src/scikit_build_core/settings/skbuild_model.py +++ b/src/scikit_build_core/settings/skbuild_model.py @@ -116,6 +116,13 @@ class CMakeSettings: The CMAKE_TOOLCHAIN_FILE used for cross-compilation. """ + python_hints: bool = True + """ + Do not pass the current environment's python hints such as ``Python_EXECUTABLE``. + Primarily used for cross-compilation where the CMAKE_TOOLCHAIN_FILE should handle it + instead. + """ + @dataclasses.dataclass class SearchSettings: From dcb86bf719f3c7046e7c3d4502426e16618fd293 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 22 Apr 2025 21:00:34 +0200 Subject: [PATCH 5/5] Document how to crosscompile with toolchain files Signed-off-by: Cristian Le --- docs/conf.py | 1 + docs/guide/crosscompile.md | 77 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 23b35cef..60a06157 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -134,6 +134,7 @@ myst_substitutions = { "version": version, } +myst_heading_anchors = 2 # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section) diff --git a/docs/guide/crosscompile.md b/docs/guide/crosscompile.md index 97b24531..ea71e441 100644 --- a/docs/guide/crosscompile.md +++ b/docs/guide/crosscompile.md @@ -1,5 +1,9 @@ # Cross-compiling +Generally scikit-build-core will try to account for environment variables that +specify to CMake directly how to cross-compile. Alternatively, you can define +manually how to cross-compile as detailed in [manual cross compilation] section. + ## macOS Unlike the other platforms, macOS has the ability to target older operating @@ -50,10 +54,7 @@ correct suffix. These values are set by cibuildwheel when cross-compiling. ## Linux -It should be possible to cross-compile to Linux, but due to the challenges of -getting the manylinux RHEL devtoolkit compilers, this is currently a TODO. See -`py-build-cmake `\_ -for an alternative package's usage of toolchain files. +See [manual cross compilation] section for the general approach. ### Intel to Emscripten (Pyodide) @@ -64,3 +65,71 @@ by setting `_PYTHON_SYSCONFIGDATA_NAME`. This causes values like `SOABI` and This is unfortunately incorrectly stripped from the cmake wrapper pyodide uses, so FindPython will report the wrong values, but pyodide-build will rename the .so's afterwards. + +## Manual cross compilation + +The manual cross compilation assumes you have [toolchain file] prepared defining +the cross-compilers and where to search for the target development files, +including the python library. A simple setup of this is to use the clang +compiler and point `CMAKE_SYSROOT` to a mounted copy of the target system's root + +```cmake +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +set(triple aarch64-linux-gnu) + +set(CMAKE_C_COMPILER clang) +set(CMAKE_CXX_COMPILER clang++) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) + +set(CMAKE_SYSROOT "/path/to/aarch64/mount/") +``` + +For more complex environments such as embedded devices, Android or iOS see +CMake's guide on how to write the [toolchain file]. + +You can pass the toolchain file using the environment variable +`CMAKE_TOOLCHAIN_FILE`, or the `cmake.toolchain-file` pyproject option. You may +also need to use `wheel.tags` to manually specify the wheel tags to use for the +file and `cmake.no-python-hints` if the target python should be detected using +the toolchain file instead. + +:::{note} + +Because most of the logic in [`FindPython`] is gated by the +`CMAKE_CROSSCOMPILING`, you generally should _not_ include the `Interpreter` +component in the `find_package` command or use the `Python_ARTIFACTS_PREFIX` +feature to distinguish the system and target components. + +::: + +:::{versionadded} 0.11 + +::: + +### Crossenv + +[Crossenv] cross compilation is supported in scikit-build-core. This tool +creates a fake virtual environment where configuration hints such as +`EXT_SUFFIX` are overwritten with the target's values. This should work without +specifying `wheel.tags` overwrites manually. + +:::{note} + +Because the target Python executable is being faked, the usage of +`CMAKE_CROSSCOMPILING_EMULATOR` for the `Interpreter` would not be correct in +this case. + +::: + +:::{versionadded} 0.11 + +::: + +[manual cross compilation]: #manual-cross-compilation +[toolchain file]: + https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling +[crossenv]: https://crossenv.readthedocs.io/en/latest/ +[`FindPython`]: https://cmake.org/cmake/help/git-master/module/FindPython.html