[mlir] Add option to disable MLIR Python dev package configuration.

Adds a CMake option MLIR_CONFIGURE_PYTHON_DEV_PACKAGES which gates doing package discovery and configuration for Python dev packages by MLIR.

The default Python setup that MLIR does has been a problem for super-projects that include LLVM for a long time because it forces a very specific package discovery mechanism that is not uniform in all uses.

When reviewing #117922, I noted that this would effectively be a break the world event for downstreams, forcing them to adapt their nanobind dep to the exact way that MLIR does it. Adding the option to just wholesale skip the built-in configuration heuristics at least gives us a mechanism to tell downstreams to migrate to, giving them complete control and not requiring packaging workarounds. This seemed a better option than (once again) creating a situation where downstreams could not integrate the dep change without doing tricky infra upgrades, and it removes the burden from the author of that patch from needing to think about how this affects super-projects that include MLIR (i.e. they can just be told to do it themselves as needed vs being in a wedged state and unable to upgrade).

Author: stellaraccident

mlir/CMakeLists.txt

@@ -162,6 +162,9 @@ configure_file(
 # Requires:
 #   The pybind11 library can be found (set with -DPYBIND_DIR=...)
 #   The python executable is correct (set with -DPython3_EXECUTABLE=...)
+# By default, find_package and probing for installed pybind11 is performed.
+# Super projects can set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES=ON to 
+# disable all package setup and control it themselves.
 #-------------------------------------------------------------------------------
 
 set(MLIR_ENABLE_BINDINGS_PYTHON 0 CACHE BOOL
@@ -170,8 +173,17 @@ set(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH 1 CACHE BOOL
        "Prime the python detection by searching for a full 'Development' \
        component first (temporary while diagnosing environment specific Python \
        detection issues)")
+set(MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES 0 CACHE BOOL
+       "Performs python dev package configuration sufficient to use all MLIR \
+       python features. Super-projects that wish to control their own setup \
+       must perform an appropriate find_package of Python3 with \
+       'Development.Module' and ensure that find_package(pybind11) is \
+       satisfied (and keep up to date as requirements evolve).")
+
 if(MLIR_ENABLE_BINDINGS_PYTHON)
   include(MLIRDetectPythonEnv)
+  # Note that both upstream and downstreams often call this macro. It gates
+  # internally on the MLIR_CONFIGURE_PYTHON_DEV_PACKAGES option.
   mlir_configure_python_dev_packages()
 endif()
 

mlir/cmake/modules/MLIRDetectPythonEnv.cmake

@@ -2,34 +2,36 @@
 
 # Finds and configures python packages needed to build MLIR Python bindings.
 macro(mlir_configure_python_dev_packages)
-  if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH)
-    # Prime the search for python to see if there is a full development
-    # package. This seems to work around cmake bugs searching only for
-    # Development.Module in some environments. However, in other environments
-    # it may interfere with the subsequent search for Development.Module.
-    find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
-      COMPONENTS Interpreter Development)
-  endif()
+  if(NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES)
+    if(MLIR_DETECT_PYTHON_ENV_PRIME_SEARCH)
+      # Prime the search for python to see if there is a full development
+      # package. This seems to work around cmake bugs searching only for
+      # Development.Module in some environments. However, in other environments
+      # it may interfere with the subsequent search for Development.Module.
+      find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
+        COMPONENTS Interpreter Development)
+    endif()
 
-  # After CMake 3.18, we are able to limit the scope of the search to just
-  # Development.Module. Searching for Development will fail in situations where
-  # the Python libraries are not available. When possible, limit to just
-  # Development.Module.
-  # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
-  set(_python_development_component Development.Module)
+    # After CMake 3.18, we are able to limit the scope of the search to just
+    # Development.Module. Searching for Development will fail in situations where
+    # the Python libraries are not available. When possible, limit to just
+    # Development.Module.
+    # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
+    set(_python_development_component Development.Module)
 
-  find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
-    COMPONENTS Interpreter ${_python_development_component} REQUIRED)
-  unset(_python_development_component)
-  message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
-  message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
-  message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
-  mlir_detect_pybind11_install()
-  find_package(pybind11 2.10 CONFIG REQUIRED)
-  message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
-  message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                 "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                 "extension = '${PYTHON_MODULE_EXTENSION}")
+    find_package(Python3 ${LLVM_MINIMUM_PYTHON_VERSION}
+      COMPONENTS Interpreter ${_python_development_component} REQUIRED)
+    unset(_python_development_component)
+    message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
+    message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
+    message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
+    mlir_detect_pybind11_install()
+    find_package(pybind11 2.10 CONFIG REQUIRED)
+    message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
+    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                  "extension = '${PYTHON_MODULE_EXTENSION}")
+  endif()
 endmacro()
 
 # Detects a pybind11 package installed in the current python environment