bpo-41282: Fix broken make install (GH-26329)

A previous commit broke a check in sysconfig when building cpython itself.
This caused builds of the standard library modules to search a wrong
location (the installed location rather than the source directory) for
header files with the net effect that a ``make install``
incorrectly caused all extension modules to be rebuilt again and
with incorrect include file paths.

When building Python, we need two distinct "include" directories:
- source .h files
- install target for .h files

Note that this doesn't matter except when building Python from source.

Historically:
- source .h files were in the distutils scheme under 'include'
- the install directory was in the distutils.command.install scheme
    under 'headers'

GH-24549 merged these; sysconfig is now the single source of truth and
distutils is derived from it.

This commit introduces a "secret" scheme path, 'headers', which contains
the install target. It is only present when building Python.
The distutils code uses it if present, and falls back to 'include'.

Co-authored-by: Ned Deily <[email protected]>
(cherry picked from commit 563bd5a)

Co-authored-by: Petr Viktorin <[email protected]>

Author: miss-islington

Lib/distutils/command/install.py

@@ -37,12 +37,19 @@
 
 # Copy from sysconfig._INSTALL_SCHEMES
 for key in SCHEME_KEYS:
-    sys_key = key
-    if key == "headers":
-        sys_key = "include"
-    INSTALL_SCHEMES["unix_prefix"][key] = sysconfig._INSTALL_SCHEMES["posix_prefix"][sys_key]
-    INSTALL_SCHEMES["unix_home"][key] = sysconfig._INSTALL_SCHEMES["posix_home"][sys_key]
-    INSTALL_SCHEMES["nt"][key] = sysconfig._INSTALL_SCHEMES["nt"][sys_key]
+    for distutils_scheme_name, sys_scheme_name in (
+            ("unix_prefix", "posix_prefix"), ("unix_home", "posix_home"),
+            ("nt", "nt")):
+        sys_key = key
+        sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name]
+        if key == "headers" and key not in sys_scheme:
+            # On POSIX-y platofrms, Python will:
+            # - Build from .h files in 'headers' (only there when
+            #   building CPython)
+            # - Install .h files to 'include'
+            # When 'headers' is missing, fall back to 'include'
+            sys_key = 'include'
+        INSTALL_SCHEMES[distutils_scheme_name][key] = sys_scheme[sys_key]
 
 # Transformation to different template format
 for main_key in INSTALL_SCHEMES:
@@ -316,6 +323,9 @@ def finalize_options(self):
             self.config_vars['userbase'] = self.install_userbase
             self.config_vars['usersite'] = self.install_usersite
 
+        if sysconfig.is_python_build(True):
+            self.config_vars['srcdir'] = sysconfig.get_config_var('srcdir')
+
         self.expand_basedirs()
 
         self.dump_dirs("post-expand_basedirs()")

Lib/sysconfig.py

@@ -182,6 +182,18 @@ def is_python_build(check_home=False):
 
 _PYTHON_BUILD = is_python_build(True)
 
+if _PYTHON_BUILD:
+    for scheme in ('posix_prefix', 'posix_home'):
+        # On POSIX-y platofrms, Python will:
+        # - Build from .h files in 'headers' (which is only added to the
+        #   scheme when building CPython)
+        # - Install .h files to 'include'
+        scheme = _INSTALL_SCHEMES[scheme]
+        scheme['headers'] = scheme['include']
+        scheme['include'] = '{srcdir}/Include'
+        scheme['platinclude'] = '{projectbase}/.'
+
+
 def _subst_vars(s, local_vars):
     try:
         return s.format(**local_vars)

Misc/NEWS.d/next/Build/2021-05-24-03-31-17.bpo-41282.L8nP44.rst

@@ -0,0 +1,3 @@
+Fix broken ``make install`` that caused standard library extension modules
+to be unnecessarily and incorrectly rebuilt during the install phase of
+cpython.