From 4af6fe76eb318b8b1519650ea65d021b95bbad8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 16 Dec 2018 18:00:42 +0100 Subject: [PATCH 1/2] bpo-35499: make profile-opt don't override CFLAGS_NODIST (GH-11164) "make profile-opt" no longer replaces CFLAGS_NODIST with CFLAGS. It now adds profile-guided optimization (PGO) flags to CFLAGS_NODIST, existing CFLAGS_NODIST flags are kept. (cherry picked from commit 640ed520dd6a43a8bf470b79542f58b5d57af9de) --- Makefile.pre.in | 4 ++-- .../next/Build/2018-12-14-19-36-05.bpo-35499.9yAldM.rst | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2018-12-14-19-36-05.bpo-35499.9yAldM.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 258236d3645096..af9504dfac7f3b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -472,7 +472,7 @@ profile-opt: $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" + $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build @@ -482,7 +482,7 @@ build_all_merge_profile: $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)" + $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)" # Compile and run with gcov .PHONY=coverage coverage-lcov coverage-report diff --git a/Misc/NEWS.d/next/Build/2018-12-14-19-36-05.bpo-35499.9yAldM.rst b/Misc/NEWS.d/next/Build/2018-12-14-19-36-05.bpo-35499.9yAldM.rst new file mode 100644 index 00000000000000..ed730b9d9b4a49 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-12-14-19-36-05.bpo-35499.9yAldM.rst @@ -0,0 +1,3 @@ +``make profile-opt`` no longer replaces ``CFLAGS_NODIST`` with ``CFLAGS``. It +now adds profile-guided optimization (PGO) flags to ``CFLAGS_NODIST``: existing +``CFLAGS_NODIST`` flags are kept. From f6a10d4289832964c176effd3f9dcff01f523d2f Mon Sep 17 00:00:00 2001 From: stratakis Date: Wed, 19 Dec 2018 18:19:01 +0100 Subject: [PATCH 2/2] bpo-35257: Avoid leaking LTO linker flags into distutils (GH-10900) When compiling 3rd party C extensions, the linker flags used by the compiler for the interpreter and the stdlib modules, will get leaked into distutils. In order to avoid that, the PY_CORE_LDFLAGS and PY_LDFLAGS_NODIST are introduced to keep those flags separated. (cherry picked from commit cf10a750f4b50b6775719cfb17bee00bc3a9c60b) --- Lib/_osx_support.py | 2 +- Lib/test/pythoninfo.py | 2 ++ Lib/test/test__osx_support.py | 2 +- Makefile.pre.in | 25 ++++++++++++------- .../2018-12-05-22-28-40.bpo-35257.dmcd_s.rst | 2 ++ configure | 4 ++- configure.ac | 3 ++- setup.py | 15 +++++++---- 8 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2018-12-05-22-28-40.bpo-35257.dmcd_s.rst diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e37852e2536c33..db6674ea293fb0 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -17,7 +17,7 @@ _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS') + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') # configuration variables that may contain compiler calls _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index c453340b80d7c5..c5586b45a5ff56 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -407,7 +407,9 @@ def collect_sysconfig(info_add): 'OPT', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', + 'PY_CORE_LDFLAGS', 'PY_LDFLAGS', + 'PY_LDFLAGS_NODIST', 'Py_DEBUG', 'Py_ENABLE_SHARED', 'SHELL', diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index bcba8caa296484..388a2b1a84b17c 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -24,7 +24,7 @@ def setUp(self): for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS'): + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS'): if cv in self.env: self.env.unset(cv) diff --git a/Makefile.pre.in b/Makefile.pre.in index af9504dfac7f3b..da259f89de6304 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -84,6 +84,10 @@ CONFIGURE_CFLAGS= @CFLAGS@ # Use it when a compiler flag should _not_ be part of the distutils CFLAGS # once Python is installed (Issue #21121). CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@ +# LDFLAGS_NODIST is used in the same manner as CFLAGS_NODIST. +# Use it when a linker flag should _not_ be part of the distutils LDFLAGS +# once Python is installed (bpo-35257) +CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@ CONFIGURE_CPPFLAGS= @CPPFLAGS@ CONFIGURE_LDFLAGS= @LDFLAGS@ # Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the @@ -96,6 +100,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) # environment variables PY_CPPFLAGS= $(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS) PY_LDFLAGS= $(CONFIGURE_LDFLAGS) $(LDFLAGS) +PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST) NO_AS_NEEDED= @NO_AS_NEEDED@ LDLAST= @LDLAST@ SGI_ABI= @SGI_ABI@ @@ -106,6 +111,8 @@ ARFLAGS= @ARFLAGS@ CFLAGSFORSHARED=@CFLAGSFORSHARED@ # C flags used for building the interpreter object files PY_CORE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE +# Linker flags used for building the interpreter object files +PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ @@ -145,7 +152,7 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) -BLDSHARED= @BLDSHARED@ $(PY_LDFLAGS) +BLDSHARED= @BLDSHARED@ $(PY_LDFLAGS_NODIST) LDCXXSHARED= @LDCXXSHARED@ DESTSHARED= $(BINLIBDEST)/lib-dynload @@ -472,7 +479,7 @@ profile-opt: $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" + $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build @@ -482,7 +489,7 @@ build_all_merge_profile: $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)" + $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)" # Compile and run with gcov .PHONY=coverage coverage-lcov coverage-report @@ -543,7 +550,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -608,7 +615,7 @@ libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(SHLIBS) $(LIBC) $(LIBM) $(LDLAST); \ + $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(SHLIBS) $(LIBC) $(LIBM) $(LDLAST); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) @@ -633,7 +640,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LIBRARY) \ $(RESSRCDIR)/Info.plist $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) - $(CC) -o $(LDLIBRARY) $(PY_LDFLAGS) -dynamiclib \ + $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ -all_load $(LIBRARY) -Wl,-single_module \ -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ @@ -686,7 +693,7 @@ Modules/Setup: $(srcdir)/Modules/Setup.dist fi Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) ############################################################################ # Importlib @@ -694,7 +701,7 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) .PHONY: regen-importlib regen-importlib: Programs/_freeze_importlib @@ -776,7 +783,7 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(IO_OBJS): $(IO_H) $(PGEN): $(PGENOBJS) - $(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) + $(CC) $(OPT) $(PY_CORE_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) .PHONY: regen-grammar regen-grammar: $(PGEN) diff --git a/Misc/NEWS.d/next/Build/2018-12-05-22-28-40.bpo-35257.dmcd_s.rst b/Misc/NEWS.d/next/Build/2018-12-05-22-28-40.bpo-35257.dmcd_s.rst new file mode 100644 index 00000000000000..fad25257829939 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-12-05-22-28-40.bpo-35257.dmcd_s.rst @@ -0,0 +1,2 @@ +Avoid leaking the linker flags from Link Time Optimizations (LTO) +into distutils when compiling C extensions. \ No newline at end of file diff --git a/configure b/configure index 7286e6eb494d50..e39c16eee2b903 100755 --- a/configure +++ b/configure @@ -666,6 +666,7 @@ SHLIB_SUFFIX LIBTOOL_CRUFT OTHER_LIBTOOL_OPT UNIVERSAL_ARCH_FLAGS +LDFLAGS_NODIST CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING @@ -6716,7 +6717,7 @@ $as_echo "$as_me: llvm-ar found via xcrun: ${LLVM_AR}" >&6;} fi CFLAGS_NODIST="$CFLAGS_NODIST $LTOFLAGS" - LDFLAGS="$LDFLAGS $LTOFLAGS" + LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" fi # Enable PGO flags. @@ -6968,6 +6969,7 @@ fi + # The -arch flags for universal builds on OSX UNIVERSAL_ARCH_FLAGS= diff --git a/configure.ac b/configure.ac index bb14a87e5463b3..cf280506bd011f 100644 --- a/configure.ac +++ b/configure.ac @@ -1401,7 +1401,7 @@ if test "$Py_LTO" = 'true' ; then fi CFLAGS_NODIST="$CFLAGS_NODIST $LTOFLAGS" - LDFLAGS="$LDFLAGS $LTOFLAGS" + LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" fi # Enable PGO flags. @@ -1561,6 +1561,7 @@ fi AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) +AC_SUBST(LDFLAGS_NODIST) # The -arch flags for universal builds on OSX UNIVERSAL_ARCH_FLAGS= diff --git a/setup.py b/setup.py index 49193f67a6117a..fb95fb5298f36e 100644 --- a/setup.py +++ b/setup.py @@ -18,11 +18,16 @@ cross_compiling = "_PYTHON_HOST_PLATFORM" in os.environ -# Add special CFLAGS reserved for building the interpreter and the stdlib -# modules (Issue #21121). -cflags = sysconfig.get_config_var('CFLAGS') -py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') -sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist +# Set common compiler and linker flags derived from the Makefile, +# reserved for building the interpreter and the stdlib modules. +# See bpo-21121 and bpo-35257 +def set_compiler_flags(compiler_flags, compiler_py_flags_nodist): + flags = sysconfig.get_config_var(compiler_flags) + py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist) + sysconfig.get_config_vars()[compiler_flags] = flags + ' ' + py_flags_nodist + +set_compiler_flags('CFLAGS', 'PY_CFLAGS_NODIST') +set_compiler_flags('LDFLAGS', 'PY_LDFLAGS_NODIST') class Dummy: """Hack for parallel build"""