Skip to content

bpo-35257: Avoid leaking the linker flags into distutils. #10900

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 19, 2018
Merged

bpo-35257: Avoid leaking the linker flags into distutils. #10900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
25eb0a1
bpo-35257: Avoid leaking the linker flags into distutils.
stratakis Dec 4, 2018
1819b28
Log PY_CORE_LDFLAGS and PY_STDMODULE_CFLAGS at
stratakis Dec 10, 2018
d7ebc21
Refer to the issue number in the bpo-* format in comments.
stratakis Dec 10, 2018
d1655f9
Move setting of the compiler and linker flags to a function.
stratakis Dec 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Lib/_osx_support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
'PY_CORE_CFLAGS')
'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS')

# configuration variables that may contain compiler calls
_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/pythoninfo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,10 @@ def collect_sysconfig(info_add):
'OPT',
'PY_CFLAGS',
'PY_CFLAGS_NODIST',
'PY_CORE_LDFLAGS',
'PY_LDFLAGS',
'PY_LDFLAGS_NODIST',
'PY_STDMODULE_CFLAGS',
'Py_DEBUG',
'Py_ENABLE_SHARED',
'SHELL',
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test__osx_support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def setUp(self):
for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS',
'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC',
'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
'PY_CORE_CFLAGS'):
'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS'):
if cv in self.env:
self.env.unset(cv)

Expand Down
25 changes: 16 additions & 9 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ CONFIGURE_CFLAGS= @CFLAGS@
# Use it when a compiler flag should _not_ be part of the distutils CFLAGS
# once Python is installed (Issue #21121).
CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
# LDFLAGS_NODIST is used in the same manner as CFLAGS_NODIST.
# Use it when a linker flag should _not_ be part of the distutils LDFLAGS
# once Python is installed (bpo-35257)
CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@
CONFIGURE_CPPFLAGS= @CPPFLAGS@
CONFIGURE_LDFLAGS= @LDFLAGS@
# Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
Expand All @@ -96,6 +100,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include
# environment variables
PY_CPPFLAGS= $(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
PY_LDFLAGS= $(CONFIGURE_LDFLAGS) $(LDFLAGS)
PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)
NO_AS_NEEDED= @NO_AS_NEEDED@
SGI_ABI= @SGI_ABI@
CCSHARED= @CCSHARED@
Expand All @@ -107,6 +112,8 @@ CFLAGSFORSHARED=@CFLAGSFORSHARED@
PY_STDMODULE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED)
PY_BUILTIN_MODULE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE_BUILTIN
PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE
# Linker flags used for building the interpreter object files
PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST)
# Strict or non-strict aliasing flags used to compile dtoa.c, see above
CFLAGS_ALIASING=@CFLAGS_ALIASING@

Expand Down Expand Up @@ -146,7 +153,7 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION)
SHLIB_SUFFIX= @SHLIB_SUFFIX@
EXT_SUFFIX= @EXT_SUFFIX@
LDSHARED= @LDSHARED@ $(PY_LDFLAGS)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You would mind to add a comment explaining why PY_LDFLAGS (and not PY_LDFLAGS_NODIST) is used for LDSHARED?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll have to dig in a bit more into the buildsystem to figure out how they are used as my approach was a bit of trial and error for LDSHARED and BLDSHARED. Will try to offer a more comprehensive explanation.

BLDSHARED= @BLDSHARED@ $(PY_LDFLAGS)
BLDSHARED= @BLDSHARED@ $(PY_LDFLAGS_NODIST)
LDCXXSHARED= @LDCXXSHARED@
DESTSHARED= $(BINLIBDEST)/lib-dynload

Expand Down Expand Up @@ -496,7 +503,7 @@ profile-run-stamp:
touch $@

build_all_generate_profile:
$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"

run_profile_task:
@ # FIXME: can't run for a cross build
Expand All @@ -510,7 +517,7 @@ build_all_merge_profile:
profile-opt: profile-run-stamp
@echo "Rebuilding with profile guided optimizations:"
-rm -f profile-clean-stamp
$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)"
$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)"

# Compile and run with gcov
.PHONY=coverage coverage-lcov coverage-report
Expand Down Expand Up @@ -567,7 +574,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c

# Build the interpreter
$(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)

platform: $(BUILDPYTHON) pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
Expand Down Expand Up @@ -632,7 +639,7 @@ libpython3.so: libpython$(LDVERSION).so
$(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^

libpython$(LDVERSION).dylib: $(LIBRARY_OBJS)
$(CC) -dynamiclib -Wl,-single_module $(PY_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \
$(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \


libpython$(VERSION).sl: $(LIBRARY_OBJS)
Expand All @@ -657,7 +664,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \
$(LIBRARY) \
$(RESSRCDIR)/Info.plist
$(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)
$(CC) -o $(LDLIBRARY) $(PY_LDFLAGS) -dynamiclib \
$(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \
-all_load $(LIBRARY) -Wl,-single_module \
-install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \
-compatibility_version $(VERSION) \
Expand Down Expand Up @@ -698,15 +705,15 @@ Makefile Modules/config.c: Makefile.pre \


Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)

############################################################################
# Importlib

Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile

Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
$(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)

.PHONY: regen-importlib
regen-importlib: Programs/_freeze_importlib
Expand Down Expand Up @@ -794,7 +801,7 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile
$(IO_OBJS): $(IO_H)

$(PGEN): $(PGENOBJS)
$(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN)
$(CC) $(OPT) $(PY_CORE_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN)

.PHONY: regen-grammar
regen-grammar: $(PGEN)
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Build/2018-12-05-22-28-40.bpo-35257.dmcd_s.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Avoid leaking the linker flags from Link Time Optimizations (LTO)
into distutils when compiling C extensions.
4 changes: 3 additions & 1 deletion configure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ SHLIB_SUFFIX
LIBTOOL_CRUFT
OTHER_LIBTOOL_OPT
UNIVERSAL_ARCH_FLAGS
LDFLAGS_NODIST
CFLAGS_NODIST
BASECFLAGS
CFLAGS_ALIASING
Expand Down Expand Up @@ -6627,7 +6628,7 @@ $as_echo "$as_me: llvm-ar found via xcrun: ${LLVM_AR}" >&6;}
fi

CFLAGS_NODIST="$CFLAGS_NODIST $LTOFLAGS"
LDFLAGS="$LDFLAGS $LTOFLAGS"
LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS"
fi

# Enable PGO flags.
Expand Down Expand Up @@ -6879,6 +6880,7 @@ fi




# The -arch flags for universal builds on OSX
UNIVERSAL_ARCH_FLAGS=

Expand Down
3 changes: 2 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1358,7 +1358,7 @@ if test "$Py_LTO" = 'true' ; then
fi

CFLAGS_NODIST="$CFLAGS_NODIST $LTOFLAGS"
LDFLAGS="$LDFLAGS $LTOFLAGS"
LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS"
fi

# Enable PGO flags.
Expand Down Expand Up @@ -1518,6 +1518,7 @@ fi

AC_SUBST(BASECFLAGS)
AC_SUBST(CFLAGS_NODIST)
AC_SUBST(LDFLAGS_NODIST)

# The -arch flags for universal builds on OSX
UNIVERSAL_ARCH_FLAGS=
Expand Down
15 changes: 10 additions & 5 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,16 @@

cross_compiling = "_PYTHON_HOST_PLATFORM" in os.environ

# Add special CFLAGS reserved for building the interpreter and the stdlib
# modules (Issue #21121).
cflags = sysconfig.get_config_var('CFLAGS')
py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist
# Set common compiler and linker flags derived from the Makefile,
# reserved for building the interpreter and the stdlib modules.
# See bpo-21121 and bpo-35257
def set_compiler_flags(compiler_flags, compiler_py_flags_nodist):
flags = sysconfig.get_config_var(compiler_flags)
py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist)
sysconfig.get_config_vars()[compiler_flags] = flags + ' ' + py_flags_nodist

set_compiler_flags('CFLAGS', 'PY_CFLAGS_NODIST')
set_compiler_flags('LDFLAGS', 'PY_LDFLAGS_NODIST')

class Dummy:
"""Hack for parallel build"""
Expand Down