diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index bc2197cfe79402..f8fff43713d594 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,13 +1,12 @@ import json import os import sys -import types from sysconfig import ( _ALWAYS_STR, + _PROJECT_BASE, _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, - get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,10 +160,8 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if get_config_var('Py_DEBUG') == '1': - pybuilddir += '-pydebug' - return pybuilddir + with open(os.path.join(_PROJECT_BASE, 'pybuilddir.txt')) as f: + return f.read() def _get_json_data_name(): @@ -203,23 +200,6 @@ def _generate_posix_vars(): name = _get_sysconfigdata_name() - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - # GH-127178: Since we started generating a .json file, we also need this to - # be able to run sysconfig.get_config_vars(). - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -243,10 +223,6 @@ def _generate_posix_vars(): print(f'Written {jsonfile}') - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - def _print_dict(title, data): for index, (key, value) in enumerate(sorted(data.items())): diff --git a/Makefile.pre.in b/Makefile.pre.in index 3ef0c6320c85db..41a7f2ab1b1c3c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -32,6 +32,7 @@ srcdir= @srcdir@ VPATH= @srcdir@ abs_srcdir= @abs_srcdir@ abs_builddir= @abs_builddir@ +PYBUILDDIR= build/lib.@MACHDEP@-@MULTIARCH@-@VERSION@-@ABIFLAGS@ CC= @CC@ @@ -135,6 +136,10 @@ MACHDEP= @MACHDEP@ MULTIARCH= @MULTIARCH@ MULTIARCH_CPPFLAGS = @MULTIARCH_CPPFLAGS@ +# Sysconfig data +SYSCONFIGDATA_NAME= @SYSCONFIGDATA_NAME@ +SYSCONFIGVARS_JSON_NAME= @SYSCONFIGVARS_JSON_NAME@ + # Install prefix for architecture-independent files prefix= @prefix@ @@ -731,11 +736,12 @@ list-targets: .PHONY: build_all build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil pybuilddir.txt \ + $(PYBUILDDIR)/build-details.json $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods + $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py python-config checksharedmods .PHONY: build_emscripten build_emscripten: build_wasm web_example @@ -918,27 +924,37 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) -platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt +platform: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform -# Create build directory and generate the sysconfig build-time data there. -# pybuilddir.txt contains the name of the build dir and is used for -# sys.path fixup -- see Modules/getpath.c. -# Since this step runs before shared modules are built, try to avoid bootstrap -# problems by creating a dummy pybuilddir.txt just to allow interpreter -# initialization to succeed. It will be overwritten by generate-posix-vars -# or removed in case of failure. -pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) - @echo "none" > ./pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ - if test $$? -ne 0 ; then \ - echo "generate-posix-vars failed" ; \ - rm -f ./pybuilddir.txt ; \ - exit 1 ; \ - fi +# Create build directory and pybuilddir.txt. +# pybuilddir.txt which is used by Modules/getpath.c to detect the build directory. +pybuilddir.txt: + mkdir -p $(PYBUILDDIR) + printf $(PYBUILDDIR) >pybuilddir.txt + +SYSCONFIG_SRC= \ + $(srcdir)/Lib/sysconfig/__init__.py \ + $(srcdir)/Lib/sysconfig/__main__.py +SYSCONFIGVARS_DEPS= \ + $(SYSCONFIG_SRC) \ + Makefile \ + Python/sysmodule.o \ + Modules/posixmodule.o + +# Generate the sysconfig data module. +$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py $(PYBUILDDIR)/$(SYSCONFIGVARS_JSON_NAME): $(SYSCONFIGVARS_DEPS) $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars + @# Sanity check, to make sure sysconfig._get_sysconfigdata_name() returns the same value. + @for file in $@; do \ + if test ! -f $<; then \ + echo "generate-posix-vars didn't generate '$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py'" \ + exit 1; \ + fi; \ + done -build-details.json: pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json +$(PYBUILDDIR)/build-details.json: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py Makefile pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py $@ # Build static library $(LIBRARY): $(LIBRARY_OBJS) @@ -1444,18 +1460,8 @@ $(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS) -# create relative links from build/lib.platform/egg.so to Modules/egg.so -# pybuilddir.txt is created too late. We cannot use it in Makefile -# targets. ln --relative is not portable. .PHONY: sharedmods -sharedmods: $(SHAREDMODS) pybuilddir.txt - @target=`cat pybuilddir.txt`; \ - $(MKDIR_P) $$target; \ - for mod in X $(SHAREDMODS); do \ - if test $$mod != X; then \ - $(LN) -sf ../../$$mod $$target/`basename $$mod`; \ - fi; \ - done +sharedmods: $(SHAREDMODS) # dependency on BUILDPYTHON ensures that the target is run last .PHONY: checksharedmods @@ -2657,9 +2663,9 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ - $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/build-details.json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against @@ -3064,7 +3070,7 @@ clean-retain-profile: pycremoval find build -name 'fficonfig.h' -exec rm -f {} ';' || true find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true - -rm -f pybuilddir.txt + -rm -rf pybuilddir.txt $(PYBUILDDIR) -rm -f _bootstrap_python -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module diff --git a/Modules/makesetup b/Modules/makesetup index 8bb971b152a522..68a2724a0fb33b 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -266,7 +266,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | esac for mod in $mods do - file="$srcdir/$mod\$(EXT_SUFFIX)" + file="\$(PYBUILDDIR)/$mod\$(EXT_SUFFIX)" case $doconfig in no) SHAREDMODS="$SHAREDMODS $file" @@ -274,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" + rule="$rule; @mkdir -p \$(PYBUILDDIR); \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" echo "$rule" >>$rulesf done done diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 4a53e0bd1bee1b..e2a6ff2f23428b 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -195,11 +195,10 @@ def configure_emscripten_python(context, working_dir): assert ( len(lib_dirs) == 1 ), f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + _, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2) sysconfig_data = ( - f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + f"{emscripten_build_dir}/build/" + f"lib.emscripten-wasm32-emscripten-{python_version}-{abiflags}" ) if pydebug: sysconfig_data += "-pydebug" @@ -227,8 +226,13 @@ def configure_emscripten_python(context, working_dir): "--enable-wasm-dynamic-linking", f"--prefix={PREFIX_DIR}", ] - if pydebug: - configure.append("--with-pydebug") + for flag in abiflags: + if flag == 'd': + configure.append("--with-pydebug") + elif flag == 't': + configure.append("--disable-gil") + else: + raise ValueError(f"Unknown ABI flag: {flag}") if context.args: configure.extend(context.args) call( diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index da847c4ff86215..acb72e0ec7c1e3 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -212,12 +212,8 @@ def configure_wasi_python(context, working_dir): python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - if pydebug: - sysconfig_data += "-pydebug" + _, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2) + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-wasi-{python_version}-{abiflags}" # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. @@ -244,8 +240,13 @@ def configure_wasi_python(context, working_dir): f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] - if pydebug: - configure.append("--with-pydebug") + for flag in abiflags: + if flag == 'd': + configure.append("--with-pydebug") + elif flag == 't': + configure.append("--disable-gil") + else: + raise ValueError(f"Unknown ABI flag: {flag}") if context.args: configure.extend(context.args) call(configure, diff --git a/configure b/configure index d7153914fe7b5e..7d86114d8186b0 100755 --- a/configure +++ b/configure @@ -1014,6 +1014,8 @@ FREEZE_MODULE FREEZE_MODULE_BOOTSTRAP_DEPS FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE +SYSCONFIGVARS_JSON_NAME +SYSCONFIGDATA_NAME PYTHON_FOR_BUILD host_os host_vendor @@ -3750,7 +3752,7 @@ fi fi ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_SYSCONFIGDATA_PATH=$(abs_builddir)/$(PYBUILDDIR) '$with_build_python { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } @@ -3770,6 +3772,11 @@ fi +SYSCONFIGDATA_NAME='_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)' +SYSCONFIGVARS_JSON_NAME='_sysconfigvars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json' + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for Python interpreter freezing" >&5 printf %s "checking for Python interpreter freezing... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_FREEZE" >&5 diff --git a/configure.ac b/configure.ac index 4e24930662c1f8..5f36d5f2e16f36 100644 --- a/configure.ac +++ b/configure.ac @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python], dnl Build Python interpreter is used for regeneration and freezing. ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_SYSCONFIGDATA_PATH=$(abs_builddir)/$(PYBUILDDIR) '$with_build_python AC_MSG_RESULT([$with_build_python]) ], [ AS_VAR_IF([cross_compiling], [yes], @@ -176,6 +176,11 @@ AC_ARG_WITH([build-python], ) AC_SUBST([PYTHON_FOR_BUILD]) +SYSCONFIGDATA_NAME='_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)' +SYSCONFIGVARS_JSON_NAME='_sysconfigvars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json' +AC_SUBST([SYSCONFIGDATA_NAME]) +AC_SUBST([SYSCONFIGVARS_JSON_NAME]) + AC_MSG_CHECKING([for Python interpreter freezing]) AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE])