From a7099b91a37238f36d9637955f0c99d622568f8b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 31 Mar 2025 20:15:44 +0800 Subject: [PATCH 01/14] Fix `get_config_var('Py_DEBUG')` usage in `sysconfig._get_pybuilddir()` --- Lib/sysconfig/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index bc2197cfe79402..b0c00175afd7e9 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -162,7 +162,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if get_config_var('Py_DEBUG') == '1': + if get_config_var('Py_DEBUG'): pybuilddir += '-pydebug' return pybuilddir From 2cf16e6e2ae5babd4d857c155d86a7d52d262253 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 31 Mar 2025 21:46:11 +0800 Subject: [PATCH 02/14] Remove a branch that always false --- Lib/sysconfig/__main__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index b0c00175afd7e9..14431f06b1c714 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -161,10 +161,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if get_config_var('Py_DEBUG'): - pybuilddir += '-pydebug' - return pybuilddir + return f'build/lib.{get_platform()}-{get_python_version()}' def _get_json_data_name(): From fcb5c179f0305747283e01d2184c8dfcf9c88eae Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:15:13 +0000 Subject: [PATCH 03/14] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst diff --git a/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst b/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst new file mode 100644 index 00000000000000..2ac3c918aabef8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst @@ -0,0 +1 @@ +Remove dead code in :func:`sysconfig._get_pybuilddir`. Patch by Xuehai Pan. From 25ae994827bc2115575e8b5cf639562b034f1726 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 1 Apr 2025 01:19:41 +0800 Subject: [PATCH 04/14] Fix non-exist reference target in news --- .../next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst b/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst index 2ac3c918aabef8..7a669b65c91d10 100644 --- a/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst +++ b/Misc/NEWS.d/next/Library/2025-03-31-17-15-09.gh-issue-127405.E7lPpD.rst @@ -1 +1 @@ -Remove dead code in :func:`sysconfig._get_pybuilddir`. Patch by Xuehai Pan. +Remove dead code in function ``sysconfig._get_pybuilddir``. Patch by Xuehai Pan. From 1bd31c483c77c3a1f35dce01d3add7c39a5a4956 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 01:07:36 +0800 Subject: [PATCH 05/14] Revert back to pre GH-127430 --- Lib/sysconfig/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 14431f06b1c714..44ca343e2f4653 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -161,7 +161,10 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): - return f'build/lib.{get_platform()}-{get_python_version()}' + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + return pybuilddir def _get_json_data_name(): From 15e91490c4bbfc3cc86abd7f10b606878789a05a Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 01:19:24 +0800 Subject: [PATCH 06/14] Try to fix `sysconfig._get_pybuilddir()` --- Lib/sysconfig/__main__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 44ca343e2f4653..a860145d7d4bdd 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -160,9 +160,11 @@ def _print_config_dict(d, stream): print ("}", file=stream) -def _get_pybuilddir(): +def _get_pybuilddir(py_debug=None): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): + if py_debug is None: + py_debug = get_config_var('Py_DEBUG') + if py_debug: pybuilddir += '-pydebug' return pybuilddir @@ -220,7 +222,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = _get_pybuilddir() + pybuilddir = _get_pybuilddir(vars['Py_DEBUG']) os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') From 01a8f944b88456816ebae152d5917db702e3e971 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 02:13:49 +0800 Subject: [PATCH 07/14] Try fix WASM build with Py_DEBUG --- Tools/wasm/wasm_build.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index bcb80212362b71..b5fd9cd6f11fcd 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -332,7 +332,14 @@ def _check_wasi() -> None: "--wasm max-wasm-stack=16777216 " "--wasi preview2 " "--dir {srcdir}::/ " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" + "--env PYTHONPATH=" + + ":".join( + ( + "/{relbuilddir}/build/lib.wasi-wasm32-{version}", + "/{relbuilddir}/build/lib.wasi-wasm32-{version}-pydebug", + "/Lib", + ) + ) ), "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, From 047d5e4afa115a497027f6c7d7b969ce8fd2009d Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 02:22:51 +0800 Subject: [PATCH 08/14] Try fix WASM build with Py_DEBUG --- Tools/wasm/wasm_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index b5fd9cd6f11fcd..0e0985078beef0 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -335,8 +335,8 @@ def _check_wasi() -> None: "--env PYTHONPATH=" + ":".join( ( - "/{relbuilddir}/build/lib.wasi-wasm32-{version}", "/{relbuilddir}/build/lib.wasi-wasm32-{version}-pydebug", + "/{relbuilddir}/build/lib.wasi-wasm32-{version}", "/Lib", ) ) From 949fd4639e5668c1939e1166d77b395e68de9b63 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 02:41:23 +0800 Subject: [PATCH 09/14] Add `/Lib` to PYTHONPATH --- Tools/wasm/wasi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index da847c4ff86215..7a3e614ad934c3 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -224,7 +224,7 @@ def configure_wasi_python(context, working_dir): args = {"GUEST_DIR": "/", "HOST_DIR": CHECKOUT, "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", + "ENV_VAR_VALUE": f"/{sysconfig_data}:/Lib", "PYTHON_WASM": working_dir / "python.wasm"} # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. From cb043128934a2f93877bec941df98edf7830e756 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 03:14:35 +0800 Subject: [PATCH 10/14] Try fix WASM build with Py_DEBUG --- Lib/sysconfig/__main__.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index a860145d7d4bdd..54c12786a6d9fe 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -160,15 +160,6 @@ def _print_config_dict(d, stream): print ("}", file=stream) -def _get_pybuilddir(py_debug=None): - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if py_debug is None: - py_debug = get_config_var('Py_DEBUG') - if py_debug: - pybuilddir += '-pydebug' - return pybuilddir - - def _get_json_data_name(): name = _get_sysconfigdata_name() assert name.startswith('_sysconfigdata') @@ -222,7 +213,10 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = _get_pybuilddir(vars['Py_DEBUG']) + pybuilddir = os.environ.get( + '_PYTHON_SYSCONFIGDATA_PATH', + f'build/lib.{get_platform()}-{get_python_version()}', + ) os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') From 1a6b81bad7cbc27bd889bdffa82a81caf57838e7 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 03:15:40 +0800 Subject: [PATCH 11/14] Try fix WASM build with Py_DEBUG --- Lib/sysconfig/__main__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 54c12786a6d9fe..72cb794d08653c 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -213,10 +213,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = os.environ.get( - '_PYTHON_SYSCONFIGDATA_PATH', - f'build/lib.{get_platform()}-{get_python_version()}', - ) + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') From bc0c675d2df98c01f71b8bf3d1e18c0f85678ca4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 03:21:54 +0800 Subject: [PATCH 12/14] Revert "Try fix WASM build with Py_DEBUG" This reverts commit 1a6b81bad7cbc27bd889bdffa82a81caf57838e7. This reverts commit cb043128934a2f93877bec941df98edf7830e756. --- Lib/sysconfig/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 72cb794d08653c..b0c00175afd7e9 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -160,6 +160,13 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if get_config_var('Py_DEBUG'): + pybuilddir += '-pydebug' + return pybuilddir + + def _get_json_data_name(): name = _get_sysconfigdata_name() assert name.startswith('_sysconfigdata') @@ -213,7 +220,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') From a5af5c850eed3709421f776da0fe194841382b83 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 03:32:56 +0800 Subject: [PATCH 13/14] Try increase stack size --- Tools/wasm/wasi.py | 4 ++-- Tools/wasm/wasm_build.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 7a3e614ad934c3..caf233c5ac4061 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -224,7 +224,7 @@ def configure_wasi_python(context, working_dir): args = {"GUEST_DIR": "/", "HOST_DIR": CHECKOUT, "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}:/Lib", + "ENV_VAR_VALUE": f"/{sysconfig_data}", "PYTHON_WASM": working_dir / "python.wasm"} # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. @@ -296,7 +296,7 @@ def main(): # Make sure the stack size will work for a pydebug # build. # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " + "--wasm max-wasm-stack=33554432 " # Enable thread support; causes use of preview1. #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 0e0985078beef0..9b6874a26e5448 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -329,7 +329,7 @@ def _check_wasi() -> None: # workaround for https://github.com/python/cpython/issues/95952 "HOSTRUNNER": ( "wasmtime run " - "--wasm max-wasm-stack=16777216 " + "--wasm max-wasm-stack=33554432 " "--wasi preview2 " "--dir {srcdir}::/ " "--env PYTHONPATH=" From 732ea149cd9c9379d54c6b7dbe08d1c45d14100c Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Wed, 2 Apr 2025 03:42:22 +0800 Subject: [PATCH 14/14] Try increase stack size --- configure | 5 ++--- configure.ac | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/configure b/configure index a058553480ca5a..75dc073949c411 100755 --- a/configure +++ b/configure @@ -7886,7 +7886,7 @@ then : fi ;; #( WASI) : - HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=33554432 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; @@ -9699,7 +9699,7 @@ then : fi - as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" + as_fn_append LDFLAGS_NODIST " -z stack-size=33554432 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( *) : @@ -35159,4 +35159,3 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi - diff --git a/configure.ac b/configure.ac index 23bd81ed4431b9..52d65ac3956995 100644 --- a/configure.ac +++ b/configure.ac @@ -1638,7 +1638,7 @@ then dnl TODO: support other WASI runtimes dnl wasmtime starts the process with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], + [WASI], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=33554432 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi @@ -2420,7 +2420,7 @@ AS_CASE([$ac_sys_system], dnl gh-117645: Set the memory size to 40 MiB, the stack size to 16 MiB, dnl and move the stack first. dnl https://github.com/WebAssembly/wasi-libc/issues/233 - AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=33554432 -Wl,--stack-first -Wl,--initial-memory=41943040"]) ] )