Skip to content

Commit 4958820

Browse files
authored
[3.11] gh-95853: Add script to automate WASM build (GH-95828, GH-95985, GH-96045, GH-96389, GH-96744) (GH-96749)
Automate WASM build with a new Python script. The script provides several build profiles with configure flags for Emscripten flavors and WASI. The script can detect and use Emscripten SDK and WASI SDK from default locations or env vars. ``configure`` now detects Node arguments and creates HOSTRUNNER arguments for Node 16. It also sets correct arguments for ``wasm64-emscripten``.
1 parent 390123b commit 4958820

16 files changed

+1236
-40
lines changed

Lib/distutils/tests/test_sysconfig.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ def test_get_config_vars(self):
4949
self.assertIsInstance(cvars, dict)
5050
self.assertTrue(cvars)
5151

52+
@unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
5253
def test_srcdir(self):
5354
# See Issues #15322, #15364.
5455
srcdir = sysconfig.get_config_var('srcdir')

Lib/test/test_decimal.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
requires_legacy_unicode_capi, check_sanitizer)
3838
from test.support import (TestFailed,
3939
run_with_locale, cpython_only,
40-
darwin_malloc_err_warning)
40+
darwin_malloc_err_warning, is_emscripten)
4141
from test.support.import_helper import import_fresh_module
4242
from test.support import threading_helper
4343
from test.support import warnings_helper
@@ -5623,6 +5623,7 @@ def __abs__(self):
56235623
# Issue 41540:
56245624
@unittest.skipIf(sys.platform.startswith("aix"),
56255625
"AIX: default ulimit: test is flaky because of extreme over-allocation")
5626+
@unittest.skipIf(is_emscripten, "Test is unstable on Emscripten")
56265627
@unittest.skipIf(check_sanitizer(address=True, memory=True),
56275628
"ASAN/MSAN sanitizer defaults to crashing "
56285629
"instead of returning NULL for malloc failure.")

Lib/test/test_sysconfig.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,7 @@ def test_platform_in_subprocess(self):
439439
self.assertEqual(status, 0)
440440
self.assertEqual(my_platform, test_platform)
441441

442+
@unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
442443
def test_srcdir(self):
443444
# See Issues #15322, #15364.
444445
srcdir = sysconfig.get_config_var('srcdir')

Lib/test/test_unicode_file_functions.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import warnings
77
from unicodedata import normalize
88
from test.support import os_helper
9+
from test import support
910

1011

1112
filenames = [
@@ -123,6 +124,10 @@ def test_open(self):
123124
# NFKD in Python is useless, because darwin will normalize it later and so
124125
# open(), os.stat(), etc. don't raise any exception.
125126
@unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
127+
@unittest.skipIf(
128+
support.is_emscripten or support.is_wasi,
129+
"test fails on Emscripten/WASI when host platform is macOS."
130+
)
126131
def test_normalize(self):
127132
files = set(self.files)
128133
others = set()

Lib/test/test_warnings/__init__.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,14 @@ def test_warn_explicit_non_ascii_filename(self):
489489
module=self.module) as w:
490490
self.module.resetwarnings()
491491
self.module.filterwarnings("always", category=UserWarning)
492-
for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
492+
filenames = ["nonascii\xe9\u20ac"]
493+
if not support.is_emscripten:
494+
# JavaScript does not like surrogates.
495+
# Invalid UTF-8 leading byte 0x80 encountered when
496+
# deserializing a UTF-8 string in wasm memory to a JS
497+
# string!
498+
filenames.append("surrogate\udc80")
499+
for filename in filenames:
493500
try:
494501
os.fsencode(filename)
495502
except UnicodeEncodeError:

Makefile.pre.in

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -817,10 +817,11 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
817817
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
818818
# --preload-file turns a relative asset path into an absolute path.
819819

820+
.PHONY: wasm_stdlib
821+
wasm_stdlib: $(WASM_STDLIB)
820822
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
821823
$(srcdir)/Tools/wasm/wasm_assets.py \
822-
Makefile pybuilddir.txt Modules/Setup.local \
823-
python.html python.worker.js
824+
Makefile pybuilddir.txt Modules/Setup.local
824825
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
825826
--buildroot . --prefix $(prefix)
826827

@@ -1713,6 +1714,10 @@ buildbottest: all
17131714
fi
17141715
$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS)
17151716

1717+
# Like testall, but run Python tests with HOSTRUNNER directly.
1718+
hostrunnertest: all
1719+
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
1720+
17161721
pythoninfo: all
17171722
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
17181723

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and
2+
test steps for building CPython on WebAssembly platforms.
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for
2+
broken EMSDK versions, and warns about pkg-config env vars.

Modules/pyexpat.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -775,7 +775,7 @@ readinst(char *buf, int buf_size, PyObject *meth)
775775
Py_ssize_t len;
776776
const char *ptr;
777777

778-
str = PyObject_CallFunction(meth, "n", buf_size);
778+
str = PyObject_CallFunction(meth, "i", buf_size);
779779
if (str == NULL)
780780
goto error;
781781

Python/sysmodule.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2775,14 +2775,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), {
27752775
if (typeof navigator == 'object') {
27762776
info = navigator.userAgent;
27772777
} else if (typeof process == 'object') {
2778-
info = "Node.js ".concat(process.version)
2778+
info = "Node.js ".concat(process.version);
27792779
} else {
2780-
info = "UNKNOWN"
2780+
info = "UNKNOWN";
27812781
}
27822782
var len = lengthBytesUTF8(info) + 1;
27832783
var res = _malloc(len);
2784-
stringToUTF8(info, res, len);
2784+
if (res) stringToUTF8(info, res, len);
2785+
#if __wasm64__
2786+
return BigInt(res);
2787+
#else
27852788
return res;
2789+
#endif
27862790
});
27872791

27882792
static PyObject *

Tools/wasm/README.md

Lines changed: 75 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
11
# Python WebAssembly (WASM) build
22

3-
**WARNING: WASM support is highly experimental! Lots of features are not working yet.**
3+
**WARNING: WASM support is work-in-progress! Lots of features are not working yet.**
44

55
This directory contains configuration and helpers to facilitate cross
6-
compilation of CPython to WebAssembly (WASM). For now we support
7-
*wasm32-emscripten* builds for modern browser and for *Node.js*. WASI
8-
(*wasm32-wasi*) is work-in-progress
6+
compilation of CPython to WebAssembly (WASM). Python supports Emscripten
7+
(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds
8+
run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds
9+
use WASM runtimes such as *wasmtime*.
10+
11+
Users and developers are encouraged to use the script
12+
`Tools/wasm/wasm_build.py`. The tool automates the build process and provides
13+
assistance with installation of SDKs.
914

1015
## wasm32-emscripten build
1116

@@ -17,7 +22,7 @@ access the file system directly.
1722

1823
Cross compiling to the wasm32-emscripten platform needs the
1924
[Emscripten](https://emscripten.org/) SDK and a build Python interpreter.
20-
Emscripten 3.1.8 or newer are recommended. All commands below are relative
25+
Emscripten 3.1.19 or newer are recommended. All commands below are relative
2126
to a repository checkout.
2227

2328
Christian Heimes maintains a container image with Emscripten SDK, Python
@@ -35,7 +40,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.
3540

3641
### Compile a build Python interpreter
3742

38-
From within the container, run the following commands:
43+
From within the container, run the following command:
44+
45+
```shell
46+
./Tools/wasm/wasm_build.py build
47+
```
48+
49+
The command is roughly equivalent to:
3950

4051
```shell
4152
mkdir -p builddir/build
@@ -45,13 +56,13 @@ make -j$(nproc)
4556
popd
4657
```
4758

48-
### Fetch and build additional emscripten ports
59+
### Cross-compile to wasm32-emscripten for browser
4960

5061
```shell
51-
embuilder build zlib bzip2
62+
./Tools/wasm/wasm_build.py emscripten-browser
5263
```
5364

54-
### Cross compile to wasm32-emscripten for browser
65+
The command is roughly equivalent to:
5566

5667
```shell
5768
mkdir -p builddir/emscripten-browser
@@ -85,22 +96,29 @@ and header files with debug builds.
8596
### Cross compile to wasm32-emscripten for node
8697

8798
```shell
88-
mkdir -p builddir/emscripten-node
89-
pushd builddir/emscripten-node
99+
./Tools/wasm/wasm_build.py emscripten-browser-dl
100+
```
101+
102+
The command is roughly equivalent to:
103+
104+
```shell
105+
mkdir -p builddir/emscripten-node-dl
106+
pushd builddir/emscripten-node-dl
90107

91108
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
92109
emconfigure ../../configure -C \
93110
--host=wasm32-unknown-emscripten \
94111
--build=$(../../config.guess) \
95112
--with-emscripten-target=node \
113+
--enable-wasm-dynamic-linking \
96114
--with-build-python=$(pwd)/../build/python
97115

98116
emmake make -j$(nproc)
99117
popd
100118
```
101119

102120
```shell
103-
node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js
121+
node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js
104122
```
105123

106124
(``--experimental-wasm-bigint`` is not needed with recent NodeJS versions)
@@ -199,6 +217,15 @@ Node builds use ``NODERAWFS``.
199217
- Node RawFS allows direct access to the host file system without need to
200218
perform ``FS.mount()`` call.
201219

220+
## wasm64-emscripten
221+
222+
- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``.
223+
- ``EM_JS`` functions must return ``BigInt()``.
224+
- ``Py_BuildValue()`` format strings must match size of types. Confusing 32
225+
and 64 bits types leads to memory corruption, see
226+
[gh-95876](https://github.com/python/cpython/issues/95876) and
227+
[gh-95878](https://github.com/python/cpython/issues/95878).
228+
202229
# Hosting Python WASM builds
203230

204231
The simple REPL terminal uses SharedArrayBuffer. For security reasons
@@ -234,6 +261,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as
234261
``pkg-config`` overrides. The script assumes that WASI-SDK is installed in
235262
``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
236263

264+
```shell
265+
./Tools/wasm/wasm_build.py wasi
266+
```
267+
268+
The command is roughly equivalent to:
269+
237270
```shell
238271
mkdir -p builddir/wasi
239272
pushd builddir/wasi
@@ -308,26 +341,46 @@ if os.name == "posix":
308341
```python
309342
>>> import os, sys
310343
>>> os.uname()
311-
posix.uname_result(sysname='Emscripten', nodename='emscripten', release='1.0', version='#1', machine='wasm32')
344+
posix.uname_result(
345+
sysname='Emscripten',
346+
nodename='emscripten',
347+
release='3.1.19',
348+
version='#1',
349+
machine='wasm32'
350+
)
312351
>>> os.name
313352
'posix'
314353
>>> sys.platform
315354
'emscripten'
316355
>>> sys._emscripten_info
317356
sys._emscripten_info(
318-
emscripten_version=(3, 1, 8),
319-
runtime='Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0',
357+
emscripten_version=(3, 1, 10),
358+
runtime='Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
320359
pthreads=False,
321360
shared_memory=False
322361
)
362+
```
363+
364+
```python
323365
>>> sys._emscripten_info
324-
sys._emscripten_info(emscripten_version=(3, 1, 8), runtime='Node.js v14.18.2', pthreads=True, shared_memory=True)
366+
sys._emscripten_info(
367+
emscripten_version=(3, 1, 19),
368+
runtime='Node.js v14.18.2',
369+
pthreads=True,
370+
shared_memory=True
371+
)
325372
```
326373

327374
```python
328375
>>> import os, sys
329376
>>> os.uname()
330-
posix.uname_result(sysname='wasi', nodename='(none)', release='0.0.0', version='0.0.0', machine='wasm32')
377+
posix.uname_result(
378+
sysname='wasi',
379+
nodename='(none)',
380+
release='0.0.0',
381+
version='0.0.0',
382+
machine='wasm32'
383+
)
331384
>>> os.name
332385
'posix'
333386
>>> sys.platform
@@ -418,7 +471,8 @@ embuilder build --pic zlib bzip2 MINIMAL_PIC
418471

419472
**NOTE**: WASI-SDK's clang may show a warning on Fedora:
420473
``/lib64/libtinfo.so.6: no version information available``,
421-
[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587).
474+
[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587). The
475+
warning can be ignored.
422476

423477
```shell
424478
export WASI_VERSION=16
@@ -443,6 +497,8 @@ ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime
443497

444498
### WASI debugging
445499

446-
* ``wasmtime run -g`` generates debugging symbols for gdb and lldb.
500+
* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The
501+
feature is currently broken, see
502+
https://github.com/bytecodealliance/wasmtime/issues/4669 .
447503
* The environment variable ``RUST_LOG=wasi_common`` enables debug and
448504
trace logging.

Tools/wasm/wasi-env

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,4 +72,5 @@ export CFLAGS LDFLAGS
7272
export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR
7373
export PATH
7474

75-
exec "$@"
75+
# no exec, it makes arvg[0] path absolute.
76+
"$@"

Tools/wasm/wasm_assets.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,14 @@
116116
"unittest/test/",
117117
)
118118

119+
SYSCONFIG_NAMES = (
120+
"_sysconfigdata__emscripten_wasm32-emscripten",
121+
"_sysconfigdata__emscripten_wasm32-emscripten",
122+
"_sysconfigdata__wasi_wasm32-wasi",
123+
"_sysconfigdata__wasi_wasm64-wasi",
124+
)
125+
126+
119127
def get_builddir(args: argparse.Namespace) -> pathlib.Path:
120128
"""Get builddir path from pybuilddir.txt
121129
"""
@@ -128,7 +136,11 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path:
128136
"""Get path to sysconfigdata relative to build root
129137
"""
130138
data_name = sysconfig._get_sysconfigdata_name()
131-
assert "emscripten_wasm32" in data_name
139+
if not data_name.startswith(SYSCONFIG_NAMES):
140+
raise ValueError(
141+
f"Invalid sysconfig data name '{data_name}'.",
142+
SYSCONFIG_NAMES
143+
)
132144
filename = data_name + ".py"
133145
return args.builddir / filename
134146

0 commit comments

Comments
 (0)