From bbb2ab94b66c53fe5d0bfeca32f7eaca845585ea Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 4 Oct 2024 16:05:25 +0200 Subject: [PATCH 01/11] gh-127111: Make web example work again I moved the web example from `Tools/wasm` into `Tools/wasm/emscripten/web_example`. I also added a new target `build_emscripten` which is `build_wasm` but also builds the web_example. The web_example needs: 1. python.html, copied 2. python.worker.mjs copied 3. python.mjs and python.wasm output from the main linking of the Python interpreter 4. The webserver that sets COOP and COEP 5. python3.14.zip This last is created by the `wasm_assets.py` script, which required a pretty small set of changes to work fine for us. The last thing that should be done is the `python.worker.mjs` script should be made independent of the Python version: currently 3.14 is hard coded. I ran into trouble doing this, so maybe I can leave it to a followup. --- Makefile.pre.in | 51 ++++++++++++------- .../{ => emscripten/web_example}/python.html | 4 +- .../web_example/python.worker.mjs} | 23 +++++++-- .../web_example/server.py} | 8 +-- .../web_example}/wasm_assets.py | 25 +++++---- configure | 13 +++-- configure.ac | 15 +++--- 7 files changed, 83 insertions(+), 56 deletions(-) rename Tools/wasm/{ => emscripten/web_example}/python.html (99%) rename Tools/wasm/{python.worker.js => emscripten/web_example/python.worker.mjs} (71%) rename Tools/wasm/{wasm_webserver.py => emscripten/web_example/server.py} (85%) rename Tools/wasm/{ => emscripten/web_example}/wasm_assets.py (91%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8d94ba361fd934..cb19e4db80a502 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@ # Other subdirectories SUBDIRSTOO= Include Lib Misc -# assets for Emscripten browser builds -WASM_ASSETS_DIR=.$(prefix) -WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py - # Files and directories to be distributed CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in DISTFILES= README.rst ChangeLog $(CONFIGFILES) @@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods +.PHONY: build_emscripten +build_emscripten: build_wasm web_example + # Check that the source is clean when building out of source. .PHONY: check-clean-src check-clean-src: @@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten browser build -# wasm assets directory is relative to current build dir, e.g. "./usr/local". -# --preload-file turns a relative asset path into an absolute path. +# wasm32-emscripten browser web example + +WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +web_example/python.html: $(WEBEX_DIR)/python.html + @mkdir -p web_example + @cp $< $@ + +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ -.PHONY: wasm_stdlib -wasm_stdlib: $(WASM_STDLIB) -$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(srcdir)/Tools/wasm/wasm_assets.py \ +web_example/server.py: $(WEBEX_DIR)/server.py + @mkdir -p web_example + @cp $< $@ + +WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip +$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(WEBEX_DIR)/wasm_assets.py \ Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ - --buildroot . --prefix $(prefix) + $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -python.html: $(srcdir)/Tools/wasm/python.html python.worker.js - @cp $(srcdir)/Tools/wasm/python.html $@ +web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + cp python.mjs web_example/python.mjs + cp python.wasm web_example/python.wasm -python.worker.js: $(srcdir)/Tools/wasm/python.worker.js - @cp $(srcdir)/Tools/wasm/python.worker.js $@ +.PHONY: web_example +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files @@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -f python.html python*.js python.data python*.symbols python*.map - -rm -f $(WASM_STDLIB) + -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Tools/wasm/python.html b/Tools/wasm/emscripten/web_example/python.html similarity index 99% rename from Tools/wasm/python.html rename to Tools/wasm/emscripten/web_example/python.html index 81a035a5c4cd93..fae1e9ad4e8acb 100644 --- a/Tools/wasm/python.html +++ b/Tools/wasm/emscripten/web_example/python.html @@ -47,7 +47,7 @@ async initialiseWorker() { if (!this.worker) { - this.worker = new Worker(this.workerURL) + this.worker = new Worker(this.workerURL, {type: "module"}) this.worker.addEventListener('message', this.handleMessageFromWorker) } } @@ -347,7 +347,7 @@ programRunning(false) } - const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback) + const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback) } diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/emscripten/web_example/python.worker.mjs similarity index 71% rename from Tools/wasm/python.worker.js rename to Tools/wasm/emscripten/web_example/python.worker.mjs index 4ce4e16fc0fa19..a549edf53c5a30 100644 --- a/Tools/wasm/python.worker.js +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -1,3 +1,5 @@ +import createEmscriptenModule from "./python.mjs"; + class StdinBuffer { constructor() { this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) @@ -59,24 +61,38 @@ const stderr = (charCode) => { const stdinBuffer = new StdinBuffer() -var Module = { +const emscriptenSettings = { noInitialRun: true, stdin: stdinBuffer.stdin, stdout: stdout, stderr: stderr, onRuntimeInitialized: () => { postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) + }, + async preRun(Module) { + // TODO: remove fixed version number + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.removeRunDependency("install-stdlib"); } } -onmessage = (event) => { +const modulePromise = createEmscriptenModule(emscriptenSettings); + + +onmessage = async (event) => { if (event.data.type === 'run') { + const Module = await modulePromise; if (event.data.files) { for (const [filename, contents] of Object.entries(event.data.files)) { Module.FS.writeFile(filename, contents) } } - const ret = callMain(event.data.args) + const ret = Module.callMain(event.data.args); postMessage({ type: 'finished', returnCode: ret @@ -84,4 +100,3 @@ onmessage = (event) => { } } -importScripts('python.js') diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/emscripten/web_example/server.py similarity index 85% rename from Tools/wasm/wasm_webserver.py rename to Tools/wasm/emscripten/web_example/server.py index 3d1d5d42a1e8c4..768e6f84e07798 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -14,13 +14,6 @@ class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): - extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) - def end_headers(self) -> None: self.send_my_headers() super().end_headers() @@ -42,5 +35,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main() diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/emscripten/web_example/wasm_assets.py similarity index 91% rename from Tools/wasm/wasm_assets.py rename to Tools/wasm/emscripten/web_example/wasm_assets.py index ffa5e303412c46..7f0fa7ae7c10ec 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/emscripten/web_example/wasm_assets.py @@ -19,7 +19,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +SRCDIR = pathlib.Path(__file__).parents[4].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -28,9 +28,7 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = ( - WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" -) +WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: assert isinstance(args.builddir, pathlib.Path) data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] if not data_name.startswith(SYSCONFIG_NAMES): - raise ValueError( - f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES - ) + raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES) filename = data_name + ".py" return args.builddir / filename @@ -131,7 +127,7 @@ def filterfunc(filename: str) -> bool: return pathname not in args.omit_files_absolute with zipfile.PyZipFile( - args.wasm_stdlib_zip, + args.output, mode="w", compression=args.compression, optimize=optimize, @@ -195,6 +191,12 @@ def path(val: str) -> pathlib.Path: default=pathlib.Path("/usr/local"), type=path, ) +parser.add_argument( + "-o", + "--output", + help="output file", + type=path, +) def main() -> None: @@ -204,7 +206,6 @@ def main() -> None: args.srcdir = SRCDIR args.srcdir_lib = SRCDIR_LIB args.wasm_root = args.buildroot / relative_prefix - args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP args.wasm_stdlib = args.wasm_root / WASM_STDLIB args.wasm_dynload = args.wasm_root / WASM_DYNLOAD @@ -234,12 +235,10 @@ def main() -> None: args.wasm_dynload.mkdir(parents=True, exist_ok=True) marker = args.wasm_dynload / ".empty" marker.touch() - # os.py is a marker for finding the correct lib directory. - shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) # The rest of stdlib that's useful in a WASM context. create_stdlib_zip(args) - size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2) - parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") + size = round(args.output.stat().st_size / 1024**2, 2) + parser.exit(0, f"Created {args.output} ({size} MiB)\n") if __name__ == "__main__": diff --git a/configure b/configure index 84b74ac3584bcd..05f0022c687a0a 100755 --- a/configure +++ b/configure @@ -8335,8 +8335,12 @@ fi fi -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - DEF_MAKE_ALL_RULE="build_wasm" +elif test "$ac_sys_system" = "Emscripten"; then + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" else @@ -9427,12 +9431,12 @@ else $as_nop wasm_debug=no fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520" + as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9449,7 +9453,6 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH" as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" diff --git a/configure.ac b/configure.ac index 8fa6cb60900ad1..5b408afa11bec4 100644 --- a/configure.ac +++ b/configure.ac @@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" ], [], [-Werror]) ]) -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - dnl Emscripten does not support shared extensions yet. Build - dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. +elif test "$ac_sys_system" = "Emscripten"; then + dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + dnl Build "python.wasm", "pybuilddir.txt", and "platform" files. DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" @@ -2321,14 +2325,14 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2339,7 +2343,6 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"]) dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" From 7144b5d6e9dacd51a2ac32a9cad8259217e23887 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 29 Nov 2024 11:29:28 +0100 Subject: [PATCH 02/11] Add workaround for defective mac coreutils --- Tools/wasm/emscripten/__main__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 9ce8dd6a364ad6..a6648cf35dc8d6 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -215,12 +215,29 @@ def configure_emscripten_python(context, working_dir): exec_script = working_dir / "python.sh" exec_script.write_text( dedent( - f"""\ + """\ #!/bin/sh + # Macs come with a defective fork of coreutils so feature detect and + # work around it. + if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which readlink > /dev/null && realpath --version | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () { + echo "$(cd $(dirname "$1");pwd)/$(basename "$2")" + } + REALPATH=abs_path + fi + """ + f"""\ # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$(realpath -s $0)" "$@" + exec {host_runner} {node_entry} "$($REALPATH $0)" "$@" """ ) ) From ab3df6dd6637677059861ced3490620b2f271066 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 30 Nov 2024 16:35:07 +0100 Subject: [PATCH 03/11] Update readme --- Tools/wasm/README.md | 106 +++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 33 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 3f4211fb1dfb28..1f5e3b2619cdd4 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable. export EM_COMPILER_WRAPPER=ccache ``` -### Compile and build Python interpreter +#### Compile and build Python interpreter You can use `python Tools/wasm/emscripten` to compile and build targetting Emscripten. You can do everything at once with: @@ -70,6 +70,78 @@ instance, to do a debug build, you can use: python Tools/wasm/emscripten build --with-py-debug ``` +### Running from node + +If you want to run the normal Python cli, you can use `python.sh`. It takes the +same options as the normal Python cli entrypoint, though the REPL does not +function and will crash. + +`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the +Python process and starts it up with the appropriate settings. If you wish to +make a node application that "embeds" the interpreter instead of acting like the +CLI you will need to write your own alternative to `node_entry.mjs`. + + +### The Web Example + +When building for Emscripten, the web example will be built automatically. It is +in the ``web_example`` directory. The web example uses ``SharedArrayBuffer``. +For security reasons browsers only provide ``SharedArrayBuffer`` in secure +environments with cross-origin isolation. The webserver must send cross-origin +headers and correct MIME types for the JavaScript and WebAssembly files. +Otherwise the terminal will fail to load with an error message like +``ReferenceError: SharedArrayBuffer is not defined``. See more information here: +https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements + +If you serve the web example with ``python server.py`` and then visit +``localhost:8000`` in a browser it should work. + +Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the +web example. If cross-origin isolation is not appropriate for your use case you +may make your own application embedding `python.mjs` which does not use +``SharedArrayBuffer`` and serve it without the cross-origin isolation headers. + +### Embedding Python in a custom JavaScript application + +You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a +minimum you must import ``createEmscriptenModule`` and you need to call +``createEmscriptenModule`` with an appropriate settings object. This settings +object will need a prerun hook that installs the Python standard library into +the Emscripten file system. + +#### NodeJs + +In Node, you can use the NodeFS to mount the standard library in your native +file system into the Emscripten file system: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: "/path/to/python/stdlib" }, "/lib/"); + } +}); +``` + +#### Browser + +In the browser, the simplest approach is to put the standard library in a zip +file it and install it. With Python 3.14 this could look like: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.removeRunDependency("install-stdlib"); + } +}); +``` + ### Limitations and issues #### Network stack @@ -151,38 +223,6 @@ python Tools/wasm/emscripten build --with-py-debug - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -### wasm32-emscripten in node - -Node builds use ``NODERAWFS``. - -- Node RawFS allows direct access to the host file system without need to - perform ``FS.mount()`` call. - -### Hosting Python WASM builds - -The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environments with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WASM files. Otherwise the terminal will fail to load -with an error message like ``Browsers disable shared array buffer``. - -#### Apache HTTP .htaccess - -Place a ``.htaccess`` file in the same directory as ``python.wasm``. - -``` -# .htaccess -Header set Cross-Origin-Opener-Policy same-origin -Header set Cross-Origin-Embedder-Policy require-corp - -AddType application/javascript js -AddType application/wasm wasm - - - AddOutputFilterByType DEFLATE text/html application/javascript application/wasm - -``` - ## WASI (wasm32-wasi) See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). From da6adafd1f0f9c5af4035abfd011acdb92308eb6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 30 Nov 2024 16:36:14 +0100 Subject: [PATCH 04/11] Add news entry --- .../next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst new file mode 100644 index 00000000000000..a3f5a1d785aa9f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst @@ -0,0 +1,2 @@ +Updated the Emscripten web example to use ES6 modules and be built into a +distinct `web_example` subfolder. From 5fc6bc34348288c403895e4eea71e4bb4d7eb98b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 1 Dec 2024 12:32:30 +0100 Subject: [PATCH 05/11] Double backticks --- .../next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst index a3f5a1d785aa9f..d90067cd3bfaa3 100644 --- a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst +++ b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst @@ -1,2 +1,2 @@ Updated the Emscripten web example to use ES6 modules and be built into a -distinct `web_example` subfolder. +distinct ``web_example`` subfolder. From 8a7e3224bf201fbe60f5bdc29af0f4829ca02f67 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 11:26:26 +0100 Subject: [PATCH 06/11] Adjust readme --- Tools/wasm/README.md | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 1f5e3b2619cdd4..e1fba2dd7d831d 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -85,17 +85,19 @@ CLI you will need to write your own alternative to `node_entry.mjs`. ### The Web Example When building for Emscripten, the web example will be built automatically. It is -in the ``web_example`` directory. The web example uses ``SharedArrayBuffer``. -For security reasons browsers only provide ``SharedArrayBuffer`` in secure -environments with cross-origin isolation. The webserver must send cross-origin -headers and correct MIME types for the JavaScript and WebAssembly files. -Otherwise the terminal will fail to load with an error message like -``ReferenceError: SharedArrayBuffer is not defined``. See more information here: +in the ``web_example`` directory. To run the web example, ``cd`` into the +``web_example`` directory, then run ``python server.py``. This will start a web +server; you can then visit ``http://localhost:8000/python.html`` in a browser to +see a simple REPL example. + +The web example uses ``SharedArrayBuffer``. For security reasons browsers only +provide ``SharedArrayBuffer`` in secure environments with cross-origin +isolation. The webserver must send cross-origin headers and correct MIME types +for the JavaScript and WebAssembly files. Otherwise the terminal will fail to +load with an error message like ``ReferenceError: SharedArrayBuffer is not +defined``. See more information here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements -If you serve the web example with ``python server.py`` and then visit -``localhost:8000`` in a browser it should work. - Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the web example. If cross-origin isolation is not appropriate for your use case you may make your own application embedding `python.mjs` which does not use @@ -117,9 +119,13 @@ file system into the Emscripten file system: import createEmscriptenModule from "./python.mjs"; await createEmscriptenModule({ - preRun(Module) { - Module.FS.mount(Module.FS.filesystems.NODEFS, { root: "/path/to/python/stdlib" }, "/lib/"); - } + preRun(Module) { + Module.FS.mount( + Module.FS.filesystems.NODEFS, + { root: "/path/to/python/stdlib" }, + "/lib/", + ); + }, }); ``` @@ -131,14 +137,16 @@ file it and install it. With Python 3.14 this could look like: import createEmscriptenModule from "./python.mjs"; await createEmscriptenModule({ - preRun(Module) { + async preRun(Module) { Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); Module.addRunDependency("install-stdlib"); const resp = await fetch("python3.14.zip"); const stdlibBuffer = await resp.arrayBuffer(); - Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { + canOwn: true, + }); Module.removeRunDependency("install-stdlib"); - } + }, }); ``` From aea9bfe8d71e06dc148667b28b26dc4aabdce6b0 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 11:33:32 +0100 Subject: [PATCH 07/11] Update shell scripts Apply shell check fixes don't disparage BSD coreutils fix indentation realpath not readlink --- Tools/wasm/emscripten/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index a6648cf35dc8d6..b99b74e3661060 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -218,26 +218,26 @@ def configure_emscripten_python(context, working_dir): """\ #!/bin/sh - # Macs come with a defective fork of coreutils so feature detect and - # work around it. + # Macs come with free BSD coreutils which doesn't have the -s option + # so feature detect and work around it. if which grealpath > /dev/null; then # It has brew installed gnu core utils, use that REALPATH="grealpath -s" - elif which readlink > /dev/null && realpath --version | grep GNU > /dev/null; then + elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then # realpath points to GNU realpath so use it. REALPATH="realpath -s" else # Shim for macs without GNU coreutils abs_path () { - echo "$(cd $(dirname "$1");pwd)/$(basename "$2")" + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" } REALPATH=abs_path fi """ - f"""\ + f""" # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$($REALPATH $0)" "$@" + exec {host_runner} {node_entry} "$($REALPATH "$0")" "$@" """ ) ) From 6711e0e9edff2696f8b9f6ad38514531e67b49de Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 11:35:10 +0100 Subject: [PATCH 08/11] Capitalize CLI --- Tools/wasm/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index e1fba2dd7d831d..21b888af88c471 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -72,8 +72,8 @@ python Tools/wasm/emscripten build --with-py-debug ### Running from node -If you want to run the normal Python cli, you can use `python.sh`. It takes the -same options as the normal Python cli entrypoint, though the REPL does not +If you want to run the normal Python CLI, you can use `python.sh`. It takes the +same options as the normal Python CLI entrypoint, though the REPL does not function and will crash. `python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the From 7c61ed16707e6128b352575d57b6d267538ef3bc Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 11:53:45 +0100 Subject: [PATCH 09/11] Don't hard code version --- Tools/wasm/emscripten/web_example/python.worker.mjs | 10 ++++++---- configure | 1 + configure.ac | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Tools/wasm/emscripten/web_example/python.worker.mjs index a549edf53c5a30..42c2e1e08af24b 100644 --- a/Tools/wasm/emscripten/web_example/python.worker.mjs +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -70,13 +70,15 @@ const emscriptenSettings = { postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) }, async preRun(Module) { - // TODO: remove fixed version number + const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16); + const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; // Prevent complaints about not finding exec-prefix by making a lib-dynload directory - Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); Module.addRunDependency("install-stdlib"); - const resp = await fetch("python3.14.zip"); + const resp = await fetch(`python${major}.${minor}.zip`); const stdlibBuffer = await resp.arrayBuffer(); - Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); Module.removeRunDependency("install-stdlib"); } } diff --git a/configure b/configure index 05f0022c687a0a..98d58e95fcaa52 100755 --- a/configure +++ b/configure @@ -9437,6 +9437,7 @@ fi as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" if test "x$enable_wasm_dynamic_linking" = xyes then : diff --git a/configure.ac b/configure.ac index 5b408afa11bec4..01efcdd22a0808 100644 --- a/configure.ac +++ b/configure.ac @@ -2333,6 +2333,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) From fcd997bebb2da5df5d50cf5f51a1c387341eeba9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 11:56:08 +0100 Subject: [PATCH 10/11] Bump recommended Emscripten version --- Tools/wasm/README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 21b888af88c471..4802d9683de52e 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information. To cross compile to the ``wasm32-emscripten`` platform you need [the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. Emscripten -version 3.1.42 or newer is recommended. All commands below are relative to a checkout -of the Python repository. +a Python interpreter, and an installation of Node version 18 or newer. +Emscripten version 3.1.73 or newer is recommended. All commands below are +relative to a checkout of the Python repository. #### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) @@ -90,12 +90,14 @@ in the ``web_example`` directory. To run the web example, ``cd`` into the server; you can then visit ``http://localhost:8000/python.html`` in a browser to see a simple REPL example. -The web example uses ``SharedArrayBuffer``. For security reasons browsers only -provide ``SharedArrayBuffer`` in secure environments with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WebAssembly files. Otherwise the terminal will fail to -load with an error message like ``ReferenceError: SharedArrayBuffer is not -defined``. See more information here: +The web example relies on a bug fix in Emscripten version 3.1.73 so if you build +with earlier versions of Emscripten it may not work. The web example uses +``SharedArrayBuffer``. For security reasons browsers only provide +``SharedArrayBuffer`` in secure environments with cross-origin isolation. The +webserver must send cross-origin headers and correct MIME types for the +JavaScript and WebAssembly files. Otherwise the terminal will fail to load with +an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See +more information here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the From 3429caf9be1ffa422d85b8168d947a54974c8ac6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 14:29:07 +0100 Subject: [PATCH 11/11] Revert changes to emscripten/__main__.py --- Tools/wasm/emscripten/__main__.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index b99b74e3661060..9ce8dd6a364ad6 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -215,29 +215,12 @@ def configure_emscripten_python(context, working_dir): exec_script = working_dir / "python.sh" exec_script.write_text( dedent( - """\ + f"""\ #!/bin/sh - # Macs come with free BSD coreutils which doesn't have the -s option - # so feature detect and work around it. - if which grealpath > /dev/null; then - # It has brew installed gnu core utils, use that - REALPATH="grealpath -s" - elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then - # realpath points to GNU realpath so use it. - REALPATH="realpath -s" - else - # Shim for macs without GNU coreutils - abs_path () { - echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" - } - REALPATH=abs_path - fi - """ - f""" # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$($REALPATH "$0")" "$@" + exec {host_runner} {node_entry} "$(realpath -s $0)" "$@" """ ) )