Skip to content

Commit 525223d

Browse files
hoodmanehoodmane
committed
gh-127111: Make web example work again
I moved the web example from `Tools/wasm` into `Tools/wasm/emscripten/web_example`. I also added a new target `build_emscripten` which is `build_wasm` but also builds the web_example. The web_example needs: 1. python.html, copied 2. python.worker.mjs copied 3. python.mjs and python.wasm output from the main linking of the Python interpreter 4. The webserver that sets COOP and COEP 5. python3.14.zip This last is created by the `wasm_assets.py` script, which required a pretty small set of changes to work fine for us. The last thing that should be done is the `python.worker.mjs` script should be made independent of the Python version: currently 3.14 is hard coded. I ran into trouble doing this, so maybe I can leave it to a followup.
1 parent ff2278e commit 525223d

File tree

7 files changed

+81
-50
lines changed

7 files changed

+81
-50
lines changed

‎Makefile.pre.in

+32-19
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@
269269
# Other subdirectories
270270
SUBDIRSTOO= Include Lib Misc
271271

272-
# assets for Emscripten browser builds
273-
WASM_ASSETS_DIR=.$(prefix)
274-
WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py
275-
276272
# Files and directories to be distributed
277273
CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
278274
DISTFILES= README.rst ChangeLog $(CONFIGFILES)
@@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh
737733
build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
738734
python-config checksharedmods
739735

736+
.PHONY: build_emscripten
737+
build_emscripten: build_wasm web_example
738+
740739
# Check that the source is clean when building out of source.
741740
.PHONY: check-clean-src
742741
check-clean-src:
@@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
10161015
else true; \
10171016
fi
10181017

1019-
# wasm32-emscripten browser build
1020-
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
1021-
# --preload-file turns a relative asset path into an absolute path.
1018+
# wasm32-emscripten browser web example
1019+
1020+
WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/
1021+
web_example/python.html: $(WEBEX_DIR)/python.html
1022+
@mkdir -p web_example
1023+
@cp $< $@
1024+
1025+
web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs
1026+
@mkdir -p web_example
1027+
@cp $< $@
10221028

1023-
.PHONY: wasm_stdlib
1024-
wasm_stdlib: $(WASM_STDLIB)
1025-
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1026-
$(srcdir)/Tools/wasm/wasm_assets.py \
1029+
web_example/server.py: $(WEBEX_DIR)/server.py
1030+
@mkdir -p web_example
1031+
@cp $< $@
1032+
1033+
WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip
1034+
$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1035+
$(WEBEX_DIR)/wasm_assets.py \
10271036
Makefile pybuilddir.txt Modules/Setup.local
1028-
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
1029-
--buildroot . --prefix $(prefix)
1037+
$(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \
1038+
--buildroot . --prefix $(prefix) -o $@
10301039

1031-
python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
1032-
@cp $(srcdir)/Tools/wasm/python.html $@
1040+
web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
1041+
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
1042+
echo "Can only build web_example when target is Emscripten" ;\
1043+
exit 1 ;\
1044+
fi
1045+
cp python.mjs web_example/python.mjs
1046+
cp python.wasm web_example/python.wasm
10331047

1034-
python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
1035-
@cp $(srcdir)/Tools/wasm/python.worker.js $@
1048+
.PHONY: web_example
1049+
web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB)
10361050

10371051
############################################################################
10381052
# Header files
@@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval
30533067
find build -name '*.py[co]' -exec rm -f {} ';' || true
30543068
-rm -f pybuilddir.txt
30553069
-rm -f _bootstrap_python
3056-
-rm -f python.html python*.js python.data python*.symbols python*.map
3057-
-rm -f $(WASM_STDLIB)
3070+
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
30583071
-rm -f Programs/_testembed Programs/_freeze_module
30593072
-rm -rf Python/deepfreeze
30603073
-rm -f Python/frozen_modules/*.h

‎Tools/wasm/python.html renamed to ‎Tools/wasm/emscripten/web_example/python.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747

4848
async initialiseWorker() {
4949
if (!this.worker) {
50-
this.worker = new Worker(this.workerURL)
50+
this.worker = new Worker(this.workerURL, {type: "module"})
5151
this.worker.addEventListener('message', this.handleMessageFromWorker)
5252
}
5353
}
@@ -347,7 +347,7 @@
347347
programRunning(false)
348348
}
349349

350-
const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback)
350+
const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback)
351351
}
352352
</script>
353353
</head>

‎Tools/wasm/python.worker.js renamed to ‎Tools/wasm/emscripten/web_example/python.worker.mjs

+19-4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import createEmscriptenModule from "./python.mjs";
2+
13
class StdinBuffer {
24
constructor() {
35
this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT)
@@ -59,29 +61,42 @@ const stderr = (charCode) => {
5961

6062
const stdinBuffer = new StdinBuffer()
6163

62-
var Module = {
64+
const emscriptenSettings = {
6365
noInitialRun: true,
6466
stdin: stdinBuffer.stdin,
6567
stdout: stdout,
6668
stderr: stderr,
6769
onRuntimeInitialized: () => {
6870
postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab})
71+
},
72+
async preRun(Module) {
73+
// TODO: remove fixed version number
74+
// Prevent complaints about not finding exec-prefix by making a lib-dynload directory
75+
Module.FS.mkdirTree("/lib/python3.14/lib-dynload/");
76+
Module.addRunDependency("install-stdlib");
77+
const resp = await fetch("python3.14.zip");
78+
const stdlibBuffer = await resp.arrayBuffer();
79+
Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true });
80+
Module.removeRunDependency("install-stdlib");
6981
}
7082
}
7183

72-
onmessage = (event) => {
84+
const modulePromise = createEmscriptenModule(emscriptenSettings);
85+
86+
87+
onmessage = async (event) => {
7388
if (event.data.type === 'run') {
89+
const Module = await modulePromise;
7490
if (event.data.files) {
7591
for (const [filename, contents] of Object.entries(event.data.files)) {
7692
Module.FS.writeFile(filename, contents)
7793
}
7894
}
79-
const ret = callMain(event.data.args)
95+
const ret = Module.callMain(event.data.args);
8096
postMessage({
8197
type: 'finished',
8298
returnCode: ret
8399
})
84100
}
85101
}
86102

87-
importScripts('python.js')

‎Tools/wasm/wasm_webserver.py renamed to ‎Tools/wasm/emscripten/web_example/server.py

+1-7
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515

1616
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
17-
extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
18-
extensions_map.update(
19-
{
20-
".wasm": "application/wasm",
21-
}
22-
)
23-
2417
def end_headers(self) -> None:
2518
self.send_my_headers()
2619
super().end_headers()
@@ -44,3 +37,4 @@ def main() -> None:
4437

4538
if __name__ == "__main__":
4639
main()
40+

‎Tools/wasm/wasm_assets.py renamed to ‎Tools/wasm/emscripten/web_example/wasm_assets.py

+10-7
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from typing import Dict
2020

2121
# source directory
22-
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
22+
SRCDIR = pathlib.Path(__file__).parents[4].absolute()
2323
SRCDIR_LIB = SRCDIR / "Lib"
2424

2525

@@ -131,7 +131,7 @@ def filterfunc(filename: str) -> bool:
131131
return pathname not in args.omit_files_absolute
132132

133133
with zipfile.PyZipFile(
134-
args.wasm_stdlib_zip,
134+
args.output,
135135
mode="w",
136136
compression=args.compression,
137137
optimize=optimize,
@@ -195,6 +195,12 @@ def path(val: str) -> pathlib.Path:
195195
default=pathlib.Path("/usr/local"),
196196
type=path,
197197
)
198+
parser.add_argument(
199+
"-o", "--output",
200+
help="output file",
201+
type=path,
202+
)
203+
198204

199205

200206
def main() -> None:
@@ -204,7 +210,6 @@ def main() -> None:
204210
args.srcdir = SRCDIR
205211
args.srcdir_lib = SRCDIR_LIB
206212
args.wasm_root = args.buildroot / relative_prefix
207-
args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
208213
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
209214
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
210215

@@ -234,12 +239,10 @@ def main() -> None:
234239
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
235240
marker = args.wasm_dynload / ".empty"
236241
marker.touch()
237-
# os.py is a marker for finding the correct lib directory.
238-
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
239242
# The rest of stdlib that's useful in a WASM context.
240243
create_stdlib_zip(args)
241-
size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
242-
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
244+
size = round(args.output.stat().st_size / 1024**2, 2)
245+
parser.exit(0, f"Created {args.output} ({size} MiB)\n")
243246

244247

245248
if __name__ == "__main__":

‎configure

+8-5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎configure.ac

+9-6
Original file line numberDiff line numberDiff line change
@@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then
18541854
LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition"
18551855
], [], [-Werror])
18561856
])
1857-
elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then
1858-
dnl Emscripten does not support shared extensions yet. Build
1859-
dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
1857+
elif test "$ac_sys_system" = "Emscripten"; then
1858+
dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
1859+
DEF_MAKE_ALL_RULE="build_emscripten"
1860+
REQUIRE_PGO="no"
1861+
DEF_MAKE_RULE="all"
1862+
elif test "$ac_sys_system" = "WASI"; then
1863+
dnl Build "python.wasm", "pybuilddir.txt", and "platform" files.
18601864
DEF_MAKE_ALL_RULE="build_wasm"
18611865
REQUIRE_PGO="no"
18621866
DEF_MAKE_RULE="all"
@@ -2321,14 +2325,14 @@ AS_CASE([$ac_sys_system],
23212325
AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no])
23222326
23232327
dnl Start with 20 MB and allow to grow
2324-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"])
2328+
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"])
23252329
23262330
dnl map int64_t and uint64_t to JS bigint
23272331
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"])
23282332
23292333
dnl Include file system support
23302334
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"])
2331-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"])
2335+
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"])
23322336
23332337
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
23342338
AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])
@@ -2339,7 +2343,6 @@ AS_CASE([$ac_sys_system],
23392343
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"])
23402344
AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"])
23412345
])
2342-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"])
23432346
dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now.
23442347
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"])
23452348
WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"

0 commit comments

Comments
 (0)