Fix Wasm Workers in node.js (#20188)

Author: medns

emscripten.py

@@ -912,6 +912,7 @@ def create_pointer_conversion_wrappers(metadata):
     '__main_argc_argv': '__PP',
     'emscripten_stack_set_limits': '_pp',
     '__set_stack_limits': '_pp',
+    '__set_thread_state': '_p___',
     '__cxa_can_catch': '_ppp',
     '__cxa_increment_exception_refcount': '_p',
     '__cxa_decrement_exception_refcount': '_p',

src/library_wasm_worker.js

@@ -47,7 +47,14 @@ addToLibrary({
   $_wasmWorkerRunPostMessage: (e) => {
     // '_wsc' is short for 'wasm call', trying to use an identifier name that
     // will never conflict with user code
-    let data = e.data, wasmCall = data['_wsc'];
+#if ENVIRONMENT_MAY_BE_NODE
+    // In Node.js environment, message event 'e' containing the actual data sent,
+    // while in the browser environment it's contained by 'e.data'.
+    let data = ENVIRONMENT_IS_NODE ? e : e.data;
+#else
+    let data = e.data;
+#endif
+    let wasmCall = data['_wsc'];
     wasmCall && getWasmTableEntry(wasmCall)(...data['x']);
   },
 
@@ -155,6 +162,12 @@ if (ENVIRONMENT_IS_WASM_WORKER) {
 #endif
       'sb': stackLowestAddress, // sb = stack bottom (lowest stack address, SP points at this when stack is full)
       'sz': stackSize,          // sz = stack size
+#if USE_OFFSET_CONVERTER
+      'wasmOffsetData': wasmOffsetConverter,
+#endif
+#if LOAD_SOURCE_MAP
+      'wasmSourceMapData': wasmSourceMap,
+#endif
     });
     worker.onmessage = _wasmWorkerRunPostMessage;
     return _wasmWorkersID++;

src/parseTools.js

@@ -736,6 +736,18 @@ function runIfMainThread(text) {
   }
 }
 
+function runIfWorkerThread(text) {
+  if (WASM_WORKERS && PTHREADS) {
+    return `if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ${text} }`;
+  } else if (WASM_WORKERS) {
+    return `if (ENVIRONMENT_IS_WASM_WORKER) { ${text} }`;
+  } else if (PTHREADS) {
+    return `if (ENVIRONMENT_IS_PTHREAD) { ${text} }`;
+  } else {
+    return '';
+  }
+}
+
 function expectToReceiveOnModule(name) {
   return INCOMING_MODULE_JS_API.has(name);
 }
@@ -1031,3 +1043,7 @@ function getPerformanceNow() {
     return 'performance.now';
   }
 }
+
+function implicitSelf() {
+  return ENVIRONMENT.includes('node') ? 'self.' : '';
+}

src/preamble.js

@@ -785,7 +785,7 @@ function instantiateSync(file, info) {
 }
 #endif
 
-#if PTHREADS && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER)
+#if (PTHREADS || WASM_WORKERS) && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER)
 // When using postMessage to send an object, it is processed by the structured
 // clone algorithm.  The prototype, and hence methods, on that object is then
 // lost. This function adds back the lost prototype.  This does not work with
@@ -1082,22 +1082,18 @@ function createWasm() {
   // path.
   if (Module['instantiateWasm']) {
 
-#if USE_OFFSET_CONVERTER && PTHREADS
-    if (ENVIRONMENT_IS_PTHREAD) {
+#if USE_OFFSET_CONVERTER
 #if ASSERTIONS
-      assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');
+{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}}
 #endif
-      wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);
-    }
+{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}}
 #endif
 
-#if LOAD_SOURCE_MAP && PTHREADS
-    if (ENVIRONMENT_IS_PTHREAD) {
+#if LOAD_SOURCE_MAP
 #if ASSERTIONS
-      assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');
+{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}}
 #endif
-      wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);
-    }
+{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}}
 #endif
 
     try {

src/wasm_worker.js

@@ -1,18 +1,51 @@
 // N.B. The contents of this file are duplicated in src/library_wasm_worker.js
 // in variable "_wasmWorkerBlobUrl" (where the contents are pre-minified) If
 // doing any changes to this file, be sure to update the contents there too.
-onmessage = function(d) {
+
+'use strict';
+
+#if ENVIRONMENT_MAY_BE_NODE
+// Node.js support
+var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string';
+if (ENVIRONMENT_IS_NODE) {
+  // Create as web-worker-like an environment as we can.
+
+  var nodeWorkerThreads = require('worker_threads');
+
+  var parentPort = nodeWorkerThreads.parentPort;
+
+  parentPort.on('message', (data) => typeof onmessage === "function" && onmessage({ data: data }));
+
+  var fs = require('fs');
+
+  Object.assign(global, {
+    self: global,
+    require,
+    location: {
+      href: __filename
+    },
+    Worker: nodeWorkerThreads.Worker,
+    importScripts: (f) => (0, eval)(fs.readFileSync(f, 'utf8') + '//# sourceURL=' + f),
+    postMessage: (msg) => parentPort.postMessage(msg),
+    performance: global.performance || { now: Date.now },
+    addEventListener: (name, handler) => parentPort.on(name, handler),
+    removeEventListener: (name, handler) => parentPort.off(name, handler),
+  });
+}
+#endif // ENVIRONMENT_MAY_BE_NODE
+
+{{{ implicitSelf() }}}onmessage = function(d) {
 	// The first message sent to the Worker is always the bootstrap message.
 	// Drop this message listener, it served its purpose of bootstrapping
 	// the Wasm Module load, and is no longer needed. Let user code register
 	// any desired message handlers from now on.
-	onmessage = null;
+	{{{ implicitSelf() }}}onmessage = null;
 	d = d.data;
 #if !MODULARIZE
 	self.{{{ EXPORT_NAME }}} = d;
 #endif
 #if !MINIMAL_RUNTIME
-	d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); receiveInstance(instance, d['wasm']); return instance.exports; }
+	d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); return receiveInstance(instance, d['wasm']); }
 #endif
 	importScripts(d.js);
 #if MODULARIZE

test/test_core.py

@@ -9912,16 +9912,23 @@ def test_emscripten_async_load_script(self):
     self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'file1.txt', 'file2.txt', '--from-emcc', '--js-output=script2.js'])
     self.do_runf(test_file('test_emscripten_async_load_script.c'), emcc_args=['-sFORCE_FILESYSTEM'])
 
+  def prep_wasm_worker_in_node(self):
+    # Auto exit after 3 seconds in Nodejs environment to get WASM Worker stdout
+    self.add_pre_run("setTimeout(()=>process.exit(), 3000);")
+
   @node_pthreads
   def test_wasm_worker_hello(self):
-    self.do_runf(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
+    self.prep_wasm_worker_in_node()
+    self.do_run_in_out_file_test(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
 
   @node_pthreads
   def test_wasm_worker_malloc(self):
-    self.do_runf(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
+    self.prep_wasm_worker_in_node()
+    self.do_run_in_out_file_test(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
 
   @node_pthreads
   def test_wasm_worker_wait_async(self):
+    self.prep_wasm_worker_in_node()
     self.do_runf(test_file('wasm_worker/wait_async.c'), emcc_args=['-sWASM_WORKERS'])
 
 

test/wasm_worker/hello_wasm_worker.c

@@ -1,12 +1,12 @@
-#include <assert.h>
+#include <emscripten.h>
 #include <emscripten/wasm_worker.h>
-#include <stdio.h>
+#include <assert.h>
 
 // This is the code example in site/source/docs/api_reference/wasm_workers.rst
 
 void run_in_worker()
 {
-  printf("Hello from wasm worker!\n");
+  emscripten_console_log("Hello from wasm worker!\n");
 #ifdef REPORT_RESULT
   REPORT_RESULT(0);
 #endif

test/wasm_worker/hello_wasm_worker.out

@@ -0,0 +1 @@
+Hello from wasm worker!

test/wasm_worker/malloc_wasm_worker.out

@@ -0,0 +1 @@
+Hello from wasm worker!