Create memory in wasm binaryn by default (#12790)

There are some cases where we want to create the memory in JS but for
most programs exporting the memory from wasm makes more sense and is
more compact and reliable.

The new EXTERNAL_MEMORY setting can be used to override this behaviour.

Author: sbc100

ChangeLog.md

@@ -20,6 +20,12 @@ See docs/process.md for more on how version tagging works.
 
 Current Trunk
 -------------
+- The WebAssembly memory used by emscripten programs is now, by default, created
+  in the wasm file and exported to JavaScript.  Previously we could create the
+  memory in JavaScript and import it into the wasm file.  The new
+  `IMPORTED_MEMORY` setting can be used to revert to the old behaviour.
+  Breaking change: This new setting is required if you provide a runtime
+  value for `wasmMemory` or `INITIAL_MEMORY` on the Module object.
 
 2.0.9: 11/16/2020
 -----------------

emcc.py

@@ -1741,6 +1741,10 @@ def include_and_export(name):
       # requires JS legalization
       shared.Settings.LEGALIZE_JS_FFI = 0
 
+    # TODO(sbc): Remove WASM2JS here once the size regression it would introduce has been fixed.
+    if shared.Settings.USE_PTHREADS or shared.Settings.RELOCATABLE or shared.Settings.ASYNCIFY_LAZY_LOAD_CODE or shared.Settings.WASM2JS:
+      shared.Settings.IMPORTED_MEMORY = 1
+
     if shared.Settings.WASM_BIGINT:
       shared.Settings.LEGALIZE_JS_FFI = 0
 

emscripten.py

@@ -536,10 +536,7 @@ def create_em_js(forwarded_json, metadata):
 
 
 def add_standard_wasm_imports(send_items_map):
-  # Normally we import these into the wasm (so that JS could use them even
-  # before the wasm loads), while in standalone mode we do not depend
-  # on JS to create them, but create them in the wasm and export them.
-  if not shared.Settings.STANDALONE_WASM:
+  if shared.Settings.IMPORTED_MEMORY:
     memory_import = 'wasmMemory'
     if shared.Settings.MODULARIZE and shared.Settings.USE_PTHREADS:
       # Pthreads assign wasmMemory in their worker startup. In MODULARIZE mode, they cannot assign inside the

src/postamble.js

@@ -8,11 +8,12 @@
 
 {{{ exportRuntime() }}}
 
-#if MEM_INIT_IN_WASM == 0
+#if !MEM_INIT_IN_WASM
+function runMemoryInitializer() {
 #if USE_PTHREADS
-if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) {
+  if (!memoryInitializer || ENVIRONMENT_IS_PTHREAD) return;
 #else
-if (memoryInitializer) {
+  if (!memoryInitializer) return
 #endif
   if (!isDataURI(memoryInitializer)) {
     memoryInitializer = locateFile(memoryInitializer);

src/postamble_minimal.js

@@ -204,6 +204,25 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) {
   /*** ASM_MODULE_EXPORTS ***/
 #endif
   wasmTable = asm['__indirect_function_table'];
+#if ASSERTIONS
+  assert(wasmTable);
+#endif
+
+#if !IMPORTED_MEMORY
+  wasmMemory = asm['memory'];
+#if ASSERTIONS
+  assert(wasmMemory);
+  assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+
+#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
+#if ASSERTIONS
+  if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
+#endif
+  HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+#endif
 
   initRuntime(asm);
 #if USE_PTHREADS && PTHREAD_POOL_SIZE

src/preamble.js

@@ -325,15 +325,16 @@ if (typeof SharedArrayBuffer === 'undefined' || typeof Atomics === 'undefined')
 #endif
 #endif
 
-#if STANDALONE_WASM
-#if ASSERTIONS
-// In standalone mode, the wasm creates the memory, and the user can't provide it.
-assert(!Module['wasmMemory']);
-#endif // ASSERTIONS
-#else // !STANDALONE_WASM
+#if IMPORTED_MEMORY
 // In non-standalone/normal mode, we create the memory here.
 #include "runtime_init_memory.js"
-#endif // !STANDALONE_WASM
+#else // IMPORTED_MEMORY
+#if ASSERTIONS
+// If memory is defined in wasm, the user can't provide it.
+assert(!Module['wasmMemory'], 'Use of `wasmMemory` detected.  Use -s IMPORTED_MEMORY to define wasmMemory externally');
+assert(INITIAL_MEMORY == {{{INITIAL_MEMORY}}}, 'Detected runtime INITIAL_MEMORY setting.  Use -s IMPORTED_MEMORY to define wasmMemory dynamically');
+#endif // ASSERTIONS
+#endif // IMPORTED_MEMORY
 
 #include "runtime_init_table.js"
 #include "runtime_stack_check.js"
@@ -833,6 +834,21 @@ function createWasm() {
 
     Module['asm'] = exports;
 
+#if !IMPORTED_MEMORY
+    wasmMemory = Module['asm']['memory'];
+#if ASSERTIONS
+    assert(wasmMemory, "memory not found in wasm exports");
+    // This assertion doesn't hold when emscripten is run in --post-link
+    // mode.
+    // TODO(sbc): Read INITIAL_MEMORY out of the wasm file in post-link mode.
+    //assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+    updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+#if !MEM_INIT_IN_WASM
+    runMemoryInitializer();
+#endif
+
 #if !RELOCATABLE
     wasmTable = Module['asm']['__indirect_function_table'];
 #if ASSERTIONS && !PURE_WASI
@@ -849,19 +865,6 @@ function createWasm() {
     // is in charge of programatically exporting them on the global object.
     exportAsmFunctions(exports);
 #endif
-#if STANDALONE_WASM
-    // In pure wasm mode the memory is created in the wasm (not imported), and
-    // then exported.
-    // TODO: do not create a Memory earlier in JS
-    wasmMemory = exports['memory'];
-#if ASSERTIONS
-    assert(wasmMemory, "memory not found in wasm exports");
-#endif
-    updateGlobalBufferAndViews(wasmMemory.buffer);
-#if STACK_OVERFLOW_CHECK
-    writeStackCookie();
-#endif
-#endif
 #if USE_PTHREADS
     // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers.
     wasmModule = module;

src/preamble_minimal.js

@@ -55,52 +55,13 @@ Module['wasm'] = base64Decode('{{{ getQuoted("WASM_BINARY_DATA") }}}');
 #include "runtime_functions.js"
 #include "runtime_strings.js"
 
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-
-#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
-var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
-#else
-var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
-#endif
-
-var wasmMemory = new WebAssembly.Memory({
-  'initial': {{{ INITIAL_MEMORY >>> 16 }}}
-#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
-  , 'maximum': wasmMaximumMemory
-#endif
-#if USE_PTHREADS
-  , 'shared': true
-#endif
-  });
-
-var wasmTable;
-var buffer = wasmMemory.buffer;
-
-#if USE_PTHREADS
-}
-#if ASSERTIONS
-assert(buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
-#endif
-#endif
-
-#if ASSERTIONS
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-assert(buffer.byteLength === {{{ INITIAL_MEMORY }}});
-#if USE_PTHREADS
-}
-#endif
-#endif // ASSERTIONS
-
-#if ALLOW_MEMORY_GROWTH
-// In ALLOW_MEMORY_GROWTH, we need to be able to re-initialize the
-// typed array buffer and heap views to the buffer whenever the heap
-// is resized.
 var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64;
+var wasmMemory, buffer, wasmTable;
+
 function updateGlobalBufferAndViews(b) {
+#if ASSERTIONS && USE_PTHREADS
+  assert(b instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
+#endif
   buffer = b;
   HEAP8 = new Int8Array(b);
   HEAP16 = new Int16Array(b);
@@ -111,34 +72,36 @@ function updateGlobalBufferAndViews(b) {
   HEAPF32 = new Float32Array(b);
   HEAPF64 = new Float64Array(b);
 }
-updateGlobalBufferAndViews(buffer);
-#else
-// In non-ALLOW_MEMORY_GROWTH scenario, we only need to initialize
-// the heap once, so optimize code size to do it statically here.
-var HEAP8 = new Int8Array(buffer);
-var HEAP16 = new Int16Array(buffer);
-var HEAP32 = new Int32Array(buffer);
-var HEAPU8 = new Uint8Array(buffer);
-var HEAPU16 = new Uint16Array(buffer);
-var HEAPU32 = new Uint32Array(buffer);
-var HEAPF32 = new Float32Array(buffer);
-var HEAPF64 = new Float64Array(buffer);
-#endif
-
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
+
+#if IMPORTED_MEMORY
+#if USE_PTHREADS
 if (!ENVIRONMENT_IS_PTHREAD) {
 #endif
-
-#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
-#if ASSERTIONS
-if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
+#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
+  var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
+#else
+  var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
 #endif
-HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+  wasmMemory = new WebAssembly.Memory({
+    'initial': {{{ INITIAL_MEMORY >>> 16 }}}
+#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
+    , 'maximum': wasmMaximumMemory
 #endif
-
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
-}
+#if USE_PTHREADS
+    , 'shared': true
 #endif
+    });
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#if USE_PTHREADS
+} else {
+#if MODULARIZE
+  updateGlobalBufferAndViews({{{EXPORT_NAME}}}.buffer);
+#else
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+}
+#endif // USE_PTHREADS
+#endif // IMPORTED_MEMORY
 
 #include "runtime_stack_check.js"
 #include "runtime_assertions.js"

src/runtime_init_memory.js

@@ -4,8 +4,11 @@
  * SPDX-License-Identifier: MIT
  */
 
-// Create the main memory. (Note: this isn't used in STANDALONE_WASM mode since the wasm
-// memory is created in the wasm, not in JS.)
+// Create the wasm memory. (Note: this only applies if IMPORTED_MEMORY is defined)
+#if !IMPORTED_MEMORY
+{{{ throw "this file should not be be included when IMPORTED_MEMORY is set"; }}}
+#endif
+
 #if USE_PTHREADS
 if (ENVIRONMENT_IS_PTHREAD) {
   wasmMemory = Module['wasmMemory'];

src/settings.js

@@ -1636,6 +1636,17 @@ var ABORT_ON_WASM_EXCEPTIONS = 0;
 // Implies STANDALONE_WASM.
 var PURE_WASI = 0;
 
+// Set to 1 to define the WebAssembly.Memory object outside of the wasm
+// module.  By default the wasm module defines the memory and exports
+// it to JavaScript.
+// Use of the following settings will enable this settings since they
+// depend on being able to define the memory in JavaScript:
+// - USE_PTHREADS
+// - RELOCATABLE
+// - ASYNCIFY_LAZY_LOAD_CODE
+// - WASM2JS (WASM=0)
+var IMPORTED_MEMORY = 0;
+
 //===========================================
 // Internal, used for testing only, from here
 //===========================================

src/shell_minimal.js

@@ -140,12 +140,6 @@ var _scriptDir = (typeof document !== 'undefined' && document.currentScript) ? d
 // coincide.
 var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'function';
 
-#if MODULARIZE
-if (ENVIRONMENT_IS_WORKER) {
-  var buffer = {{{EXPORT_NAME}}}.buffer;
-}
-#endif
-
 var currentScriptUrl = typeof _scriptDir !== 'undefined' ? _scriptDir : ((typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined);
 #endif // USE_PTHREADS
 

tests/code_size/hello_webgl2_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4957,
-  "a.js.gz": 2373,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 16413,
-  "total_gz": 9673
+  "a.js": 4927,
+  "a.js.gz": 2352,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 16385,
+  "total_gz": 9656
 }

tests/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 21297,
-  "a.js.gz": 8263,
+  "a.js": 21317,
+  "a.js.gz": 8276,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 25056,
-  "total_gz": 11364
+  "total": 25076,
+  "total_gz": 11377
 }

tests/code_size/hello_webgl_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4444,
-  "a.js.gz": 2199,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 15900,
-  "total_gz": 9499
+  "a.js": 4410,
+  "a.js.gz": 2172,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 15868,
+  "total_gz": 9476
 }

tests/code_size/hello_webgl_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 20785,
-  "a.js.gz": 8102,
+  "a.js": 20801,
+  "a.js.gz": 8114,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 24544,
-  "total_gz": 11203
+  "total": 24560,
+  "total_gz": 11215
 }

tests/code_size/hello_world_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 665,
   "a.html.gz": 427,
-  "a.js": 360,
-  "a.js.gz": 281,
-  "a.wasm": 102,
-  "a.wasm.gz": 114,
-  "total": 1127,
-  "total_gz": 822
+  "a.js": 322,
+  "a.js.gz": 259,
+  "a.wasm": 104,
+  "a.wasm.gz": 112,
+  "total": 1091,
+  "total_gz": 798
 }