Create memory in wasm by default

ChangeLog.md

@@ -20,6 +20,12 @@ See docs/process.md for more on how version tagging works.
 
 Current Trunk
 -------------
+- The WebAssembly memory used by emscripten programs is now, by default, created
+  in the wasm file and exported to JavaScript.  Previously we could create the
+  memory in JavaScript and import it into the wasm file.  The new
+  `IMPORTED_MEMORY` setting can be used to revert to the old behaviour.
+  Breaking change: This new setting is required if you provide a runtime
+  value for `wasmMemory` or `INITIAL_MEMORY` on the Module object.
 
 2.0.9: 11/16/2020
 -----------------

emcc.py

@@ -1741,6 +1741,10 @@ def include_and_export(name):
       # requires JS legalization
       shared.Settings.LEGALIZE_JS_FFI = 0
 
+    # TODO(sbc): Remove WASM2JS here once the size regression it would introduce has been fixed.
+    if shared.Settings.USE_PTHREADS or shared.Settings.RELOCATABLE or shared.Settings.ASYNCIFY_LAZY_LOAD_CODE or shared.Settings.WASM2JS:
+      shared.Settings.IMPORTED_MEMORY = 1
+
     if shared.Settings.WASM_BIGINT:
       shared.Settings.LEGALIZE_JS_FFI = 0
 

emscripten.py

@@ -536,10 +536,7 @@ def create_em_js(forwarded_json, metadata):
 
 
 def add_standard_wasm_imports(send_items_map):
-  # Normally we import these into the wasm (so that JS could use them even
-  # before the wasm loads), while in standalone mode we do not depend
-  # on JS to create them, but create them in the wasm and export them.
-  if not shared.Settings.STANDALONE_WASM:
+  if shared.Settings.IMPORTED_MEMORY:
     memory_import = 'wasmMemory'
     if shared.Settings.MODULARIZE and shared.Settings.USE_PTHREADS:
       # Pthreads assign wasmMemory in their worker startup. In MODULARIZE mode, they cannot assign inside the

src/postamble.js

@@ -8,11 +8,12 @@
 
 {{{ exportRuntime() }}}
 
-#if MEM_INIT_IN_WASM == 0
+#if !MEM_INIT_IN_WASM
+function runMemoryInitializer() {
 #if USE_PTHREADS
-if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) {
+  if (!memoryInitializer || ENVIRONMENT_IS_PTHREAD) return;
 #else
-if (memoryInitializer) {
+  if (!memoryInitializer) return
 #endif
   if (!isDataURI(memoryInitializer)) {
     memoryInitializer = locateFile(memoryInitializer);

src/postamble_minimal.js

@@ -204,6 +204,25 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) {
   /*** ASM_MODULE_EXPORTS ***/
 #endif
   wasmTable = asm['__indirect_function_table'];
+#if ASSERTIONS
+  assert(wasmTable);
+#endif
+
+#if !IMPORTED_MEMORY
+  wasmMemory = asm['memory'];
+#if ASSERTIONS
+  assert(wasmMemory);
+  assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+
+#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
+#if ASSERTIONS
+  if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
+#endif
+  HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+#endif
 
   initRuntime(asm);
 #if USE_PTHREADS && PTHREAD_POOL_SIZE

src/preamble.js

@@ -325,15 +325,16 @@ if (typeof SharedArrayBuffer === 'undefined' || typeof Atomics === 'undefined')
 #endif
 #endif
 
-#if STANDALONE_WASM
-#if ASSERTIONS
-// In standalone mode, the wasm creates the memory, and the user can't provide it.
-assert(!Module['wasmMemory']);
-#endif // ASSERTIONS
-#else // !STANDALONE_WASM
+#if IMPORTED_MEMORY
 // In non-standalone/normal mode, we create the memory here.
 #include "runtime_init_memory.js"
-#endif // !STANDALONE_WASM
+#else // IMPORTED_MEMORY
+#if ASSERTIONS
+// If memory is defined in wasm, the user can't provide it.
+assert(!Module['wasmMemory'], 'Use of `wasmMemory` detected.  Use -s IMPORTED_MEMORY to define wasmMemory externally');
+assert(INITIAL_MEMORY == {{{INITIAL_MEMORY}}}, 'Detected runtime INITIAL_MEMORY setting.  Use -s IMPORTED_MEMORY to define wasmMemory dynamically');
+#endif // ASSERTIONS
+#endif // IMPORTED_MEMORY
 
 #include "runtime_init_table.js"
 #include "runtime_stack_check.js"
@@ -833,6 +834,21 @@ function createWasm() {
 
     Module['asm'] = exports;
 
+#if !IMPORTED_MEMORY
+    wasmMemory = Module['asm']['memory'];
+#if ASSERTIONS
+    assert(wasmMemory, "memory not found in wasm exports");
+    // This assertion doesn't hold when emscripten is run in --post-link
+    // mode.
+    // TODO(sbc): Read INITIAL_MEMORY out of the wasm file in post-link mode.
+    //assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+    updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+#if !MEM_INIT_IN_WASM
+    runMemoryInitializer();
+#endif
+
 #if !RELOCATABLE
     wasmTable = Module['asm']['__indirect_function_table'];
 #if ASSERTIONS && !PURE_WASI
@@ -849,19 +865,6 @@ function createWasm() {
     // is in charge of programatically exporting them on the global object.
     exportAsmFunctions(exports);
 #endif
-#if STANDALONE_WASM
-    // In pure wasm mode the memory is created in the wasm (not imported), and
-    // then exported.
-    // TODO: do not create a Memory earlier in JS
-    wasmMemory = exports['memory'];
-#if ASSERTIONS
-    assert(wasmMemory, "memory not found in wasm exports");
-#endif
-    updateGlobalBufferAndViews(wasmMemory.buffer);
-#if STACK_OVERFLOW_CHECK
-    writeStackCookie();
-#endif
-#endif
 #if USE_PTHREADS
     // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers.
     wasmModule = module;

src/preamble_minimal.js

@@ -55,52 +55,13 @@ Module['wasm'] = base64Decode('{{{ getQuoted("WASM_BINARY_DATA") }}}');
 #include "runtime_functions.js"
 #include "runtime_strings.js"
 
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-
-#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
-var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
-#else
-var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
-#endif
-
-var wasmMemory = new WebAssembly.Memory({
-  'initial': {{{ INITIAL_MEMORY >>> 16 }}}
-#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
-  , 'maximum': wasmMaximumMemory
-#endif
-#if USE_PTHREADS
-  , 'shared': true
-#endif
-  });
-
-var wasmTable;
-var buffer = wasmMemory.buffer;
-
-#if USE_PTHREADS
-}
-#if ASSERTIONS
-assert(buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
-#endif
-#endif
-
-#if ASSERTIONS
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-assert(buffer.byteLength === {{{ INITIAL_MEMORY }}});
-#if USE_PTHREADS
-}
-#endif
-#endif // ASSERTIONS
-
-#if ALLOW_MEMORY_GROWTH
-// In ALLOW_MEMORY_GROWTH, we need to be able to re-initialize the
-// typed array buffer and heap views to the buffer whenever the heap
-// is resized.
 var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64;
+var wasmMemory, buffer, wasmTable;
+
 function updateGlobalBufferAndViews(b) {
+#if ASSERTIONS && USE_PTHREADS
+  assert(b instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
+#endif
   buffer = b;
   HEAP8 = new Int8Array(b);
   HEAP16 = new Int16Array(b);
@@ -111,34 +72,36 @@ function updateGlobalBufferAndViews(b) {
   HEAPF32 = new Float32Array(b);
   HEAPF64 = new Float64Array(b);
 }
-updateGlobalBufferAndViews(buffer);
-#else
-// In non-ALLOW_MEMORY_GROWTH scenario, we only need to initialize
-// the heap once, so optimize code size to do it statically here.
-var HEAP8 = new Int8Array(buffer);
-var HEAP16 = new Int16Array(buffer);
-var HEAP32 = new Int32Array(buffer);
-var HEAPU8 = new Uint8Array(buffer);
-var HEAPU16 = new Uint16Array(buffer);
-var HEAPU32 = new Uint32Array(buffer);
-var HEAPF32 = new Float32Array(buffer);
-var HEAPF64 = new Float64Array(buffer);
-#endif
-
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
+
+#if IMPORTED_MEMORY
+#if USE_PTHREADS
 if (!ENVIRONMENT_IS_PTHREAD) {
 #endif
-
-#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
-#if ASSERTIONS
-if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
+#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
+  var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
+#else
+  var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
 #endif
-HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+  wasmMemory = new WebAssembly.Memory({
+    'initial': {{{ INITIAL_MEMORY >>> 16 }}}
+#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
+    , 'maximum': wasmMaximumMemory
 #endif
-
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
-}
+#if USE_PTHREADS
+    , 'shared': true
 #endif
+    });
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#if USE_PTHREADS
+} else {
+#if MODULARIZE
+  updateGlobalBufferAndViews({{{EXPORT_NAME}}}.buffer);
+#else
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+}
+#endif // USE_PTHREADS
+#endif // IMPORTED_MEMORY
 
 #include "runtime_stack_check.js"
 #include "runtime_assertions.js"

src/runtime_init_memory.js

@@ -4,8 +4,11 @@
  * SPDX-License-Identifier: MIT
  */
 
-// Create the main memory. (Note: this isn't used in STANDALONE_WASM mode since the wasm
-// memory is created in the wasm, not in JS.)
+// Create the wasm memory. (Note: this only applies if IMPORTED_MEMORY is defined)
+#if !IMPORTED_MEMORY
+{{{ throw "this file should not be be included when IMPORTED_MEMORY is set"; }}}
+#endif
+
 #if USE_PTHREADS
 if (ENVIRONMENT_IS_PTHREAD) {
   wasmMemory = Module['wasmMemory'];

src/settings.js

@@ -1636,6 +1636,17 @@ var ABORT_ON_WASM_EXCEPTIONS = 0;
 // Implies STANDALONE_WASM.
 var PURE_WASI = 0;
 
+// Set to 1 to define the WebAssembly.Memory object outside of the wasm
+// module.  By default the wasm module defines the memory and exports
+// it to JavaScript.
+// Use of the following settings will enable this settings since they
+// depend on being able to define the memory in JavaScript:
+// - USE_PTHREADS
+// - RELOCATABLE
+// - ASYNCIFY_LAZY_LOAD_CODE
+// - WASM2JS (WASM=0)
+var IMPORTED_MEMORY = 0;
+
 //===========================================
 // Internal, used for testing only, from here
 //===========================================

src/shell_minimal.js

@@ -140,12 +140,6 @@ var _scriptDir = (typeof document !== 'undefined' && document.currentScript) ? d
 // coincide.
 var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'function';
 
-#if MODULARIZE
-if (ENVIRONMENT_IS_WORKER) {
-  var buffer = {{{EXPORT_NAME}}}.buffer;
-}
-#endif
-
 var currentScriptUrl = typeof _scriptDir !== 'undefined' ? _scriptDir : ((typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined);
 #endif // USE_PTHREADS
 

tests/code_size/hello_webgl2_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4957,
-  "a.js.gz": 2373,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 16413,
-  "total_gz": 9673
+  "a.js": 4927,
+  "a.js.gz": 2352,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 16385,
+  "total_gz": 9656
 }

tests/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 21297,
-  "a.js.gz": 8263,
+  "a.js": 21317,
+  "a.js.gz": 8276,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 25056,
-  "total_gz": 11364
+  "total": 25076,
+  "total_gz": 11377
 }

tests/code_size/hello_webgl_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4444,
-  "a.js.gz": 2199,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 15900,
-  "total_gz": 9499
+  "a.js": 4410,
+  "a.js.gz": 2172,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 15868,
+  "total_gz": 9476
 }

tests/code_size/hello_webgl_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 20785,
-  "a.js.gz": 8102,
+  "a.js": 20801,
+  "a.js.gz": 8114,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 24544,
-  "total_gz": 11203
+  "total": 24560,
+  "total_gz": 11215
 }

tests/code_size/hello_world_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 665,
   "a.html.gz": 427,
-  "a.js": 360,
-  "a.js.gz": 281,
-  "a.wasm": 102,
-  "a.wasm.gz": 114,
-  "total": 1127,
-  "total_gz": 822
+  "a.js": 322,
+  "a.js.gz": 259,
+  "a.wasm": 104,
+  "a.wasm.gz": 112,
+  "total": 1091,
+  "total_gz": 798
 }