Create memory in wasm binaryn by default

There are some cases where we want to create the memory in JS but for
most programs exporting the memory from wasm makes more sense and is
more compact and reliable.

The new EXTERNAL_MEMORY setting can be used to override this behaviour.

Author: sbc100

emcc.py

@@ -1739,6 +1739,9 @@ def include_and_export(name):
       # requires JS legalization
       shared.Settings.LEGALIZE_JS_FFI = 0
 
+    if shared.Settings.USE_PTHREADS or shared.Settings.RELOCATABLE or shared.Settings.ASYNCIFY_LAZY_LOAD_CODE:
+      shared.Settings.EXTERNAL_MEMORY = 1
+
     if shared.Settings.WASM_BIGINT:
       shared.Settings.LEGALIZE_JS_FFI = 0
 

emscripten.py

@@ -536,10 +536,7 @@ def create_em_js(forwarded_json, metadata):
 
 
 def add_standard_wasm_imports(send_items_map):
-  # Normally we import these into the wasm (so that JS could use them even
-  # before the wasm loads), while in standalone mode we do not depend
-  # on JS to create them, but create them in the wasm and export them.
-  if not shared.Settings.STANDALONE_WASM:
+  if shared.Settings.EXTERNAL_MEMORY:
     memory_import = 'wasmMemory'
     if shared.Settings.MODULARIZE and shared.Settings.USE_PTHREADS:
       # Pthreads assign wasmMemory in their worker startup. In MODULARIZE mode, they cannot assign inside the

src/postamble.js

@@ -8,11 +8,12 @@
 
 {{{ exportRuntime() }}}
 
-#if MEM_INIT_IN_WASM == 0
+#if !MEM_INIT_IN_WASM
+function runMemoryInitializer() {
 #if USE_PTHREADS
-if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) {
+  if (!memoryInitializer || ENVIRONMENT_IS_PTHREAD) return;
 #else
-if (memoryInitializer) {
+  if (!memoryInitializer) return
 #endif
   if (!isDataURI(memoryInitializer)) {
     memoryInitializer = locateFile(memoryInitializer);

src/postamble_minimal.js

@@ -204,6 +204,25 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) {
   /*** ASM_MODULE_EXPORTS ***/
 #endif
   wasmTable = asm['__indirect_function_table'];
+#if ASSERTIONS
+  assert(wasmTable);
+#endif
+
+#if !EXTERNAL_MEMORY
+  wasmMemory = asm['memory'];
+#if ASSERTIONS
+  assert(wasmMemory);
+  assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+  updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+
+#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
+#if ASSERTIONS
+  if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
+#endif
+  HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+#endif
 
   initRuntime(asm);
 #if USE_PTHREADS && PTHREAD_POOL_SIZE

src/preamble.js

@@ -325,15 +325,16 @@ if (typeof SharedArrayBuffer === 'undefined' || typeof Atomics === 'undefined')
 #endif
 #endif
 
-#if STANDALONE_WASM
-#if ASSERTIONS
-// In standalone mode, the wasm creates the memory, and the user can't provide it.
-assert(!Module['wasmMemory']);
-#endif // ASSERTIONS
-#else // !STANDALONE_WASM
+#if EXTERNAL_MEMORY
 // In non-standalone/normal mode, we create the memory here.
 #include "runtime_init_memory.js"
-#endif // !STANDALONE_WASM
+#else // EXTERNAL_MEMORY
+#if ASSERTIONS
+// If memory is defined in wasm, the user can't provide it.
+assert(!Module['wasmMemory'], 'Use of `wasmMemory` detected.  Use -s EXTERNAL_MEMORY to define wasmMemory externally');
+assert(INITIAL_MEMORY == {{{INITIAL_MEMORY}}}, 'Detected runtime INITIAL_MEMORY setting.  Use -s EXTERNAL_MEMORY to define wasmMemory dynamically');
+#endif // ASSERTIONS
+#endif // EXTERNAL_MEMORY
 
 #include "runtime_init_table.js"
 #include "runtime_stack_check.js"
@@ -833,6 +834,18 @@ function createWasm() {
 
     Module['asm'] = exports;
 
+#if !EXTERNAL_MEMORY
+    wasmMemory = Module['asm']['memory'];
+#if ASSERTIONS
+    assert(wasmMemory);
+    assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
+#endif
+    updateGlobalBufferAndViews(wasmMemory.buffer);
+#endif
+#if !MEM_INIT_IN_WASM
+    runMemoryInitializer();
+#endif
+
 #if !RELOCATABLE
     wasmTable = Module['asm']['__indirect_function_table'];
 #if ASSERTIONS && !PURE_WASI

src/preamble_minimal.js

@@ -55,52 +55,13 @@ Module['wasm'] = base64Decode('{{{ getQuoted("WASM_BINARY_DATA") }}}');
 #include "runtime_functions.js"
 #include "runtime_strings.js"
 
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-
-#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
-var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
-#else
-var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
-#endif
-
-var wasmMemory = new WebAssembly.Memory({
-  'initial': {{{ INITIAL_MEMORY >>> 16 }}}
-#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
-  , 'maximum': wasmMaximumMemory
-#endif
-#if USE_PTHREADS
-  , 'shared': true
-#endif
-  });
-
-var wasmTable;
-var buffer = wasmMemory.buffer;
-
-#if USE_PTHREADS
-}
-#if ASSERTIONS
-assert(buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
-#endif
-#endif
-
-#if ASSERTIONS
-#if USE_PTHREADS
-if (!ENVIRONMENT_IS_PTHREAD) {
-#endif
-assert(buffer.byteLength === {{{ INITIAL_MEMORY }}});
-#if USE_PTHREADS
-}
-#endif
-#endif // ASSERTIONS
-
-#if ALLOW_MEMORY_GROWTH
-// In ALLOW_MEMORY_GROWTH, we need to be able to re-initialize the
-// typed array buffer and heap views to the buffer whenever the heap
-// is resized.
 var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64;
+var wasmMemory, buffer, wasmTable;
+
 function updateGlobalBufferAndViews(b) {
+#if ASSERTIONS && USE_PTHREADS
+  assert(b instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag');
+#endif
   buffer = b;
   HEAP8 = new Int8Array(b);
   HEAP16 = new Int16Array(b);
@@ -111,34 +72,33 @@ function updateGlobalBufferAndViews(b) {
   HEAPF32 = new Float32Array(b);
   HEAPF64 = new Float64Array(b);
 }
-updateGlobalBufferAndViews(buffer);
-#else
-// In non-ALLOW_MEMORY_GROWTH scenario, we only need to initialize
-// the heap once, so optimize code size to do it statically here.
-var HEAP8 = new Int8Array(buffer);
-var HEAP16 = new Int16Array(buffer);
-var HEAP32 = new Int32Array(buffer);
-var HEAPU8 = new Uint8Array(buffer);
-var HEAPU16 = new Uint16Array(buffer);
-var HEAPU32 = new Uint32Array(buffer);
-var HEAPF32 = new Float32Array(buffer);
-var HEAPF64 = new Float64Array(buffer);
-#endif
 
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
+#if EXTERNAL_MEMORY
 if (!ENVIRONMENT_IS_PTHREAD) {
+#if ALLOW_MEMORY_GROWTH && MAXIMUM_MEMORY != -1
+  var wasmMaximumMemory = {{{ MAXIMUM_MEMORY >>> 16 }}};
+#else
+  var wasmMaximumMemory = {{{ INITIAL_MEMORY >>> 16}}};
 #endif
-
-#if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE
-#if ASSERTIONS
-if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
-#endif
-HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
+  wasmMemory = new WebAssembly.Memory({
+    'initial': {{{ INITIAL_MEMORY >>> 16 }}}
+#if USE_PTHREADS || !ALLOW_MEMORY_GROWTH || MAXIMUM_MEMORY != -1
+    , 'maximum': wasmMaximumMemory
 #endif
+    , 'shared': true
+    });
+}
 
-#if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
+#if MODULARIZE
+if (ENVIRONMENT_IS_WORKER) {
+  updateGlobalBufferAndViews({{{EXPORT_NAME}}}.buffer);
+} else {
+  updateGlobalBufferAndViews(wasmMemory.buffer);
 }
+#else
+updateGlobalBufferAndViews(wasmMemory.buffer);
 #endif
+#endif // EXTERNAL_MEMORY
 
 #include "runtime_stack_check.js"
 #include "runtime_assertions.js"

src/runtime_init_memory.js

@@ -4,8 +4,11 @@
  * SPDX-License-Identifier: MIT
  */
 
-// Create the main memory. (Note: this isn't used in STANDALONE_WASM mode since the wasm
-// memory is created in the wasm, not in JS.)
+// Create the wasm memory. (Note: this only applies if EXTERNAL_MEMORY is defined)
+#if !EXTERNAL_MEMORY
+assert(false, "should be be included when EXTERNAL_MEMORY is set");
+#endif
+
 #if USE_PTHREADS
 if (ENVIRONMENT_IS_PTHREAD) {
   wasmMemory = Module['wasmMemory'];

src/settings.js

@@ -1630,6 +1630,16 @@ var ABORT_ON_WASM_EXCEPTIONS = 0;
 // Implies STANDALONE_WASM.
 var PURE_WASI = 0;
 
+// Set to 1 to define the WebAssembly.Memory object outside of the wasm
+// module.  By default the wasm module defines the memory and exports
+// it to JavaScript.
+// Use of the following settings will enable this settings since they
+// depend on being able to define the memory in JavaScript:
+// - USE_PTHREADS
+// - RELOCATABLE
+// - ASYNCIFY_LAZY_LOAD_CODE
+var EXTERNAL_MEMORY = 0;
+
 //===========================================
 // Internal, used for testing only, from here
 //===========================================

src/shell_minimal.js

@@ -140,12 +140,6 @@ var _scriptDir = (typeof document !== 'undefined' && document.currentScript) ? d
 // coincide.
 var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'function';
 
-#if MODULARIZE
-if (ENVIRONMENT_IS_WORKER) {
-  var buffer = {{{EXPORT_NAME}}}.buffer;
-}
-#endif
-
 var currentScriptUrl = typeof _scriptDir !== 'undefined' ? _scriptDir : ((typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined);
 #endif // USE_PTHREADS
 

tests/code_size/hello_webgl2_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4957,
-  "a.js.gz": 2373,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 16413,
-  "total_gz": 9673
+  "a.js": 4927,
+  "a.js.gz": 2352,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 16385,
+  "total_gz": 9656
 }

tests/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 21297,
-  "a.js.gz": 8263,
+  "a.js": 21310,
+  "a.js.gz": 8280,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 25056,
-  "total_gz": 11364
+  "total": 25069,
+  "total_gz": 11381
 }

tests/code_size/hello_webgl_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 563,
   "a.html.gz": 377,
-  "a.js": 4444,
-  "a.js.gz": 2199,
-  "a.wasm": 10893,
-  "a.wasm.gz": 6923,
-  "total": 15900,
-  "total_gz": 9499
+  "a.js": 4410,
+  "a.js.gz": 2172,
+  "a.wasm": 10895,
+  "a.wasm.gz": 6927,
+  "total": 15868,
+  "total_gz": 9476
 }

tests/code_size/hello_webgl_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 588,
   "a.html.gz": 386,
-  "a.js": 20785,
-  "a.js.gz": 8102,
+  "a.js": 20794,
+  "a.js.gz": 8109,
   "a.mem": 3171,
   "a.mem.gz": 2715,
-  "total": 24544,
-  "total_gz": 11203
+  "total": 24553,
+  "total_gz": 11210
 }

tests/code_size/hello_world_wasm.json

@@ -1,10 +1,10 @@
 {
   "a.html": 665,
   "a.html.gz": 427,
-  "a.js": 360,
-  "a.js.gz": 281,
-  "a.wasm": 102,
-  "a.wasm.gz": 114,
-  "total": 1127,
-  "total_gz": 822
+  "a.js": 322,
+  "a.js.gz": 259,
+  "a.wasm": 104,
+  "a.wasm.gz": 112,
+  "total": 1091,
+  "total_gz": 798
 }

tests/code_size/hello_world_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 697,
   "a.html.gz": 437,
-  "a.js": 865,
-  "a.js.gz": 503,
+  "a.js": 1131,
+  "a.js.gz": 605,
   "a.mem": 6,
   "a.mem.gz": 32,
-  "total": 1568,
-  "total_gz": 972
+  "total": 1834,
+  "total_gz": 1074
 }

tests/code_size/random_printf_wasm.json

@@ -1,6 +1,6 @@
 {
-  "a.html": 13665,
-  "a.html.gz": 7286,
-  "total": 13665,
-  "total_gz": 7286
+  "a.html": 13742,
+  "a.html.gz": 7291,
+  "total": 13742,
+  "total_gz": 7291
 }

tests/code_size/random_printf_wasm2js.json

@@ -1,6 +1,6 @@
 {
-  "a.html": 18844,
-  "a.html.gz": 7893,
-  "total": 18844,
-  "total_gz": 7893
+  "a.html": 18972,
+  "a.html.gz": 7942,
+  "total": 18972,
+  "total_gz": 7942
 }

tests/other/metadce/hello_libcxx_O2.exports

@@ -7,6 +7,7 @@ dynCall_iiiiijj
 dynCall_jiji
 dynCall_viijii
 main
+memory
 stackAlloc
 stackRestore
 stackSave

tests/other/metadce/hello_libcxx_O2.imports

@@ -1,7 +1,6 @@
 env.abort
 env.emscripten_memcpy_big
 env.emscripten_resize_heap
-env.memory
 env.setTempRet0
 env.strftime_l
 wasi_snapshot_preview1.environ_get