Remove DYNAMICTOP_PTR (#12057)

DYNAMICTOP_PTR has been a pointer to where DYNAMICTOP is stored,
which represented the top of "dynamic" memory (not static, and not stack; or
in other words, dynamic == managed by sbrk/malloc). We represented it on the
JS side so that we could allocate from JS directly, in particular during startup.
However, that has caused a bunch of complexity that is not really worth it, and
it's also work done after link that we'd like to remove (WebAssembly/binaryen#3043).

This PR moves us to a place where malloc-like allocation is disallowed from
JS during startup. Instead, JS should call sbrk or malloc normally (which can
only be done after startup).

This change removes the final static allocation from JS, which means that we
no longer adjust memory layout after link, and therefore this PR removes the
special updating of the stack pointer and sbrk location that we used to do.
(This just removes the --pass-arg stuff for those in emcc.py, to show that
this works and passes tests - there is a bunch more code that can be removed
but is deferred to keep this as small as possible.)

Dynamic linking, however is an exception: we need to allocate room for
dynamic libraries during startup (the main module needs them before it
starts to run). To support that, this PR still keeps around getMemory()
(which does a dynamic allocation during startup) and dynamicAlloc in
a simplified form. That form just updates __heap_base as we allocate,
and then when the main program starts, __heap_base is a normal
extern global that it receives, and it initializes sbrk using that (after
that point, dynamic allocations are disallowed and asserted against).

This change should not be user-visible, except for removing the allocate()
JS function's ALLOC_DYNAMIC option (since we no longer allow that type
of allocation).

This can only land when it removes the last static allocation from JS.

This has a small code size benefit, basically it removes things like
these:

340,341d339
< var DYNAMIC_BASE = 5263904, DYNAMICTOP_PTR = 20992;
< 
361,362d358
< HEAP32[DYNAMICTOP_PTR >> 2] = DYNAMIC_BASE;
< 

The benefit to pthreads builds is larger as it removes a bunch of special
code for DYNAMICTOP_PTR that we had there.

Note that I'm not totally happy with the sbrk implementation, but we
can improve it once malloc is not linked in by default (see comment).

Author: kripken

ChangeLog.md

@@ -17,6 +17,8 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
+- Remove `ALLOC_DYNAMIC` and deprecate `dynamicAlloc`. (#12057, which also
+  removes the internal `DYNAMICTOP_PTR` API.)
 
 2.0.2: 09/02/2020
 -----------------
@@ -26,13 +28,13 @@ Current Trunk
   in chrome), and it made things more complex. The behavior has been changed
   to be simpler and just leave the browser's error code as it is.
 - Enable `--no-heap-copy` file packager option by default, and remove the old
-  defualt behavior entirely. That is the behavior we should have had from the
-  beginning as it is more memory-efficient.
+  default behavior entirely. That is the behavior we should have had from the
+  beginning as it is more memory-efficient. (#12027)
 - `--no-entry` is now required in `STANDALONE_WASM` mode when building a reactor
   (application without a main function).  Previously exporting a list of
   functions that didn't include `_main` would imply this.  Now the list of
   `EXPORTED_FUNCTIONS` is not relevant in the deciding the type of application
-  to build.
+  to build. (#12020)
 - Allow polymorphic types to be used without RTTI when using embind. (#10914)
 - Do not remove `__original_main` using `--inline-main`. We used to do this
   so that it didn't show up in stack traces (which could be confusing because
@@ -43,7 +45,7 @@ Current Trunk
   to add `__original_main` to there (since you are doing manual fine-tuning of
   the list of functions, which depends on the wasm's internals). Note that this
   should not matter in `-O2+` anyhow as normal inlining generally removes
-  `__original_main`.
+  `__original_main`. (#11995)
 
 2.0.1: 08/21/2020
 -----------------

emcc.py

@@ -1264,6 +1264,14 @@ def check(input_file):
     if shared.Settings.RELOCATABLE:
       shared.Settings.ALLOW_TABLE_GROWTH = 1
 
+    # various settings require sbrk() access
+    if shared.Settings.DETERMINISTIC or \
+       shared.Settings.EMSCRIPTEN_TRACING or \
+       shared.Settings.MALLOC == 'emmalloc' or \
+       shared.Settings.SAFE_HEAP or \
+       shared.Settings.MEMORYPROFILER:
+      shared.Settings.EXPORTED_FUNCTIONS += ['_sbrk']
+
     if shared.Settings.ASYNCIFY:
       # See: https://github.com/emscripten-core/emscripten/issues/12065
       # See: https://github.com/emscripten-core/emscripten/issues/12066
@@ -1466,8 +1474,9 @@ def check(input_file):
       shared.Settings.GLOBAL_BASE = 1024
 
     if shared.Settings.SAFE_HEAP:
-      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr().
-      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['emscripten_get_sbrk_ptr', '$unSign']
+      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr() from wasm
+      shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_get_sbrk_ptr']
+      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$unSign']
 
     if not shared.Settings.DECLARE_ASM_MODULE_EXPORTS:
       shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$exportAsmFunctions']
@@ -1520,7 +1529,6 @@ def check(input_file):
         ]
 
       shared.Settings.EXPORTED_RUNTIME_METHODS += [
-        'getMemory',
         'addRunDependency',
         'removeRunDependency',
       ]
@@ -2595,20 +2603,6 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target,
   intermediate_debug_info = bool(debug_info or options.emit_symbol_map or shared.Settings.ASYNCIFY_ONLY or shared.Settings.ASYNCIFY_REMOVE or shared.Settings.ASYNCIFY_ADD)
 
   if options.binaryen_passes:
-    if '--post-emscripten' in options.binaryen_passes and not shared.Settings.SIDE_MODULE:
-      if not shared.Settings.RELOCATABLE:
-        # With the wasm backend stack point value is baked in at link time.  However, emscripten's
-        # JS compiler can allocate more static data which then shifts the stack pointer.
-        # See `makeStaticAlloc` in the JS compiler.
-        options.binaryen_passes += ['--pass-arg=stack-pointer@%d' % shared.Settings.STACK_BASE]
-      # the value of the sbrk pointer has been computed by the JS compiler, and we can apply it in the wasm
-      # (we can't add this value when we placed post-emscripten in the proper position in the list of
-      # passes because that was before the value was computed)
-      # note that we don't pass this for a side module, as the value can't be applied - it must be
-      # imported
-      options.binaryen_passes += ['--pass-arg=emscripten-sbrk-ptr@%d' % shared.Settings.DYNAMICTOP_PTR]
-      if shared.Settings.STANDALONE_WASM:
-        options.binaryen_passes += ['--pass-arg=emscripten-sbrk-val@%d' % shared.Settings.DYNAMIC_BASE]
     # note that wasm-ld can strip DWARF info for us too (--strip-debug), but it
     # also strips the Names section. so to emit just the Names section we don't
     # tell wasm-ld to strip anything, and we do it here.

emscripten.py

@@ -120,6 +120,7 @@ def align_memory(addr):
 
 
 def align_static_bump(metadata):
+  # TODO: remove static bump entirely
   metadata['staticBump'] = align_memory(metadata['staticBump'])
   return metadata['staticBump']
 
@@ -191,7 +192,6 @@ def apply_forwarded_data(forwarded_data):
   forwarded_json = json.loads(forwarded_data)
   # Be aware of JS static allocations
   shared.Settings.STATIC_BUMP = forwarded_json['STATIC_BUMP']
-  shared.Settings.DYNAMICTOP_PTR = forwarded_json['DYNAMICTOP_PTR']
   # Be aware of JS static code hooks
   StaticCodeHooks.atinits = str(forwarded_json['ATINITS'])
   StaticCodeHooks.atmains = str(forwarded_json['ATMAINS'])

site/source/docs/api_reference/preamble.js.rst

@@ -404,7 +404,7 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
 .. COMMENT (not rendered) : The following methods are explicitly not part of the public API and not documented. Note that in some case referred to by function name, other cases by Module assignment.
 
   function allocate(slab, types, allocator, ptr) — Internal and use is discouraged. Documentation can remain in source code but not here.
-    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_DYNAMIC, ALLOC_NONE
+    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_NONE
 
   function addOnPreRun
   function addOnInit
@@ -413,7 +413,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
   function addOnPostRun
   Module['ALLOC_NORMAL'] = ALLOC_NORMAL;
   Module['ALLOC_STACK'] = ALLOC_STACK;
-  Module['ALLOC_DYNAMIC'] = ALLOC_DYNAMIC;
   Module['ALLOC_NONE'] = ALLOC_NONE;
   Module['HEAP'] = HEAP;
   Module['IHEAP'] = IHEAP;

src/deterministic.js

@@ -21,7 +21,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil
 
 function hashMemory(id) {
   var ret = 0;
-  var len = HEAP32[DYNAMICTOP_PTR>>2];
+  var len = _sbrk();
   for (var i = 0; i < len; i++) {
     ret = (ret*17 + HEAPU8[i])|0;
   }

src/library.js

@@ -444,12 +444,6 @@ LibraryManager.library = {
     return HEAPU8.length;
   },
 
-  emscripten_get_sbrk_ptr__asm: true,
-  emscripten_get_sbrk_ptr__sig: 'i',
-  emscripten_get_sbrk_ptr: function() {
-    return {{{ DYNAMICTOP_PTR }}};
-  },
-
 #if ABORTING_MALLOC
   $abortOnCannotGrowMemory: function(requestedSize) {
 #if ASSERTIONS

src/library_emmalloc.js

@@ -5,9 +5,8 @@
  */
 
 mergeInto(LibraryManager.library, {
-  emmalloc_unclaimed_heap_memory__deps: ['emscripten_get_sbrk_ptr'],
   emmalloc_unclaimed_heap_memory: function() {
-  	var dynamicTop = HEAPU32[_emscripten_get_sbrk_ptr()>>2];
+	var dynamicTop = _sbrk();
 #if ALLOW_MEMORY_GROWTH
 #if WASM
 #if MAXIMUM_MEMORY != -1

src/library_pthread.js

@@ -415,9 +415,8 @@ var LibraryPThread = {
         'asmJsUrlOrBlob': Module["asmJsUrlOrBlob"],
 #endif
 #if !MINIMAL_RUNTIME
-        'DYNAMIC_BASE': DYNAMIC_BASE,
+        'DYNAMIC_BASE': DYNAMIC_BASE
 #endif
-        'DYNAMICTOP_PTR': DYNAMICTOP_PTR
       });
     },
 

src/library_trace.js

@@ -269,7 +269,7 @@ var LibraryTracing = {
 #if !MINIMAL_RUNTIME
         'dynamic_base': DYNAMIC_BASE,
 #endif
-        'dynamic_top':  HEAP32[DYNAMICTOP_PTR>>2],
+        'dynamic_top':  _sbrk(),
         'total_memory': HEAP8.length
       };
       var now = EmscriptenTrace.now();

src/memoryprofiler.js

@@ -486,7 +486,11 @@ var emscriptenMemoryProfiler = {
     html += '<br />STACK memory area used now (should be zero): ' + self.formatBytes(STACKTOP - STACK_BASE) + '.' + colorBar('#FFFF00') + ' STACK watermark highest seen usage (approximate lower-bound!): ' + self.formatBytes(Math.abs(self.stackTopWatermark - STACK_BASE));
 
     var DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}};
-    var DYNAMICTOP = HEAP32[DYNAMICTOP_PTR>>2];
+    // During startup sbrk may not be defined yet. Ideally we should probably
+    // refactor memoryprofiler so that it only gets here after compiled code is
+    // ready to be called. For now, if the runtime is not yet initialized,
+    // assume the brk is right after the stack.
+    var DYNAMICTOP = runtimeInitialized ? _sbrk() : STACK_BASE;
     html += "<br />DYNAMIC memory area size: " + self.formatBytes(DYNAMICTOP - DYNAMIC_BASE);
     html += ". DYNAMIC_BASE: " + toHex(DYNAMIC_BASE, width);
     html += ". DYNAMICTOP: " + toHex(DYNAMICTOP, width) + ".";