Remove DYNAMICTOP_PTR (#12057)

DYNAMICTOP_PTR has been a pointer to where DYNAMICTOP is stored,
which represented the top of "dynamic" memory (not static, and not stack; or
in other words, dynamic == managed by sbrk/malloc). We represented it on the
JS side so that we could allocate from JS directly, in particular during startup.
However, that has caused a bunch of complexity that is not really worth it, and
it's also work done after link that we'd like to remove (WebAssembly/binaryen#3043).

This PR moves us to a place where malloc-like allocation is disallowed from
JS during startup. Instead, JS should call sbrk or malloc normally (which can
only be done after startup).

This change removes the final static allocation from JS, which means that we
no longer adjust memory layout after link, and therefore this PR removes the
special updating of the stack pointer and sbrk location that we used to do.
(This just removes the --pass-arg stuff for those in emcc.py, to show that
this works and passes tests - there is a bunch more code that can be removed
but is deferred to keep this as small as possible.)

Dynamic linking, however is an exception: we need to allocate room for
dynamic libraries during startup (the main module needs them before it
starts to run). To support that, this PR still keeps around getMemory()
(which does a dynamic allocation during startup) and dynamicAlloc in
a simplified form. That form just updates __heap_base as we allocate,
and then when the main program starts, __heap_base is a normal
extern global that it receives, and it initializes sbrk using that (after
that point, dynamic allocations are disallowed and asserted against).

This change should not be user-visible, except for removing the allocate()
JS function's ALLOC_DYNAMIC option (since we no longer allow that type
of allocation).

This can only land when it removes the last static allocation from JS.

This has a small code size benefit, basically it removes things like
these:

340,341d339
< var DYNAMIC_BASE = 5263904, DYNAMICTOP_PTR = 20992;
< 
361,362d358
< HEAP32[DYNAMICTOP_PTR >> 2] = DYNAMIC_BASE;
< 

The benefit to pthreads builds is larger as it removes a bunch of special
code for DYNAMICTOP_PTR that we had there.

Note that I'm not totally happy with the sbrk implementation, but we
can improve it once malloc is not linked in by default (see comment).

Author: kripken

ChangeLog.md

@@ -17,6 +17,8 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
+- Remove `ALLOC_DYNAMIC` and deprecate `dynamicAlloc`. (#12057, which also
+  removes the internal `DYNAMICTOP_PTR` API.)
 
 2.0.2: 09/02/2020
 -----------------
@@ -26,13 +28,13 @@ Current Trunk
   in chrome), and it made things more complex. The behavior has been changed
   to be simpler and just leave the browser's error code as it is.
 - Enable `--no-heap-copy` file packager option by default, and remove the old
-  defualt behavior entirely. That is the behavior we should have had from the
-  beginning as it is more memory-efficient.
+  default behavior entirely. That is the behavior we should have had from the
+  beginning as it is more memory-efficient. (#12027)
 - `--no-entry` is now required in `STANDALONE_WASM` mode when building a reactor
   (application without a main function).  Previously exporting a list of
   functions that didn't include `_main` would imply this.  Now the list of
   `EXPORTED_FUNCTIONS` is not relevant in the deciding the type of application
-  to build.
+  to build. (#12020)
 - Allow polymorphic types to be used without RTTI when using embind. (#10914)
 - Do not remove `__original_main` using `--inline-main`. We used to do this
   so that it didn't show up in stack traces (which could be confusing because
@@ -43,7 +45,7 @@ Current Trunk
   to add `__original_main` to there (since you are doing manual fine-tuning of
   the list of functions, which depends on the wasm's internals). Note that this
   should not matter in `-O2+` anyhow as normal inlining generally removes
-  `__original_main`.
+  `__original_main`. (#11995)
 
 2.0.1: 08/21/2020
 -----------------

emcc.py

@@ -1264,6 +1264,14 @@ def check(input_file):
     if shared.Settings.RELOCATABLE:
       shared.Settings.ALLOW_TABLE_GROWTH = 1
 
+    # various settings require sbrk() access
+    if shared.Settings.DETERMINISTIC or \
+       shared.Settings.EMSCRIPTEN_TRACING or \
+       shared.Settings.MALLOC == 'emmalloc' or \
+       shared.Settings.SAFE_HEAP or \
+       shared.Settings.MEMORYPROFILER:
+      shared.Settings.EXPORTED_FUNCTIONS += ['_sbrk']
+
     if shared.Settings.ASYNCIFY:
       # See: https://github.com/emscripten-core/emscripten/issues/12065
       # See: https://github.com/emscripten-core/emscripten/issues/12066
@@ -1466,8 +1474,9 @@ def check(input_file):
       shared.Settings.GLOBAL_BASE = 1024
 
     if shared.Settings.SAFE_HEAP:
-      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr().
-      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['emscripten_get_sbrk_ptr', '$unSign']
+      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr() from wasm
+      shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_get_sbrk_ptr']
+      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$unSign']
 
     if not shared.Settings.DECLARE_ASM_MODULE_EXPORTS:
       shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$exportAsmFunctions']
@@ -1520,7 +1529,6 @@ def check(input_file):
         ]
 
       shared.Settings.EXPORTED_RUNTIME_METHODS += [
-        'getMemory',
         'addRunDependency',
         'removeRunDependency',
       ]
@@ -2595,20 +2603,6 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target,
   intermediate_debug_info = bool(debug_info or options.emit_symbol_map or shared.Settings.ASYNCIFY_ONLY or shared.Settings.ASYNCIFY_REMOVE or shared.Settings.ASYNCIFY_ADD)
 
   if options.binaryen_passes:
-    if '--post-emscripten' in options.binaryen_passes and not shared.Settings.SIDE_MODULE:
-      if not shared.Settings.RELOCATABLE:
-        # With the wasm backend stack point value is baked in at link time.  However, emscripten's
-        # JS compiler can allocate more static data which then shifts the stack pointer.
-        # See `makeStaticAlloc` in the JS compiler.
-        options.binaryen_passes += ['--pass-arg=stack-pointer@%d' % shared.Settings.STACK_BASE]
-      # the value of the sbrk pointer has been computed by the JS compiler, and we can apply it in the wasm
-      # (we can't add this value when we placed post-emscripten in the proper position in the list of
-      # passes because that was before the value was computed)
-      # note that we don't pass this for a side module, as the value can't be applied - it must be
-      # imported
-      options.binaryen_passes += ['--pass-arg=emscripten-sbrk-ptr@%d' % shared.Settings.DYNAMICTOP_PTR]
-      if shared.Settings.STANDALONE_WASM:
-        options.binaryen_passes += ['--pass-arg=emscripten-sbrk-val@%d' % shared.Settings.DYNAMIC_BASE]
     # note that wasm-ld can strip DWARF info for us too (--strip-debug), but it
     # also strips the Names section. so to emit just the Names section we don't
     # tell wasm-ld to strip anything, and we do it here.

emscripten.py

@@ -120,6 +120,7 @@ def align_memory(addr):
 
 
 def align_static_bump(metadata):
+  # TODO: remove static bump entirely
   metadata['staticBump'] = align_memory(metadata['staticBump'])
   return metadata['staticBump']
 
@@ -191,7 +192,6 @@ def apply_forwarded_data(forwarded_data):
   forwarded_json = json.loads(forwarded_data)
   # Be aware of JS static allocations
   shared.Settings.STATIC_BUMP = forwarded_json['STATIC_BUMP']
-  shared.Settings.DYNAMICTOP_PTR = forwarded_json['DYNAMICTOP_PTR']
   # Be aware of JS static code hooks
   StaticCodeHooks.atinits = str(forwarded_json['ATINITS'])
   StaticCodeHooks.atmains = str(forwarded_json['ATMAINS'])

site/source/docs/api_reference/preamble.js.rst

@@ -404,7 +404,7 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
 .. COMMENT (not rendered) : The following methods are explicitly not part of the public API and not documented. Note that in some case referred to by function name, other cases by Module assignment.
 
   function allocate(slab, types, allocator, ptr) — Internal and use is discouraged. Documentation can remain in source code but not here.
-    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_DYNAMIC, ALLOC_NONE
+    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_NONE
 
   function addOnPreRun
   function addOnInit
@@ -413,7 +413,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
   function addOnPostRun
   Module['ALLOC_NORMAL'] = ALLOC_NORMAL;
   Module['ALLOC_STACK'] = ALLOC_STACK;
-  Module['ALLOC_DYNAMIC'] = ALLOC_DYNAMIC;
   Module['ALLOC_NONE'] = ALLOC_NONE;
   Module['HEAP'] = HEAP;
   Module['IHEAP'] = IHEAP;

src/deterministic.js

@@ -21,7 +21,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil
 
 function hashMemory(id) {
   var ret = 0;
-  var len = HEAP32[DYNAMICTOP_PTR>>2];
+  var len = _sbrk();
   for (var i = 0; i < len; i++) {
     ret = (ret*17 + HEAPU8[i])|0;
   }

src/library.js

@@ -444,12 +444,6 @@ LibraryManager.library = {
     return HEAPU8.length;
   },
 
-  emscripten_get_sbrk_ptr__asm: true,
-  emscripten_get_sbrk_ptr__sig: 'i',
-  emscripten_get_sbrk_ptr: function() {
-    return {{{ DYNAMICTOP_PTR }}};
-  },
-
 #if ABORTING_MALLOC
   $abortOnCannotGrowMemory: function(requestedSize) {
 #if ASSERTIONS

src/library_emmalloc.js

@@ -5,9 +5,8 @@
  */
 
 mergeInto(LibraryManager.library, {
-  emmalloc_unclaimed_heap_memory__deps: ['emscripten_get_sbrk_ptr'],
   emmalloc_unclaimed_heap_memory: function() {
-  	var dynamicTop = HEAPU32[_emscripten_get_sbrk_ptr()>>2];
+	var dynamicTop = _sbrk();
 #if ALLOW_MEMORY_GROWTH
 #if WASM
 #if MAXIMUM_MEMORY != -1

src/library_pthread.js

@@ -415,9 +415,8 @@ var LibraryPThread = {
         'asmJsUrlOrBlob': Module["asmJsUrlOrBlob"],
 #endif
 #if !MINIMAL_RUNTIME
-        'DYNAMIC_BASE': DYNAMIC_BASE,
+        'DYNAMIC_BASE': DYNAMIC_BASE
 #endif
-        'DYNAMICTOP_PTR': DYNAMICTOP_PTR
       });
     },
 

src/library_trace.js

@@ -269,7 +269,7 @@ var LibraryTracing = {
 #if !MINIMAL_RUNTIME
         'dynamic_base': DYNAMIC_BASE,
 #endif
-        'dynamic_top':  HEAP32[DYNAMICTOP_PTR>>2],
+        'dynamic_top':  _sbrk(),
         'total_memory': HEAP8.length
       };
       var now = EmscriptenTrace.now();

src/memoryprofiler.js

@@ -486,7 +486,11 @@ var emscriptenMemoryProfiler = {
     html += '<br />STACK memory area used now (should be zero): ' + self.formatBytes(STACKTOP - STACK_BASE) + '.' + colorBar('#FFFF00') + ' STACK watermark highest seen usage (approximate lower-bound!): ' + self.formatBytes(Math.abs(self.stackTopWatermark - STACK_BASE));
 
     var DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}};
-    var DYNAMICTOP = HEAP32[DYNAMICTOP_PTR>>2];
+    // During startup sbrk may not be defined yet. Ideally we should probably
+    // refactor memoryprofiler so that it only gets here after compiled code is
+    // ready to be called. For now, if the runtime is not yet initialized,
+    // assume the brk is right after the stack.
+    var DYNAMICTOP = runtimeInitialized ? _sbrk() : STACK_BASE;
     html += "<br />DYNAMIC memory area size: " + self.formatBytes(DYNAMICTOP - DYNAMIC_BASE);
     html += ". DYNAMIC_BASE: " + toHex(DYNAMIC_BASE, width);
     html += ". DYNAMICTOP: " + toHex(DYNAMICTOP, width) + ".";

src/modules.js

@@ -522,7 +522,6 @@ function exportRuntime() {
   var runtimeNumbers = [
     'ALLOC_NORMAL',
     'ALLOC_STACK',
-    'ALLOC_DYNAMIC',
     'ALLOC_NONE',
   ];
   if (ASSERTIONS) {
@@ -547,7 +546,6 @@ var PassManager = {
       Functions: Functions,
       EXPORTED_FUNCTIONS: EXPORTED_FUNCTIONS,
       STATIC_BUMP: STATIC_BUMP, // updated with info from JS
-      DYNAMICTOP_PTR: DYNAMICTOP_PTR,
       ATINITS: ATINITS.join('\n'),
       ATMAINS: ATMAINS.join('\n'),
       ATEXITS: ATEXITS.join('\n'),

src/preamble.js

@@ -189,8 +189,7 @@ function cwrap(ident, returnType, argTypes, opts) {
 
 var ALLOC_NORMAL = 0; // Tries to use _malloc()
 var ALLOC_STACK = 1; // Lives for the duration of the current function call
-var ALLOC_DYNAMIC = 2; // Cannot be freed except through sbrk
-var ALLOC_NONE = 3; // Do not allocate
+var ALLOC_NONE = 2; // Do not allocate
 
 // allocate(): This is for internal use. You can use it yourself as well, but the interface
 //             is a little tricky (see docs right below). The reason is that it is optimized
@@ -228,7 +227,7 @@ function allocate(slab, types, allocator, ptr) {
 #else
     typeof stackAlloc !== 'undefined' ? stackAlloc : null,
 #endif
-    dynamicAlloc][allocator](Math.max(size, singleType ? 1 : types.length));
+    ][allocator](Math.max(size, singleType ? 1 : types.length));
   }
 
   if (zeroinit) {
@@ -283,12 +282,6 @@ function allocate(slab, types, allocator, ptr) {
   return ret;
 }
 
-// Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready
-function getMemory(size) {
-  if (!runtimeInitialized) return dynamicAlloc(size);
-  return _malloc(size);
-}
-
 #include "runtime_strings.js"
 #include "runtime_strings_extra.js"
 
@@ -340,17 +333,51 @@ var STATIC_BASE = {{{ GLOBAL_BASE }}},
     STACK_BASE = {{{ getQuoted('STACK_BASE') }}},
     STACKTOP = STACK_BASE,
     STACK_MAX = {{{ getQuoted('STACK_MAX') }}},
-    DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}},
-    DYNAMICTOP_PTR = {{{ DYNAMICTOP_PTR }}};
+    DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}};
 
 #if ASSERTIONS
 assert(STACK_BASE % 16 === 0, 'stack must start aligned');
 assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned');
 #endif
 
+#if RELOCATABLE
+// We support some amount of allocation during startup in the case of
+// dynamic linking, which needs to allocate memory for dynamic libraries that
+// are loaded. That has to happen before the main program can start to run,
+// because the main program needs those linked in before it runs (so we can't
+// use normally malloc from the main program to do these allocations).
+
+// Allocate memory no even if malloc isn't ready yet.
+function getMemory(size) {
+  if (!runtimeInitialized) return dynamicAlloc(size);
+  return _malloc(size);
+}
+
+// To support such allocations during startup, track them on __heap_base and
+// then when the main module is loaded it reads that value and uses it to
+// initialize sbrk (the main module is relocatable itself, and so it does not
+// have __heap_base hardcoded into it - it receives it from JS as an extern
+// global, basically).
+Module['___heap_base'] = DYNAMIC_BASE;
+
+function dynamicAlloc(size) {
+  // After the runtime is initialized, we must only use sbrk() normally.
+  assert(!runtimeInitialized);
+#if USE_PTHREADS
+  assert(!ENVIRONMENT_IS_PTHREAD); // this function is not thread-safe
+#endif
+  var ret = Module['___heap_base'];
+  var end = (ret + size + 15) & -16;
+#if ASSERTIONS
+  assert(end <= HEAP8.length, 'failure to dynamicAlloc - memory growth etc. is not supported there, call malloc/sbrk directly or increase INITIAL_MEMORY');
+#endif
+  Module['___heap_base'] = end;
+  return ret;
+}
+#endif // RELOCATABLE
+
 #if USE_PTHREADS
 if (ENVIRONMENT_IS_PTHREAD) {
-
   // At the 'load' stage of Worker startup, we are just loading this script
   // but not ready to run yet. At 'run' we receive proper values for the stack
   // etc. and can launch a pthread. Set some fake values there meanwhile to
@@ -359,7 +386,6 @@ if (ENVIRONMENT_IS_PTHREAD) {
   STACK_MAX = STACKTOP = STACK_MAX = 0x7FFFFFFF;
 #endif
   // TODO DYNAMIC_BASE = Module['DYNAMIC_BASE'];
-  // TODO DYNAMICTOP_PTR = Module['DYNAMICTOP_PTR'];
 }
 #endif
 

src/preamble_minimal.js

@@ -70,9 +70,6 @@ var GLOBAL_BASE = {{{ GLOBAL_BASE }}},
     STACK_BASE = {{{ getQuoted('STACK_BASE') }}},
     STACKTOP = STACK_BASE,
     STACK_MAX = {{{ getQuoted('STACK_MAX') }}}
-#if USES_DYNAMIC_ALLOC
-    , DYNAMICTOP_PTR = {{{ DYNAMICTOP_PTR }}};
-#endif
     ;
 
 #if WASM
@@ -179,10 +176,6 @@ HEAPU8.set(new Uint8Array(Module['mem']), GLOBAL_BASE);
 
 #endif
 
-#if USES_DYNAMIC_ALLOC
-  HEAP32[DYNAMICTOP_PTR>>2] = {{{ getQuoted('DYNAMIC_BASE') }}};
-#endif
-
 #if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
 }
 #endif

src/runtime.js

@@ -64,10 +64,6 @@ var Runtime = {
 
 // Additional runtime elements, that need preprocessing
 
-// Allocated here in JS, after we have the runtime etc. prepared.
-// This constant is emitted into the JS or wasm code.
-var DYNAMICTOP_PTR = makeStaticAlloc(4);
-
 // "Process info" for syscalls is static and cannot change, so define it using
 // some fixed values
 var PROCINFO = {

src/runtime_init_memory.js

@@ -85,9 +85,6 @@ updateGlobalBufferAndViews(buffer);
 #if USE_PTHREADS
 if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time
 #endif
-#if !STANDALONE_WASM // in standalone mode the value is in the wasm
-HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;
-#endif // !STANDALONE_WASM
 #if USE_PTHREADS
 }
 #endif

src/runtime_safe_heap.js

@@ -106,11 +106,12 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) {
 #if SAFE_HEAP_LOG
   out('SAFE_HEAP store: ' + [dest, value, bytes, isFloat, SAFE_HEAP_COUNTER++]);
 #endif
+  var brk = _sbrk() >>> 0;
   if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest);
   if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes);
-  if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]);
-  assert(DYNAMICTOP_PTR);
-  assert(HEAP32[DYNAMICTOP_PTR>>2] <= HEAP8.length);
+  if (dest + bytes > brk) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + brk);
+  assert(brk >= STACK_BASE); // sbrk-managed memory must be above the stack
+  assert(brk <= HEAP8.length);
   setValue(dest, value, getSafeHeapType(bytes, isFloat), 1);
 }
 function SAFE_HEAP_STORE_D(dest, value, bytes) {
@@ -122,11 +123,12 @@ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) {
 #if CAN_ADDRESS_2GB
   dest >>>= 0;
 #endif
+  var brk = _sbrk() >>> 0;
   if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest);
   if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes);
-  if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]);
-  assert(DYNAMICTOP_PTR);
-  assert(HEAP32[DYNAMICTOP_PTR>>2] <= HEAP8.length);
+  if (dest + bytes > brk) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + brk);
+  assert(brk >= STACK_BASE); // sbrk-managed memory must be above the stack
+  assert(brk <= HEAP8.length);
   var type = getSafeHeapType(bytes, isFloat);
   var ret = getValue(dest, type, 1);
   if (unsigned) ret = unSign(ret, parseInt(type.substr(1), 10));

src/settings_internal.js

@@ -62,9 +62,6 @@ var EMBIND = 0;
 // Whether the main() function reads the argc/argv parameters.
 var MAIN_READS_PARAMS = 1;
 
-// The computed location of the pointer to the sbrk position.
-var DYNAMICTOP_PTR = -1;
-
 // The computed initial value of the program break (the sbrk position), which
 // is called DYNAMIC_BASE as it is the start of dynamically-allocated memory.
 var DYNAMIC_BASE = -1;

src/shell_minimal.js

@@ -152,7 +152,6 @@ var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'f
 if (ENVIRONMENT_IS_WORKER) {
   var buffer = {{{EXPORT_NAME}}}.buffer;
   var STATICTOP = {{{EXPORT_NAME}}}.STATICTOP;
-  var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}.DYNAMICTOP_PTR;
   var STACK_BASE = {{{EXPORT_NAME}}}.STACK_BASE;
   var STACKTOP = {{{EXPORT_NAME}}}.STACKTOP;
   var STACK_MAX = {{{EXPORT_NAME}}}.STACK_MAX;