Remove DYNAMICTOP_PTR

ChangeLog.md

@@ -17,6 +17,8 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
+- Remove `ALLOC_DYNAMIC` and deprecate `dynamicAlloc`. (#12057, which also
+  removes the internal `DYNAMICTOP_PTR` API.)
 
 2.0.2: 09/02/2020
 -----------------
@@ -26,13 +28,13 @@ Current Trunk
   in chrome), and it made things more complex. The behavior has been changed
   to be simpler and just leave the browser's error code as it is.
 - Enable `--no-heap-copy` file packager option by default, and remove the old
-  defualt behavior entirely. That is the behavior we should have had from the
-  beginning as it is more memory-efficient.
+  default behavior entirely. That is the behavior we should have had from the
+  beginning as it is more memory-efficient. (#12027)
 - `--no-entry` is now required in `STANDALONE_WASM` mode when building a reactor
   (application without a main function).  Previously exporting a list of
   functions that didn't include `_main` would imply this.  Now the list of
   `EXPORTED_FUNCTIONS` is not relevant in the deciding the type of application
-  to build.
+  to build. (#12020)
 - Allow polymorphic types to be used without RTTI when using embind. (#10914)
 - Do not remove `__original_main` using `--inline-main`. We used to do this
   so that it didn't show up in stack traces (which could be confusing because
@@ -43,7 +45,7 @@ Current Trunk
   to add `__original_main` to there (since you are doing manual fine-tuning of
   the list of functions, which depends on the wasm's internals). Note that this
   should not matter in `-O2+` anyhow as normal inlining generally removes
-  `__original_main`.
+  `__original_main`. (#11995)
 
 2.0.1: 08/21/2020
 -----------------

emcc.py

@@ -1264,6 +1264,14 @@ def check(input_file):
     if shared.Settings.RELOCATABLE:
       shared.Settings.ALLOW_TABLE_GROWTH = 1
 
+    # various settings require sbrk() access
+    if shared.Settings.DETERMINISTIC or \
+       shared.Settings.EMSCRIPTEN_TRACING or \
+       shared.Settings.MALLOC == 'emmalloc' or \
+       shared.Settings.SAFE_HEAP or \
+       shared.Settings.MEMORYPROFILER:
+      shared.Settings.EXPORTED_FUNCTIONS += ['_sbrk']
+
     if shared.Settings.ASYNCIFY:
       # See: https://github.com/emscripten-core/emscripten/issues/12065
       # See: https://github.com/emscripten-core/emscripten/issues/12066
@@ -1466,8 +1474,9 @@ def check(input_file):
       shared.Settings.GLOBAL_BASE = 1024
 
     if shared.Settings.SAFE_HEAP:
-      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr().
-      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['emscripten_get_sbrk_ptr', '$unSign']
+      # SAFE_HEAP check includes calling emscripten_get_sbrk_ptr() from wasm
+      shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_get_sbrk_ptr']
+      shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$unSign']
 
     if not shared.Settings.DECLARE_ASM_MODULE_EXPORTS:
       shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$exportAsmFunctions']
@@ -1520,7 +1529,6 @@ def check(input_file):
         ]
 
       shared.Settings.EXPORTED_RUNTIME_METHODS += [
-        'getMemory',
         'addRunDependency',
         'removeRunDependency',
       ]
@@ -2595,20 +2603,6 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target,
   intermediate_debug_info = bool(debug_info or options.emit_symbol_map or shared.Settings.ASYNCIFY_ONLY or shared.Settings.ASYNCIFY_REMOVE or shared.Settings.ASYNCIFY_ADD)
 
   if options.binaryen_passes:
-    if '--post-emscripten' in options.binaryen_passes and not shared.Settings.SIDE_MODULE:
-      if not shared.Settings.RELOCATABLE:
-        # With the wasm backend stack point value is baked in at link time.  However, emscripten's
-        # JS compiler can allocate more static data which then shifts the stack pointer.
-        # See `makeStaticAlloc` in the JS compiler.
-        options.binaryen_passes += ['--pass-arg=stack-pointer@%d' % shared.Settings.STACK_BASE]
-      # the value of the sbrk pointer has been computed by the JS compiler, and we can apply it in the wasm
-      # (we can't add this value when we placed post-emscripten in the proper position in the list of
-      # passes because that was before the value was computed)
-      # note that we don't pass this for a side module, as the value can't be applied - it must be
-      # imported
-      options.binaryen_passes += ['--pass-arg=emscripten-sbrk-ptr@%d' % shared.Settings.DYNAMICTOP_PTR]
-      if shared.Settings.STANDALONE_WASM:
-        options.binaryen_passes += ['--pass-arg=emscripten-sbrk-val@%d' % shared.Settings.DYNAMIC_BASE]
     # note that wasm-ld can strip DWARF info for us too (--strip-debug), but it
     # also strips the Names section. so to emit just the Names section we don't
     # tell wasm-ld to strip anything, and we do it here.

emscripten.py

@@ -120,6 +120,7 @@ def align_memory(addr):
 
 
 def align_static_bump(metadata):
+  # TODO: remove static bump entirely
   metadata['staticBump'] = align_memory(metadata['staticBump'])
   return metadata['staticBump']
 
@@ -191,7 +192,6 @@ def apply_forwarded_data(forwarded_data):
   forwarded_json = json.loads(forwarded_data)
   # Be aware of JS static allocations
   shared.Settings.STATIC_BUMP = forwarded_json['STATIC_BUMP']
-  shared.Settings.DYNAMICTOP_PTR = forwarded_json['DYNAMICTOP_PTR']
   # Be aware of JS static code hooks
   StaticCodeHooks.atinits = str(forwarded_json['ATINITS'])
   StaticCodeHooks.atmains = str(forwarded_json['ATMAINS'])

site/source/docs/api_reference/preamble.js.rst

@@ -404,7 +404,7 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
 .. COMMENT (not rendered) : The following methods are explicitly not part of the public API and not documented. Note that in some case referred to by function name, other cases by Module assignment.
 
   function allocate(slab, types, allocator, ptr) — Internal and use is discouraged. Documentation can remain in source code but not here.
-    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_DYNAMIC, ALLOC_NONE
+    associated constants ALLOC_NORMAL, ALLOC_STACK, ALLOC_NONE
 
   function addOnPreRun
   function addOnInit
@@ -413,7 +413,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
   function addOnPostRun
   Module['ALLOC_NORMAL'] = ALLOC_NORMAL;
   Module['ALLOC_STACK'] = ALLOC_STACK;
-  Module['ALLOC_DYNAMIC'] = ALLOC_DYNAMIC;
   Module['ALLOC_NONE'] = ALLOC_NONE;
   Module['HEAP'] = HEAP;
   Module['IHEAP'] = IHEAP;

src/deterministic.js

@@ -21,7 +21,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil
 
 function hashMemory(id) {
   var ret = 0;
-  var len = HEAP32[DYNAMICTOP_PTR>>2];
+  var len = _sbrk();
   for (var i = 0; i < len; i++) {
     ret = (ret*17 + HEAPU8[i])|0;
   }

src/library.js

@@ -444,12 +444,6 @@ LibraryManager.library = {
     return HEAPU8.length;
   },
 
-  emscripten_get_sbrk_ptr__asm: true,
-  emscripten_get_sbrk_ptr__sig: 'i',
-  emscripten_get_sbrk_ptr: function() {
-    return {{{ DYNAMICTOP_PTR }}};
-  },
-
 #if ABORTING_MALLOC
   $abortOnCannotGrowMemory: function(requestedSize) {
 #if ASSERTIONS

src/library_emmalloc.js

@@ -5,9 +5,8 @@
  */
 
 mergeInto(LibraryManager.library, {
-  emmalloc_unclaimed_heap_memory__deps: ['emscripten_get_sbrk_ptr'],
   emmalloc_unclaimed_heap_memory: function() {
-  	var dynamicTop = HEAPU32[_emscripten_get_sbrk_ptr()>>2];
+	var dynamicTop = _sbrk();
 #if ALLOW_MEMORY_GROWTH
 #if WASM
 #if MAXIMUM_MEMORY != -1

src/library_pthread.js

@@ -415,9 +415,8 @@ var LibraryPThread = {
         'asmJsUrlOrBlob': Module["asmJsUrlOrBlob"],
 #endif
 #if !MINIMAL_RUNTIME
-        'DYNAMIC_BASE': DYNAMIC_BASE,
+        'DYNAMIC_BASE': DYNAMIC_BASE
 #endif
-        'DYNAMICTOP_PTR': DYNAMICTOP_PTR
       });
     },
 

src/library_trace.js

@@ -269,7 +269,7 @@ var LibraryTracing = {
 #if !MINIMAL_RUNTIME
         'dynamic_base': DYNAMIC_BASE,
 #endif
-        'dynamic_top':  HEAP32[DYNAMICTOP_PTR>>2],
+        'dynamic_top':  _sbrk(),
         'total_memory': HEAP8.length
       };
       var now = EmscriptenTrace.now();

src/memoryprofiler.js

@@ -486,7 +486,11 @@ var emscriptenMemoryProfiler = {
     html += '<br />STACK memory area used now (should be zero): ' + self.formatBytes(STACKTOP - STACK_BASE) + '.' + colorBar('#FFFF00') + ' STACK watermark highest seen usage (approximate lower-bound!): ' + self.formatBytes(Math.abs(self.stackTopWatermark - STACK_BASE));
 
     var DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}};
-    var DYNAMICTOP = HEAP32[DYNAMICTOP_PTR>>2];
+    // During startup sbrk may not be defined yet. Ideally we should probably
+    // refactor memoryprofiler so that it only gets here after compiled code is
+    // ready to be called. For now, if the runtime is not yet initialized,
+    // assume the brk is right after the stack.
+    var DYNAMICTOP = runtimeInitialized ? _sbrk() : STACK_BASE;
     html += "<br />DYNAMIC memory area size: " + self.formatBytes(DYNAMICTOP - DYNAMIC_BASE);
     html += ". DYNAMIC_BASE: " + toHex(DYNAMIC_BASE, width);
     html += ". DYNAMICTOP: " + toHex(DYNAMICTOP, width) + ".";

src/modules.js

@@ -522,7 +522,6 @@ function exportRuntime() {
   var runtimeNumbers = [
     'ALLOC_NORMAL',
     'ALLOC_STACK',
-    'ALLOC_DYNAMIC',
     'ALLOC_NONE',
   ];
   if (ASSERTIONS) {
@@ -547,7 +546,6 @@ var PassManager = {
       Functions: Functions,
       EXPORTED_FUNCTIONS: EXPORTED_FUNCTIONS,
       STATIC_BUMP: STATIC_BUMP, // updated with info from JS
-      DYNAMICTOP_PTR: DYNAMICTOP_PTR,
       ATINITS: ATINITS.join('\n'),
       ATMAINS: ATMAINS.join('\n'),
       ATEXITS: ATEXITS.join('\n'),

src/preamble.js

@@ -189,8 +189,7 @@ function cwrap(ident, returnType, argTypes, opts) {
 
 var ALLOC_NORMAL = 0; // Tries to use _malloc()
 var ALLOC_STACK = 1; // Lives for the duration of the current function call
-var ALLOC_DYNAMIC = 2; // Cannot be freed except through sbrk
-var ALLOC_NONE = 3; // Do not allocate
+var ALLOC_NONE = 2; // Do not allocate
 
 // allocate(): This is for internal use. You can use it yourself as well, but the interface
 //             is a little tricky (see docs right below). The reason is that it is optimized
@@ -228,7 +227,7 @@ function allocate(slab, types, allocator, ptr) {
 #else
     typeof stackAlloc !== 'undefined' ? stackAlloc : null,
 #endif
-    dynamicAlloc][allocator](Math.max(size, singleType ? 1 : types.length));
+    ][allocator](Math.max(size, singleType ? 1 : types.length));
   }
 
   if (zeroinit) {
@@ -283,12 +282,6 @@ function allocate(slab, types, allocator, ptr) {
   return ret;
 }
 
-// Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready
-function getMemory(size) {
-  if (!runtimeInitialized) return dynamicAlloc(size);
-  return _malloc(size);
-}
-
 #include "runtime_strings.js"
 #include "runtime_strings_extra.js"
 
@@ -340,17 +333,51 @@ var STATIC_BASE = {{{ GLOBAL_BASE }}},
     STACK_BASE = {{{ getQuoted('STACK_BASE') }}},
     STACKTOP = STACK_BASE,
     STACK_MAX = {{{ getQuoted('STACK_MAX') }}},
-    DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}},
-    DYNAMICTOP_PTR = {{{ DYNAMICTOP_PTR }}};
+    DYNAMIC_BASE = {{{ getQuoted('DYNAMIC_BASE') }}};
 
 #if ASSERTIONS
 assert(STACK_BASE % 16 === 0, 'stack must start aligned');
 assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned');
 #endif
 
+#if RELOCATABLE
+// We support some amount of allocation during startup in the case of
+// dynamic linking, which needs to allocate memory for dynamic libraries that
+// are loaded. That has to happen before the main program can start to run,
+// because the main program needs those linked in before it runs (so we can't
+// use normally malloc from the main program to do these allocations).
+
+// Allocate memory no even if malloc isn't ready yet.
+function getMemory(size) {
+  if (!runtimeInitialized) return dynamicAlloc(size);
+  return _malloc(size);
+}
+
+// To support such allocations during startup, track them on __heap_base and
+// then when the main module is loaded it reads that value and uses it to
+// initialize sbrk (the main module is relocatable itself, and so it does not
+// have __heap_base hardcoded into it - it receives it from JS as an extern
+// global, basically).
+Module['___heap_base'] = DYNAMIC_BASE;
+
+function dynamicAlloc(size) {
+  // After the runtime is initialized, we must only use sbrk() normally.
+  assert(!runtimeInitialized);
+#if USE_PTHREADS
+  assert(!ENVIRONMENT_IS_PTHREAD); // this function is not thread-safe
+#endif
+  var ret = Module['___heap_base'];
+  var end = (ret + size + 15) & -16;
+#if ASSERTIONS
+  assert(end <= HEAP8.length, 'failure to dynamicAlloc - memory growth etc. is not supported there, call malloc/sbrk directly or increase INITIAL_MEMORY');
+#endif
+  Module['___heap_base'] = end;
+  return ret;
+}
+#endif // RELOCATABLE
+
 #if USE_PTHREADS
 if (ENVIRONMENT_IS_PTHREAD) {
-
   // At the 'load' stage of Worker startup, we are just loading this script
   // but not ready to run yet. At 'run' we receive proper values for the stack
   // etc. and can launch a pthread. Set some fake values there meanwhile to
@@ -359,7 +386,6 @@ if (ENVIRONMENT_IS_PTHREAD) {
   STACK_MAX = STACKTOP = STACK_MAX = 0x7FFFFFFF;
 #endif
   // TODO DYNAMIC_BASE = Module['DYNAMIC_BASE'];
-  // TODO DYNAMICTOP_PTR = Module['DYNAMICTOP_PTR'];
 }
 #endif
 

src/preamble_minimal.js

@@ -70,9 +70,6 @@ var GLOBAL_BASE = {{{ GLOBAL_BASE }}},
     STACK_BASE = {{{ getQuoted('STACK_BASE') }}},
     STACKTOP = STACK_BASE,
     STACK_MAX = {{{ getQuoted('STACK_MAX') }}}
-#if USES_DYNAMIC_ALLOC
-    , DYNAMICTOP_PTR = {{{ DYNAMICTOP_PTR }}};
-#endif
     ;
 
 #if WASM
@@ -179,10 +176,6 @@ HEAPU8.set(new Uint8Array(Module['mem']), GLOBAL_BASE);
 
 #endif
 
-#if USES_DYNAMIC_ALLOC
-  HEAP32[DYNAMICTOP_PTR>>2] = {{{ getQuoted('DYNAMIC_BASE') }}};
-#endif
-
 #if USE_PTHREADS && ((MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE) || USES_DYNAMIC_ALLOC)
 }
 #endif

src/runtime.js

@@ -64,10 +64,6 @@ var Runtime = {
 
 // Additional runtime elements, that need preprocessing
 
-// Allocated here in JS, after we have the runtime etc. prepared.
-// This constant is emitted into the JS or wasm code.
-var DYNAMICTOP_PTR = makeStaticAlloc(4);
-
 // "Process info" for syscalls is static and cannot change, so define it using
 // some fixed values
 var PROCINFO = {

src/runtime_init_memory.js

@@ -85,9 +85,6 @@ updateGlobalBufferAndViews(buffer);
 #if USE_PTHREADS
 if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time
 #endif
-#if !STANDALONE_WASM // in standalone mode the value is in the wasm
-HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;
-#endif // !STANDALONE_WASM
 #if USE_PTHREADS
 }
 #endif

src/runtime_safe_heap.js

@@ -106,11 +106,12 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) {
 #if SAFE_HEAP_LOG
   out('SAFE_HEAP store: ' + [dest, value, bytes, isFloat, SAFE_HEAP_COUNTER++]);
 #endif
+  var brk = _sbrk() >>> 0;
   if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest);
   if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes);
-  if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]);
-  assert(DYNAMICTOP_PTR);
-  assert(HEAP32[DYNAMICTOP_PTR>>2] <= HEAP8.length);
+  if (dest + bytes > brk) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + brk);
+  assert(brk >= STACK_BASE); // sbrk-managed memory must be above the stack
+  assert(brk <= HEAP8.length);
   setValue(dest, value, getSafeHeapType(bytes, isFloat), 1);
 }
 function SAFE_HEAP_STORE_D(dest, value, bytes) {
@@ -122,11 +123,12 @@ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) {
 #if CAN_ADDRESS_2GB
   dest >>>= 0;
 #endif
+  var brk = _sbrk() >>> 0;
   if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest);
   if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes);
-  if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]);
-  assert(DYNAMICTOP_PTR);
-  assert(HEAP32[DYNAMICTOP_PTR>>2] <= HEAP8.length);
+  if (dest + bytes > brk) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + brk);
+  assert(brk >= STACK_BASE); // sbrk-managed memory must be above the stack
+  assert(brk <= HEAP8.length);
   var type = getSafeHeapType(bytes, isFloat);
   var ret = getValue(dest, type, 1);
   if (unsigned) ret = unSign(ret, parseInt(type.substr(1), 10));

src/settings_internal.js

@@ -62,9 +62,6 @@ var EMBIND = 0;
 // Whether the main() function reads the argc/argv parameters.
 var MAIN_READS_PARAMS = 1;
 
-// The computed location of the pointer to the sbrk position.
-var DYNAMICTOP_PTR = -1;
-
 // The computed initial value of the program break (the sbrk position), which
 // is called DYNAMIC_BASE as it is the start of dynamically-allocated memory.
 var DYNAMIC_BASE = -1;

src/shell_minimal.js

@@ -152,7 +152,6 @@ var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'f
 if (ENVIRONMENT_IS_WORKER) {
   var buffer = {{{EXPORT_NAME}}}.buffer;
   var STATICTOP = {{{EXPORT_NAME}}}.STATICTOP;
-  var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}.DYNAMICTOP_PTR;
   var STACK_BASE = {{{EXPORT_NAME}}}.STACK_BASE;
   var STACKTOP = {{{EXPORT_NAME}}}.STACKTOP;
   var STACK_MAX = {{{EXPORT_NAME}}}.STACK_MAX;