Add implementation of emscripten_memcpy_big based on bulk memory. (#19128)

These new functions live in `libbulkmemory` which only gets included
if bulk memory is enabled (either via `-mbulk-memory` directly or
indirectly via `-pthread).

benchmark results for benchmark.test_memcpy_1mb:

```
 v8:                       mean: 1.666
 v8-bulkmemory:            mean: 1.598
 v8-standalone-bulkmemory: mean: 1.576
 v8-standalone:            mean: 3.197
```

Here we can see the that when bulk memory is enabled its at least as
fast if not faster than the JS version.

v8-standalone doesn't have emscripten_memcpy_big at all is is much
slower, as expected.  By adding `-mbulk-memory` the standalone version
becomes just as fast as the non-standalone.

Author: sbc100

embuilder.py

@@ -28,6 +28,7 @@
 
 # Minimal subset of targets used by CI systems to build enough to useful
 MINIMAL_TASKS = [
+    'libbulkmemory',
     'libcompiler_rt',
     'libcompiler_rt-wasm-sjlj',
     'libc',

emcc.py

@@ -1599,6 +1599,9 @@ def phase_setup(options, state, newargs):
     if '-mbulk-memory' not in newargs:
       newargs += ['-mbulk-memory']
 
+  if settings.SHARED_MEMORY:
+    settings.BULK_MEMORY = 1
+
   if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings:
     # If we get here then the user specified both DISABLE_EXCEPTION_CATCHING and EXCEPTION_CATCHING_ALLOWED
     # on the command line.  This is no longer valid so report either an error or a warning (for
@@ -2434,6 +2437,8 @@ def phase_linker_setup(options, state, newargs):
     settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_wasm_worker.js')))
 
   settings.SUPPORTS_GLOBALTHIS = feature_matrix.caniuse(feature_matrix.Feature.GLOBALTHIS)
+  if not settings.BULK_MEMORY:
+    settings.BULK_MEMORY = feature_matrix.caniuse(feature_matrix.Feature.BULK_MEMORY)
 
   if settings.AUDIO_WORKLET:
     if not settings.SUPPORTS_GLOBALTHIS:
@@ -3565,6 +3570,10 @@ def consume_arg_file():
       settings.DISABLE_EXCEPTION_CATCHING = 1
       settings.DISABLE_EXCEPTION_THROWING = 1
       settings.WASM_EXCEPTIONS = 0
+    elif arg == '-mbulk-memory':
+      settings.BULK_MEMORY = 1
+    elif arg == '-mno-bulk-memory':
+      settings.BULK_MEMORY = 0
     elif arg == '-fexceptions':
       # TODO Currently -fexceptions only means Emscripten EH. Switch to wasm
       # exception handling by default when -fexceptions is given when wasm

src/library.js

@@ -389,9 +389,11 @@ mergeInto(LibraryManager.library, {
   // variant, so we should never emit emscripten_memcpy_big() in the build.
   // In STANDALONE_WASM we avoid the emscripten_memcpy_big dependency so keep
   // the wasm file standalone.
+  // In BULK_MEMORY mode we include native versions of these functions based
+  // on memory.fill and memory.copy.
   // In MAIN_MODULE=1 or EMCC_FORCE_STDLIBS mode all of libc is force included
   // so we cannot override parts of it, and therefore cannot use libc_optz.
-#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM
+#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM && !BULK_MEMORY
 
 #if MIN_CHROME_VERSION < 45 || MIN_EDGE_VERSION < 14 || MIN_FIREFOX_VERSION < 34 || MIN_IE_VERSION != TARGET_NOT_SUPPORTED || MIN_SAFARI_VERSION < 100101
   // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/copyWithin lists browsers that support TypedArray.prototype.copyWithin, but it

src/settings_internal.js

@@ -256,3 +256,5 @@ var POST_JS_FILES = [];
 
 // Set when -pthread / -sPTHREADS is passed
 var PTHREADS = false;
+
+var BULK_MEMORY = false;

system/lib/libc/emscripten_internal.h

@@ -30,6 +30,7 @@ extern "C" {
 void emscripten_memcpy_big(void* __restrict__ dest,
                            const void* __restrict__ src,
                            size_t n) EM_IMPORT(emscripten_memcpy_big);
+void emscripten_memset_big(void* ptr, char value, size_t n);
 
 void emscripten_notify_memory_growth(size_t memory_index);
 

system/lib/libc/emscripten_memcpy.c

@@ -29,7 +29,7 @@ static void *__memcpy(void *restrict dest, const void *restrict src, size_t n) {
   unsigned char *block_aligned_d_end;
   unsigned char *d_end;
 
-#ifndef EMSCRIPTEN_STANDALONE_WASM
+#if !defined(EMSCRIPTEN_STANDALONE_WASM) || defined(__wasm_bulk_memory__)
   if (n >= 512) {
     emscripten_memcpy_big(dest, src, n);
     return dest;

system/lib/libc/emscripten_memcpy_big.S

@@ -0,0 +1,14 @@
+#ifdef __wasm64__
+#define PTR i64
+#else
+#define PTR i32
+#endif
+
+.globl emscripten_memcpy_big
+emscripten_memcpy_big:
+  .functype emscripten_memcpy_big (PTR, PTR, PTR) -> ()
+  local.get 0
+  local.get 1
+  local.get 2
+  memory.copy 0, 0
+  end_function

system/lib/libc/emscripten_memset.c

@@ -1,23 +1,44 @@
-// XXX EMSCRIPTEN ASAN: build an uninstrumented version of memset
-#if defined(__EMSCRIPTEN__) && defined(__has_feature)
-#if __has_feature(address_sanitizer)
-#define memset __attribute__((no_sanitize("address"))) emscripten_builtin_memset
-#endif
+#include "emscripten_internal.h" // for emscripten_memset_big
+
+#if defined(__has_feature) && __has_feature(address_sanitizer)
+// build an uninstrumented version of memset
+__attribute__((no_sanitize("address"))) void *__musl_memset(void *str, int c, size_t n);
+__attribute__((no_sanitize("address"))) void *__memset(void *str, int c, size_t n);
 #endif
 
-#ifdef EMSCRIPTEN_OPTIMIZE_FOR_OZ
+__attribute__((__weak__)) void *__musl_memset(void *str, int c, size_t n);
+__attribute__((__weak__)) void *__memset(void *str, int c, size_t n);
 
-#include <stddef.h>
+#ifdef EMSCRIPTEN_OPTIMIZE_FOR_OZ
 
-void *memset(void *str, int c, size_t n) {
+void *__memset(void *str, int c, size_t n) {
   unsigned char *s = (unsigned char *)str;
 #pragma clang loop unroll(disable)
   while(n--) *s++ = c;
   return str;
 }
 
+#elif defined(__wasm_bulk_memory__)
+
+#define memset __musl_memset
+#include "musl/src/string/memset.c"
+#undef memset
+
+void *__memset(void *str, int c, size_t n) {
+  if (n >= 512) {
+    emscripten_memset_big(str, c, n);
+    return str;
+  }
+  return __musl_memset(str, c, n);
+}
+
 #else
 
+#define memset __memset
 #include "musl/src/string/memset.c"
+#undef memset
 
 #endif
+
+weak_alias(__memset, emscripten_builtin_memset);
+weak_alias(__memset, memset);

system/lib/libc/emscripten_memset_big.S

@@ -0,0 +1,14 @@
+#ifdef __wasm64__
+#define PTR i64
+#else
+#define PTR i32
+#endif
+
+.globl emscripten_memset_big
+emscripten_memset_big:
+  .functype emscripten_memset_big (PTR, i32, PTR) -> ()
+  local.get 0
+  local.get 1
+  local.get 2
+  memory.fill 0
+  end_function

system/lib/standalone/standalone.c

@@ -152,7 +152,7 @@ int emscripten_resize_heap(size_t size) {
 }
 
 double emscripten_get_now(void) {
-  return (1000 * clock()) / (double)CLOCKS_PER_SEC;
+  return (1000ll * clock()) / (double)CLOCKS_PER_SEC;
 }
 
 // C++ ABI

test/other/metadce/test_metadce_hello_O0.funcs

@@ -9,6 +9,7 @@ $__lock
 $__lockfile
 $__lshrti3
 $__memcpy
+$__memset
 $__ofl_lock
 $__ofl_unlock
 $__original_main
@@ -41,7 +42,6 @@ $isdigit
 $legalstub$dynCall_jiji
 $main
 $memchr
-$memset
 $out
 $pad
 $pop_arg

test/other/metadce/test_metadce_minimal_pthreads.funcs

@@ -1,6 +1,7 @@
 $__emscripten_stdout_seek
 $__errno_location
 $__memcpy
+$__memset
 $__pthread_mutex_lock
 $__pthread_mutex_trylock
 $__pthread_mutex_unlock
@@ -63,7 +64,6 @@ $get_tasks_for_thread
 $init_file_lock
 $init_mparams
 $main
-$memset
 $nodtor
 $pthread_attr_destroy
 $receive_notification

test/other/test_memops_bulk_memory.c

@@ -0,0 +1,13 @@
+#include <assert.h>
+#include <string.h>
+
+const char *hello = "hello";
+const char *world = "world";
+
+int main() {
+  char buffer[100];
+  memset(buffer, 'a', 100);
+  memcpy(buffer, hello, strlen(hello) + 1);
+  assert(strcmp(buffer, hello) == 0);
+  return 0;
+}

test/test_other.py

@@ -12219,7 +12219,7 @@ def test_standard_library_mapping(self):
     # Test the `-l` flags on the command line get mapped the correct libraries variant
     self.run_process([EMBUILDER, 'build', 'libc-mt-debug', 'libcompiler_rt-mt', 'libdlmalloc-mt'])
 
-    libs = ['-lc', '-lcompiler_rt', '-lmalloc']
+    libs = ['-lc', '-lbulkmemory', '-lcompiler_rt', '-lmalloc']
     err = self.run_process([EMCC, test_file('hello_world.c'), '-pthread', '-nodefaultlibs', '-v'] + libs, stderr=PIPE).stderr
 
     # Check that the linker was run with `-mt` variants because `-pthread` was passed.
@@ -13412,3 +13412,30 @@ def test_wasi_random_get(self):
   @requires_node
   def test_wasi_sched_yield(self):
     self.run_wasi_test_suite_test('wasi_sched_yield')
+
+  def test_memops_bulk_memory(self):
+    self.emcc_args += ['--profiling-funcs', '-fno-builtin']
+
+    def run(args, expect_bulk_mem):
+      self.do_runf(test_file('other/test_memops_bulk_memory.c'), emcc_args=args)
+      funcs = self.parse_wasm('test_memops_bulk_memory.wasm')[2]
+      js = read_file('test_memops_bulk_memory.js')
+      if expect_bulk_mem:
+        self.assertNotContained('_emscripten_memcpy_big', js)
+        self.assertIn('$emscripten_memcpy_big', funcs)
+      else:
+        self.assertContained('_emscripten_memcpy_big', js)
+        self.assertNotIn('$emscripten_memcpy_big', funcs)
+
+    # By default we expect to find _emscripten_memcpy_big in the generaed JS and not in the
+    # native code.
+    run([], expect_bulk_mem=False)
+
+    # With bulk memory enabled we expect *not* to find it.
+    run(['-mbulk-memory'], expect_bulk_mem=True)
+
+    run(['-mbulk-memory', '-mno-bulk-memory'], expect_bulk_mem=False)
+
+    # -pthread implicitly enables bulk memory too.
+    self.setup_node_pthreads()
+    run(['-pthread'], expect_bulk_mem=True)

tools/settings.py

@@ -71,6 +71,7 @@
     'DEFAULT_TO_CXX',
     'WASM_OBJECT_FILES',
     'WASM_WORKERS',
+    'BULK_MEMORY',
 
     # Internal settings used during compilation
     'EXCEPTION_CATCHING_ALLOWED',

tools/system_libs.py

@@ -1274,6 +1274,17 @@ def can_use(self):
         not settings.LINKABLE and not os.environ.get('EMCC_FORCE_STDLIBS')
 
 
+class libbulkmemory(MuslInternalLibrary, AsanInstrumentedLibrary):
+  name = 'libbulkmemory'
+  src_dir = 'system/lib/libc'
+  src_files = ['emscripten_memcpy.c', 'emscripten_memset.c',
+               'emscripten_memcpy_big.S', 'emscripten_memset_big.S']
+  cflags = ['-mbulk-memory']
+
+  def can_use(self):
+    return super(libbulkmemory, self).can_use() and settings.BULK_MEMORY
+
+
 class libprintf_long_double(libc):
   name = 'libprintf_long_double'
   cflags = ['-DEMSCRIPTEN_PRINTF_LONG_DOUBLE']
@@ -1945,7 +1956,7 @@ def get_files(self):
                    '__main_void.c'])
     files += files_in_path(
         path='system/lib/libc',
-        filenames=['emscripten_memcpy.c'])
+        filenames=['emscripten_memcpy.c', 'emscripten_memset.c'])
     # It is more efficient to use JS methods for time, normally.
     files += files_in_path(
         path='system/lib/libc/musl/src/time',
@@ -2154,7 +2165,8 @@ def add_sanitizer_libs():
   if settings.SHRINK_LEVEL >= 2 and not settings.LINKABLE and \
      not os.environ.get('EMCC_FORCE_STDLIBS'):
     add_library('libc_optz')
-
+  if settings.BULK_MEMORY:
+    add_library('libbulkmemory')
   if settings.STANDALONE_WASM:
     add_library('libstandalonewasm')
   if settings.ALLOW_UNIMPLEMENTED_SYSCALLS: