Remove deps_info system and and the running of llvm-nm on input file. NFC

This uses a new "stub object" construct to tell the linker (wasm-ld)
not only about the existence of the JS library symbols but the native
symbols on which they depend (a.k.a reverse dependencies).

This allows us to completely remove deps_info.py in favor of just using
normal `__deps` entries in the library files.  It also means we no
longer need to run `llvm-nm` on the linker inputs to discover the
symbols they use.

Depends on: https://reviews.llvm.org/D145308

Fixes: #18875

Author: sbc100

emcc.py

@@ -50,7 +50,6 @@
 from tools.minimal_runtime_shell import generate_minimal_runtime_html
 import tools.line_endings
 from tools import feature_matrix
-from tools import deps_info
 from tools import js_manipulation
 from tools import wasm2c
 from tools import webassembly
@@ -1275,7 +1274,7 @@ def run(args):
     process_libraries(state, [])
     if len(input_files) != 1:
       exit_with_error('--post-link requires a single input file')
-    phase_post_link(options, state, input_files[0][1], wasm_target, target)
+    phase_post_link(options, state, input_files[0][1], wasm_target, target, {})
     return 0
 
   ## Compile source code to object files
@@ -1316,11 +1315,10 @@ def run(args):
     js_info = get_js_sym_info()
     if not settings.SIDE_MODULE:
       js_syms = js_info['deps']
-      deps_info.append_deps_info(js_syms)
     if settings.ASYNCIFY:
       settings.ASYNCIFY_IMPORTS += ['env.' + x for x in js_info['asyncFuncs']]
 
-  phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs)
+  phase_calculate_system_libraries(state, linker_arguments, newargs)
 
   phase_link(linker_arguments, wasm_target, js_syms)
 
@@ -1336,7 +1334,7 @@ def run(args):
 
   # Perform post-link steps (unless we are running bare mode)
   if options.oformat != OFormat.BARE:
-    phase_post_link(options, state, wasm_target, wasm_target, target)
+    phase_post_link(options, state, wasm_target, wasm_target, target, js_syms)
 
   return 0
 
@@ -2082,11 +2080,6 @@ def phase_linker_setup(options, state, newargs):
       settings.INCLUDE_FULL_LIBRARY = 1
     settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$loadDylibs']
 
-  # If we are including the entire JS library then we know for sure we will, by definition,
-  # require all the reverse dependencies.
-  if settings.INCLUDE_FULL_LIBRARY:
-    default_setting('REVERSE_DEPS', 'all')
-
   if settings.MAIN_MODULE == 1 or settings.SIDE_MODULE == 1:
     settings.LINKABLE = 1
 
@@ -3086,14 +3079,13 @@ def compile_source_file(i, input_file):
 
 
 @ToolchainProfiler.profile_block('calculate system libraries')
-def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs):
+def phase_calculate_system_libraries(state, linker_arguments, newargs):
   extra_files_to_link = []
   # Link in ports and system libraries, if necessary
   if not settings.SIDE_MODULE:
     # Ports are always linked into the main module, never the side module.
     extra_files_to_link += ports.get_libs(settings)
-  all_linker_inputs = [f for _, f in sorted(linker_inputs)] + extra_files_to_link
-  extra_files_to_link += system_libs.calculate(all_linker_inputs, newargs, forced=state.forced_stdlibs)
+  extra_files_to_link += system_libs.calculate(newargs, forced=state.forced_stdlibs)
   linker_arguments.extend(extra_files_to_link)
 
 
@@ -3112,7 +3104,7 @@ def phase_link(linker_arguments, wasm_target, js_syms):
 
 
 @ToolchainProfiler.profile_block('post_link')
-def phase_post_link(options, state, in_wasm, wasm_target, target):
+def phase_post_link(options, state, in_wasm, wasm_target, target, js_syms):
   global final_js
 
   target_basename = unsuffixed_basename(target)
@@ -3134,7 +3126,7 @@ def phase_post_link(options, state, in_wasm, wasm_target, target):
   else:
     memfile = shared.replace_or_append_suffix(target, '.mem')
 
-  phase_emscript(options, in_wasm, wasm_target, memfile)
+  phase_emscript(options, in_wasm, wasm_target, memfile, js_syms)
 
   if options.js_transform:
     phase_source_transforms(options)
@@ -3152,7 +3144,7 @@ def phase_post_link(options, state, in_wasm, wasm_target, target):
 
 
 @ToolchainProfiler.profile_block('emscript')
-def phase_emscript(options, in_wasm, wasm_target, memfile):
+def phase_emscript(options, in_wasm, wasm_target, memfile, js_syms):
   # Emscripten
   logger.debug('emscript')
 
@@ -3161,7 +3153,7 @@ def phase_emscript(options, in_wasm, wasm_target, memfile):
     # _read in shell.js depends on intArrayToString when SUPPORT_BASE64_EMBEDDING is set
     settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.append('$intArrayToString')
 
-  emscripten.run(in_wasm, wasm_target, final_js, memfile)
+  emscripten.run(in_wasm, wasm_target, final_js, memfile, js_syms)
   save_intermediate('original')
 
 

emscripten.py

@@ -296,7 +296,7 @@ def create_named_globals(metadata):
   return '\n'.join(named_globals)
 
 
-def emscript(in_wasm, out_wasm, outfile_js, memfile):
+def emscript(in_wasm, out_wasm, outfile_js, memfile, js_syms):
   # Overview:
   #   * Run wasm-emscripten-finalize to extract metadata and modify the binary
   #     to use emscripten's wasm<->JS ABI
@@ -309,7 +309,7 @@ def emscript(in_wasm, out_wasm, outfile_js, memfile):
     # set file locations, so that JS glue can find what it needs
     settings.WASM_BINARY_FILE = js_manipulation.escape_for_js_string(os.path.basename(out_wasm))
 
-  metadata = finalize_wasm(in_wasm, out_wasm, memfile)
+  metadata = finalize_wasm(in_wasm, out_wasm, memfile, js_syms)
 
   if settings.RELOCATABLE and settings.MEMORY64 == 2:
     metadata.imports += ['__memory_base32']
@@ -461,7 +461,7 @@ def get_metadata(infile, outfile, modify_wasm, args):
   return metadata
 
 
-def finalize_wasm(infile, outfile, memfile):
+def finalize_wasm(infile, outfile, memfile, js_syms):
   building.save_intermediate(infile, 'base.wasm')
   args = []
 
@@ -536,13 +536,16 @@ def finalize_wasm(infile, outfile, memfile):
 
   expected_exports = set(settings.EXPORTED_FUNCTIONS)
   expected_exports.update(asmjs_mangle(s) for s in settings.REQUIRED_EXPORTS)
+  for deps in js_syms.values():
+    expected_exports.update(asmjs_mangle(s) for s in deps)
 
   # Calculate the subset of exports that were explicitly marked with llvm.used.
   # These are any exports that were not requested on the command line and are
   # not known auto-generated system functions.
   unexpected_exports = [e for e in metadata.exports if treat_as_user_function(e)]
   unexpected_exports = [asmjs_mangle(e) for e in unexpected_exports]
   unexpected_exports = [e for e in unexpected_exports if e not in expected_exports]
+
   building.user_requested_exports.update(unexpected_exports)
   settings.EXPORTED_FUNCTIONS.extend(unexpected_exports)
 
@@ -922,5 +925,5 @@ def normalize_line_endings(text):
   return text
 
 
-def run(in_wasm, out_wasm, outfile_js, memfile):
-  emscript(in_wasm, out_wasm, outfile_js, memfile)
+def run(in_wasm, out_wasm, outfile_js, memfile, js_syms):
+  emscript(in_wasm, out_wasm, outfile_js, memfile, js_syms)

site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst

@@ -596,9 +596,6 @@ See the `library_*.js`_ files for other examples.
      This is useful when all the implemented methods use a JavaScript
      singleton containing helper methods. See ``library_webgl.js`` for
      an example.
-   - If a JavaScript library depends on a compiled C library (like most
-     of *libc*), you must edit `src/deps_info.json`_. Search for
-     "deps_info" in `tools/system_libs.py`_.
    - The keys passed into `mergeInto` generate functions that are prefixed
      by ``_``. In other words ``my_func: function() {},`` becomes
      ``function _my_func() {}``, as all C methods in emscripten have a ``_`` prefix. Keys starting with ``$`` have the ``$``
@@ -810,7 +807,6 @@ you can give it a try. See `Emnapi documentation`_ for more details.
 
 .. _library.js: https://github.com/emscripten-core/emscripten/blob/main/src/library.js
 .. _test_js_libraries: https://github.com/emscripten-core/emscripten/blob/1.29.12/tests/test_core.py#L5043
-.. _src/deps_info.json: https://github.com/emscripten-core/emscripten/blob/main/src/deps_info.json
 .. _tools/system_libs.py: https://github.com/emscripten-core/emscripten/blob/main/tools/system_libs.py
 .. _library_\*.js: https://github.com/emscripten-core/emscripten/tree/main/src
 .. _test_add_function in test/test_core.py: https://github.com/emscripten-core/emscripten/blob/1.29.12/tests/test_core.py#L6237

src/jsifier.js

@@ -66,6 +66,26 @@ function isDefined(symName) {
   return false;
 }
 
+function getTransitiveDeps(symbol) {
+  // TODO(sbc): Use some kind of cache to avoid quadratic behaviour here.
+  const transitiveDeps = new Set();
+  const seen = new Set();
+  const toVisit = [symbol];
+  while (toVisit.length) {
+    const sym = toVisit.pop();
+    if (!seen.has(sym)) {
+      let directDeps = LibraryManager.library[sym + '__deps'] || [];
+      directDeps = directDeps.filter((d) => typeof d === 'string');
+      if (directDeps.length) {
+        directDeps.forEach(transitiveDeps.add, transitiveDeps);
+        toVisit.push(...directDeps);
+      }
+      seen.add(sym);
+    }
+  }
+  return Array.from(transitiveDeps);
+}
+
 function runJSify() {
   const libraryItems = [];
   const symbolDeps = {};
@@ -221,7 +241,7 @@ function ${name}(${args}) {
     // the number specifies the number of arguments. In Emscripten, route all
     // these to a single function '__cxa_find_matching_catch' that variadically
     // processes all of these functions using JS 'arguments' object.
-    if (symbol.startsWith('__cxa_find_matching_catch_')) {
+    if (!WASM_EXCEPTIONS && symbol.startsWith('__cxa_find_matching_catch_')) {
       if (DISABLE_EXCEPTION_THROWING) {
         error('DISABLE_EXCEPTION_THROWING was set (likely due to -fno-exceptions), which means no C++ exception throwing support code is linked in, but exception catching code appears. Either do not set DISABLE_EXCEPTION_THROWING (if you do want exception throwing) or compile all source files with -fno-except (so that no exceptions support code is required); also make sure DISABLE_EXCEPTION_CATCHING is set to the right value - if you want exceptions, it should be off, and vice versa.');
         return;
@@ -260,8 +280,14 @@ function ${name}(${args}) {
 
       if (symbolsOnly) {
         if (!isJsOnlySymbol(symbol) && LibraryManager.library.hasOwnProperty(symbol)) {
-          externalDeps = deps.filter((d) => !isJsOnlySymbol(d) && !(d in LibraryManager.library) && typeof d === 'string');
-          symbolDeps[symbol] = externalDeps;
+          var value = LibraryManager.library[symbol];
+          var resolvedSymbol = symbol;
+          // Resolve aliases before looking up deps
+          if (typeof value == 'string' && value[0] != '=' && LibraryManager.library.hasOwnProperty(value)) {
+            resolvedSymbol = value;
+          }
+          var transtiveDeps = getTransitiveDeps(resolvedSymbol);
+          symbolDeps[symbol] = transtiveDeps.filter((d) => !isJsOnlySymbol(d) && !(d in LibraryManager.library));
         }
         return;
       }
@@ -565,6 +591,7 @@ function ${name}(${args}) {
   }
 
   if (symbolsOnly) {
+    if (abortExecution) throw Error('Aborting compilation due to previous errors');
     print(JSON.stringify({
       deps: symbolDeps,
       asyncFuncs

src/library.js

@@ -1212,6 +1212,13 @@ mergeInto(LibraryManager.library, {
   // ==========================================================================
 
 #if SUPPORT_LONGJMP == 'emscripten'
+  // In WebAssemblyLowerEmscriptenEHSjLj pass in the LLVM backend, function
+  // calls that exist in the same function with setjmp are converted to a code
+  // sequence that includes invokes, malloc, free, saveSetjmp, and
+  // emscripten_longjmp.  setThrew is called from invokes, but we don't have
+  // any way to express that dependency so we use emscripten_throw_longjmp as
+  // a proxy and declare the dependency here.
+  _emscripten_throw_longjmp__deps: ['setThrew'],
   _emscripten_throw_longjmp: function() {
 #if EXCEPTION_STACK_TRACES
     throw new EmscriptenSjLj;
@@ -1721,7 +1728,7 @@ mergeInto(LibraryManager.library, {
     return { family: family, addr: addr, port: port };
   },
   $writeSockaddr__docs: '/** @param {number=} addrlen */',
-  $writeSockaddr__deps: ['$Sockets', '$inetPton4', '$inetPton6', '$zeroMemory'],
+  $writeSockaddr__deps: ['$Sockets', '$inetPton4', '$inetPton6', '$zeroMemory', 'htons'],
   $writeSockaddr: function (sa, family, addr, port, addrlen) {
     switch (family) {
       case {{{ cDefs.AF_INET }}}:
@@ -1858,7 +1865,7 @@ mergeInto(LibraryManager.library, {
     return 0;
   },
 
-  getaddrinfo__deps: ['$Sockets', '$DNS', '$inetPton4', '$inetNtop4', '$inetPton6', '$inetNtop6', '$writeSockaddr'],
+  getaddrinfo__deps: ['$Sockets', '$DNS', '$inetPton4', '$inetNtop4', '$inetPton6', '$inetNtop6', '$writeSockaddr', 'malloc', 'htonl'],
   getaddrinfo__proxy: 'sync',
   getaddrinfo: function(node, service, hint, out) {
     // Note getaddrinfo currently only returns a single addrinfo with ai_next defaulting to NULL. When NULL

src/library_async.js

@@ -22,7 +22,11 @@ mergeInto(LibraryManager.library, {
 #if ASYNCIFY
   $Asyncify__deps: ['$runAndAbortIfError', '$callUserCallback', '$sigToWasmTypes',
 #if !MINIMAL_RUNTIME
-    '$runtimeKeepalivePush', '$runtimeKeepalivePop'
+    '$runtimeKeepalivePush', '$runtimeKeepalivePop',
+#endif
+#if ASYNCIFY == 1
+    // Needed by allocateData and handleSleep respectively
+    'malloc', 'free',
 #endif
   ],
 

src/library_browser.js

@@ -1212,6 +1212,7 @@ var LibraryBrowser = {
 
   // To avoid creating worker parent->child chains, always proxies to execute on the main thread.
   emscripten_create_worker__proxy: 'sync',
+  emscripten_create_worker__deps: ['$UTF8ToString', 'malloc', 'free'],
   emscripten_create_worker: function(url) {
     url = UTF8ToString(url);
     var id = Browser.workers.length;
@@ -1253,6 +1254,7 @@ var LibraryBrowser = {
     return id;
   },
 
+  emscripten_destroy_worker__deps: ['free'],
   emscripten_destroy_worker__proxy: 'sync',
   emscripten_destroy_worker: function(id) {
     var info = Browser.workers[id];

src/library_exceptions.js

@@ -23,9 +23,12 @@ var LibraryExceptions = {
   //
   // excPtr - Thrown object pointer to wrap. Metadata pointer is calculated from it.
   $ExceptionInfo__docs: '/** @constructor */',
+  $ExceptionInfo__deps: [
+    '__cxa_is_pointer_type',
 #if EXCEPTION_DEBUG
-  $ExceptionInfo__deps: ['$ptrToString'],
+    '$ptrToString'
 #endif
+  ],
   $ExceptionInfo: function(excPtr) {
     this.excPtr = excPtr;
     this.ptr = excPtr - {{{ C_STRUCTS.__cxa_exception.__size__ }}};
@@ -238,7 +241,6 @@ var LibraryExceptions = {
   // functionality boils down to picking a suitable 'catch' block.
   // We'll do that here, instead, to keep things simpler.
   __cxa_find_matching_catch__deps: ['$exceptionLast', '$ExceptionInfo', '__resumeException', '__cxa_can_catch', 'setTempRet0'],
-  //__cxa_find_matching_catch__sig: 'p',
   __cxa_find_matching_catch: function() {
     var thrown = 
 #if EXCEPTION_STACK_TRACES
@@ -408,6 +410,7 @@ var LibraryExceptions = {
 #endif
 };
 
+#if !WASM_EXCEPTIONS
 // In LLVM, exceptions generate a set of functions of form
 // __cxa_find_matching_catch_1(), __cxa_find_matching_catch_2(), etc.  where the
 // number specifies the number of arguments.  In Emscripten, route all these to
@@ -417,4 +420,12 @@ addCxaCatch = function(n) {
   LibraryManager.library['__cxa_find_matching_catch_' + n] = '__cxa_find_matching_catch';
 };
 
+// Add the first 10 catch handlers premptively.  Others get added on demand in
+// jsifier.  This is done here primarily so that these symbols end up with the
+// correct deps in the stub library that we pass to wasm-ld.
+for (let i = 1; i < 10; i++) {
+  addCxaCatch(i)
+}
+#endif
+
 mergeInto(LibraryManager.library, LibraryExceptions);

src/library_exceptions_stub.js

@@ -24,7 +24,7 @@ var LibraryExceptions = {};
 #if !INCLUDE_FULL_LIBRARY
   // This method of link-time error genertation is not compatible with INCLUDE_FULL_LIBRARY
   LibraryExceptions[name + '__deps'] = [function() {
-    error('DISABLE_EXCEPTION_THROWING was set (likely due to -fno-exceptions), which means no C++ exception throwing support code is linked in, but such support is required by symbol ' + name + '. Either do not set DISABLE_EXCEPTION_THROWING (if you do want exception throwing) or compile all source files with -fno-except (so that no exceptions support code is required); also make sure DISABLE_EXCEPTION_CATCHING is set to the right value - if you want exceptions, it should be off, and vice versa.');
+    error(`DISABLE_EXCEPTION_THROWING was set (likely due to -fno-exceptions), which means no C++ exception throwing support code is linked in, but such support is required by symbol '${name}'. Either do not set DISABLE_EXCEPTION_THROWING (if you do want exception throwing) or compile all source files with -fno-except (so that no exceptions support code is required); also make sure DISABLE_EXCEPTION_CATCHING is set to the right value - if you want exceptions, it should be off, and vice versa.`);
   }];
 #endif
 });

src/library_glew.js

@@ -20,7 +20,7 @@
  */
 
 var LibraryGLEW = {
-  $GLEW__deps: ['glGetString', '$stringToNewUTF8'],
+  $GLEW__deps: ['glGetString', '$stringToNewUTF8', '$UTF8ToString'],
   $GLEW: {
     isLinaroFork: 1,
     extensions: null,

src/library_glfw.js

@@ -1123,7 +1123,7 @@ var LibraryGLFW = {
 /*******************************************************************************
  * GLFW FUNCTIONS
  ******************************************************************************/
-  glfwInit__deps: ['emscripten_get_device_pixel_ratio'],
+  glfwInit__deps: ['emscripten_get_device_pixel_ratio', 'malloc', 'free'],
   glfwInit__sig: 'i',
   glfwInit: function() {
     if (GLFW.windows) return 1; // GL_TRUE
@@ -1271,6 +1271,7 @@ var LibraryGLFW = {
   glfwPostEmptyEvent__sig: 'v',
   glfwPostEmptyEvent: function() {},
 
+  glfwGetMonitors__deps: ['malloc'],
   glfwGetMonitors__sig: 'ii',
   glfwGetMonitors__deps: ['malloc'],
   glfwGetMonitors: function(count) {