Build a complete sysroot in the cache directory

Rather than adding various include paths, copy any needed headers into
the sysroot along with any libraries.

This means that emscripten can work a lot more like the
traditional cross compiler (e.g. clang -target=xxx --sysroot=yyy),
and we can start to think of the emscripten driver as a seperate
thing to the sysroot.

Fixes: #9353

Author: sbc100

ChangeLog.md

@@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works.
 
 Current Trunk
 -------------
+- Emscripten now builds a complete sysroot insde the EM_CACHE directory.
+  This includes the system headers which get copied into place there rather
+  than adding a sequence of extra include directories.
 
 2.0.11: 12/17/2020
 ------------------

embuilder.py

@@ -129,13 +129,9 @@ def main():
 
   if args.lto:
     shared.Settings.LTO = "full"
-    # Reconfigure the cache dir to reflect the change
-    shared.reconfigure_cache()
 
   if args.pic:
     shared.Settings.RELOCATABLE = 1
-    # Reconfigure the cache dir to reflect the change
-    shared.reconfigure_cache()
 
   if args.force:
     force = True
@@ -177,6 +173,10 @@ def main():
       if force:
         library.erase()
       library.get_path()
+    elif what == 'sysroot':
+      if force:
+        shared.Cache.erase_file('sysroot_install.stamp')
+      system_libs.ensure_sysroot()
     elif what == 'struct_info':
       if force:
         shared.Cache.erase_file('generated_struct_info.json')

emcc.py

@@ -1329,16 +1329,6 @@ def filter_out_duplicate_dynamic_libs(inputs):
                                              '_emscripten_stack_get_end',
                                              '_emscripten_stack_set_limits']
 
-    # Reconfigure the cache now that settings have been applied. Some settings
-    # such as LTO and SIDE_MODULE/MAIN_MODULE effect which cache directory we use.
-    shared.reconfigure_cache()
-
-    if not compile_only and not options.post_link:
-      ldflags = shared.emsdk_ldflags(newargs)
-      for f in ldflags:
-        newargs.append(f)
-        add_link_flag(len(newargs), f)
-
     # SSEx is implemented on top of SIMD128 instruction set, but do not pass SSE flags to LLVM
     # so it won't think about generating native x86 SSE code.
     newargs = [x for x in newargs if x not in shared.SIMD_INTEL_FEATURE_TOWER and x not in shared.SIMD_NEON_FLAGS]
@@ -1943,6 +1933,7 @@ def is_link_flag(flag):
 
     compile_args = [a for a in newargs if a and not is_link_flag(a)]
     cflags = calc_cflags(options)
+    system_libs.ensure_sysroot()
     system_libs.add_ports_cflags(cflags, shared.Settings)
 
     def use_cxx(src):
@@ -1969,9 +1960,8 @@ def get_compiler(cxx):
       return CC
 
     def get_clang_command(src_file):
-      cxx = use_cxx(src_file)
-      per_file_cflags = shared.get_cflags(args, cxx)
-      return get_compiler(cxx) + cflags + per_file_cflags + compile_args + [src_file]
+      per_file_cflags = shared.get_cflags(args)
+      return get_compiler(use_cxx(src_file)) + cflags + per_file_cflags + compile_args + [src_file]
 
     def get_clang_command_asm(src_file):
       asflags = shared.get_asmflags()
@@ -2067,6 +2057,10 @@ def compile_source_file(i, input_file):
   if specified_target and specified_target.startswith('-'):
     exit_with_error('invalid output filename: `%s`' % specified_target)
 
+  ldflags = shared.emsdk_ldflags(newargs)
+  for f in ldflags:
+    add_link_flag(sys.maxsize, f)
+
   using_lld = not (link_to_object and shared.Settings.LTO)
   link_flags = filter_link_flags(link_flags, using_lld)
 

src/struct_info.json

@@ -1525,7 +1525,7 @@
             ]
     },
     {
-        "file": "../lib/libc/musl/src/internal/pthread_impl.h",
+        "file": "system/lib/libc/musl/src/internal/pthread_impl.h",
         "structs": {
             "pthread": [
               "threadStatus",
@@ -1548,7 +1548,7 @@
         "defines": ["__ATTRP_C11_THREAD"]
     },
     {
-        "file": "../lib/libc/musl/src/internal/libc.h",
+        "file": "system/lib/libc/musl/src/internal/libc.h",
         "structs": {
             "libc": [
               "global_locale"

system/lib/fetch/asmfs.cpp

@@ -15,7 +15,7 @@
 #include <emscripten/emscripten.h>
 #include <emscripten/fetch.h>
 #include <emscripten/threading.h>
-#include <libc/fcntl.h>
+#include <fcntl.h>
 #include <math.h>
 #include <string.h>
 #include <sys/stat.h>

tests/test_other.py

@@ -471,10 +471,10 @@ def test_emcc_asm_v_wasm(self):
   def test_emcc_cflags(self):
     output = self.run_process([EMCC, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], False)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     output = self.run_process([EMXX, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], True)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     # check they work
     cmd = [CLANG_CXX, path_from_root('tests', 'hello_world.cpp')] + shlex.split(flags.replace('\\', '\\\\')) + ['-c', '-emit-llvm', '-o', 'a.bc']
     self.run_process(cmd)

tests/test_sanity.py

@@ -21,7 +21,7 @@
 from tools.shared import EXPECTED_LLVM_VERSION, Cache
 from tools import shared, system_libs, utils
 
-SANITY_FILE = shared.Cache.get_path('sanity.txt', root=True)
+SANITY_FILE = shared.Cache.get_path('sanity.txt')
 commands = [[EMCC], [PYTHON, path_from_root('tests', 'runner.py'), 'blahblah']]
 
 
@@ -415,31 +415,29 @@ def ensure_cache(self):
     self.do([EMCC, '-O2', path_from_root('tests', 'hello_world.c')])
 
   def test_emcc_caching(self):
-    BUILDING_MESSAGE = 'generating system library: X'
+    BUILDING_MESSAGE = 'generating system library: %s'
 
     restore_and_set_up()
     self.erase_cache()
 
     # Building a file that *does* need something *should* trigger cache
     # generation, but only the first time
-    libname = 'libc++'
+    libname = Cache.get_lib_name('libc++.a')
     for i in range(3):
       print(i)
       self.clear()
       output = self.do([EMCC, '-O' + str(i), path_from_root('tests', 'hello_libcxx.cpp'), '-s', 'DISABLE_EXCEPTION_CATCHING=0'])
       print('\n\n\n', output)
-      self.assertContainedIf(BUILDING_MESSAGE.replace('X', libname), output, i == 0)
+      self.assertContainedIf(BUILDING_MESSAGE % libname, output, i == 0)
       self.assertContained('hello, world!', self.run_js('a.out.js'))
       self.assertExists(Cache.dirname)
-      full_libname = libname + '.bc' if libname != 'libc++' else libname + '.a'
-      self.assertExists(os.path.join(Cache.dirname, full_libname))
+      self.assertExists(os.path.join(Cache.dirname, libname))
 
   def test_cache_clearing_manual(self):
     # Manual cache clearing
     restore_and_set_up()
     self.ensure_cache()
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     output = self.do([EMCC, '--clear-cache'])
     self.assertIn('clearing cache', output)
     self.assertIn(SANITY_MESSAGE, output)
@@ -463,12 +461,10 @@ def test_FROZEN_CACHE(self):
     self.erase_cache()
     self.ensure_cache()
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     # changing config file should not clear cache
     add_to_config('FROZEN_CACHE = True')
     self.do([EMCC])
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     # building libraries is disallowed
     output = self.do([EMBUILDER, 'build', 'libemmalloc'])
     self.assertIn('FROZEN_CACHE disallows building system libs', output)
@@ -487,6 +483,7 @@ def test_emcc_multiprocess_cache_access(self):
       }
       ''')
     cache_dir_name = self.in_dir('test_cache')
+    libname = Cache.get_lib_name('libc.a')
     with env_modify({'EM_CACHE': cache_dir_name}):
       tasks = []
       num_times_libc_was_built = 0
@@ -495,13 +492,13 @@ def test_emcc_multiprocess_cache_access(self):
         tasks += [p]
       for p in tasks:
         print('stdout:\n', p.stdout)
-        if 'generating system library: libc' in p.stdout:
+        if 'generating system library: ' + libname in p.stdout:
           num_times_libc_was_built += 1
 
     # The cache directory must exist after the build
     self.assertTrue(os.path.exists(cache_dir_name))
     # The cache directory must contain a built libc
-    self.assertTrue(os.path.exists(os.path.join(cache_dir_name, 'wasm', 'libc.a')))
+    self.assertTrue(os.path.exists(os.path.join(cache_dir_name, libname)))
     # Exactly one child process should have triggered libc build!
     self.assertEqual(num_times_libc_was_built, 1)
 
@@ -535,12 +532,11 @@ def test_emcc_ports(self):
     restore_and_set_up()
 
     # listing ports
-
     out = self.do([EMCC, '--show-ports'])
-    assert 'Available ports:' in out, out
-    assert 'SDL2' in out, out
-    assert 'SDL2_image' in out, out
-    assert 'SDL2_net' in out, out
+    self.assertContained('Available ports:', out)
+    self.assertContained('SDL2', out)
+    self.assertContained('SDL2_image', out)
+    self.assertContained('SDL2_net', out)
 
     # using ports
     RETRIEVING_MESSAGE = 'retrieving port'
@@ -555,27 +551,27 @@ def test_emcc_ports(self):
         try_delete(PORTS_DIR)
       else:
         self.do([EMCC, '--clear-ports'])
-      assert not os.path.exists(PORTS_DIR)
+      self.assertNotExists(PORTS_DIR)
 
       # Building a file that doesn't need ports should not trigger anything
       output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp')])
       assert RETRIEVING_MESSAGE not in output, output
       assert BUILDING_MESSAGE not in output
       print('no', output)
-      assert not os.path.exists(PORTS_DIR)
+      self.assertNotExists(PORTS_DIR)
 
       def first_use():
         output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'USE_SDL=2'])
-        assert RETRIEVING_MESSAGE in output, output
-        assert BUILDING_MESSAGE in output, output
+        self.assertContained(RETRIEVING_MESSAGE, output)
+        self.assertContained(BUILDING_MESSAGE, output)
         self.assertExists(PORTS_DIR)
         print('yes', output)
 
       def second_use():
         # Using it again avoids retrieve and build
         output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'USE_SDL=2'])
-        assert RETRIEVING_MESSAGE not in output, output
-        assert BUILDING_MESSAGE not in output, output
+        self.assertNotContained(RETRIEVING_MESSAGE, output)
+        self.assertNotContained(BUILDING_MESSAGE, output)
 
       # Building a file that need a port does trigger stuff
       first_use()
@@ -699,10 +695,10 @@ def test_embuilder_wasm_backend(self):
     # the --lto flag makes us build wasm-bc
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm-lto'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
 
   def test_binaryen_version(self):
     restore_and_set_up()

tools/cache.py

@@ -24,22 +24,9 @@ class Cache:
   # acquired.
   EM_EXCLUSIVE_CACHE_ACCESS = int(os.environ.get('EM_EXCLUSIVE_CACHE_ACCESS', '0'))
 
-  def __init__(self, dirname, use_subdir=True):
+  def __init__(self, dirname):
     # figure out the root directory for all caching
     dirname = os.path.normpath(dirname)
-    self.root_dirname = dirname
-
-    # if relevant, use a subdir of the cache
-    if use_subdir:
-      subdir = 'wasm'
-      if shared.Settings.LTO:
-        subdir += '-lto'
-      if shared.Settings.RELOCATABLE:
-        subdir += '-pic'
-      if shared.Settings.MEMORY64:
-        subdir += '-memory64'
-      dirname = os.path.join(dirname, subdir)
-
     self.dirname = dirname
     self.acquired_count = 0
 
@@ -90,28 +77,57 @@ def ensure(self):
 
   def erase(self):
     with self.lock():
-      if os.path.exists(self.root_dirname):
-        for f in os.listdir(self.root_dirname):
-          tempfiles.try_delete(os.path.join(self.root_dirname, f))
+      if os.path.exists(self.dirname):
+        for f in os.listdir(self.dirname):
+          tempfiles.try_delete(os.path.join(self.dirname, f))
+
+  def get_path(self, name):
+    return os.path.join(self.dirname, name)
+
+  def get_sysroot_dir(self, absolute):
+    if absolute:
+      return os.path.join(self.dirname, 'sysroot')
+    return 'sysroot'
+
+  def get_include_dir(self):
+    return os.path.join(self.get_sysroot_dir(absolute=True), 'include')
 
-  def get_path(self, shortname, root=False):
-    if root:
-      return os.path.join(self.root_dirname, shortname)
-    return os.path.join(self.dirname, shortname)
+  def get_lib_dir(self, absolute):
+    path = os.path.join(self.get_sysroot_dir(absolute=absolute), 'lib')
+    if shared.Settings.MEMORY64:
+      path = os.path.join(path, 'wasm64-emscripten')
+    else:
+      path = os.path.join(path, 'wasm32-emscripten')
+    # if relevant, use a subdir of the cache
+    subdir = []
+    if shared.Settings.LTO:
+      subdir.append('lto')
+    if shared.Settings.RELOCATABLE:
+      subdir.append('pic')
+    if subdir:
+      path = os.path.join(path, '-'.join(subdir))
+    return path
+
+  def get_lib_name(self, name):
+    return os.path.join(self.get_lib_dir(absolute=False), name)
+
+  def erase_lib(self, name):
+    self.erase_file(self.get_lib_name(name))
 
   def erase_file(self, shortname):
     name = os.path.join(self.dirname, shortname)
     if os.path.exists(name):
       logging.info('Cache: deleting cached file: %s', name)
       tempfiles.try_delete(name)
 
+  def getlib(self, libname, *args, **kwargs):
+    name = self.get_lib_name(libname)
+    return self.get(name, *args, **kwargs)
+
   # Request a cached file. If it isn't in the cache, it will be created with
   # the given creator function
-  def get(self, shortname, creator, what=None, force=False, root=False):
-    if root:
-      cachename = os.path.join(self.root_dirname, shortname)
-    else:
-      cachename = os.path.join(self.dirname, shortname)
+  def get(self, shortname, creator, what=None, force=False):
+    cachename = os.path.join(self.dirname, shortname)
     cachename = os.path.abspath(cachename)
     # Check for existence before taking the lock in case we can avoid the
     # lock completely.

tools/filelock.py

@@ -274,7 +274,7 @@ def acquire(self, timeout=None, poll_intervall=0.05):
                         self._acquire()
 
                 if self.is_locked:
-                    logger().info('Lock %s acquired on %s', lock_id, lock_filename)
+                    logger().debug('Lock %s acquired on %s', lock_id, lock_filename)
                     break
                 elif timeout >= 0 and time.time() - start_time > timeout:
                     logger().debug('Timeout on acquiring lock %s on %s', lock_id, lock_filename)
@@ -318,7 +318,7 @@ def release(self, force = False):
                     logger().debug('Attempting to release lock %s on %s', lock_id, lock_filename)
                     self._release()
                     self._lock_counter = 0
-                    logger().info('Lock %s released on %s', lock_id, lock_filename)
+                    logger().debug('Lock %s released on %s', lock_id, lock_filename)
 
         return None
 

tools/gen_struct_info.py

@@ -246,6 +246,7 @@ def inspect_code(headers, cpp_opts, structs, defines):
                                     '-O0',
                                     '-Werror',
                                     '-Wno-format',
+                                    '-I', shared.path_from_root(),
                                     '-s', 'BOOTSTRAPPING_STRUCT_INFO=1',
                                     '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0',
                                     '-s', 'STRICT=1',