Build a complete sysroot in the cache directory

Rather than adding various include paths, copy any needed headers into
the sysroot along with any libraries.

This means that emscripten can work a lot more like the
traditional cross compiler (e.g. clang -target=xxx --sysroot=yyy),
and we can start to think of the emscripten driver as a seperate
thing to the sysroot.

Fixes: #9353

Author: sbc100

ChangeLog.md

@@ -27,6 +27,9 @@ Current Trunk
   may notice that `RelWithDebInfo` will now include debug info (it did not
   before, which appears to have been an error), and that `Release` will
   use `-O3` instead of `-O2` (which is a better choice anyhow). (#13083)
+- Emscripten now builds a complete sysroot insde the EM_CACHE directory.
+  This includes the system headers which get copied into place there rather
+  than adding a sequence of extra include directories.
 
 2.0.11: 12/17/2020
 ------------------

embuilder.py

@@ -171,6 +171,10 @@ def main():
       if force:
         library.erase()
       library.get_path()
+    elif what == 'sysroot':
+      if force:
+        shared.Cache.erase_file('sysroot_install.stamp')
+      system_libs.ensure_sysroot()
     elif what == 'struct_info':
       if force:
         shared.Cache.erase_file('generated_struct_info.json')

emcc.py

@@ -1331,12 +1331,6 @@ def filter_out_duplicate_dynamic_libs(inputs):
                                              '_emscripten_stack_get_end',
                                              '_emscripten_stack_set_limits']
 
-    if not compile_only and not options.post_link:
-      ldflags = shared.emsdk_ldflags(newargs)
-      for f in ldflags:
-        newargs.append(f)
-        add_link_flag(len(newargs), f)
-
     # SSEx is implemented on top of SIMD128 instruction set, but do not pass SSE flags to LLVM
     # so it won't think about generating native x86 SSE code.
     newargs = [x for x in newargs if x not in shared.SIMD_INTEL_FEATURE_TOWER and x not in shared.SIMD_NEON_FLAGS]
@@ -1950,6 +1944,7 @@ def is_link_flag(flag):
 
     compile_args = [a for a in newargs if a and not is_link_flag(a)]
     cflags = calc_cflags(options)
+    system_libs.ensure_sysroot()
     system_libs.add_ports_cflags(cflags, shared.Settings)
 
     def use_cxx(src):
@@ -1976,9 +1971,8 @@ def get_compiler(cxx):
       return CC
 
     def get_clang_command(src_file):
-      cxx = use_cxx(src_file)
-      per_file_cflags = shared.get_cflags(args, cxx)
-      return get_compiler(cxx) + cflags + per_file_cflags + compile_args + [src_file]
+      per_file_cflags = shared.get_cflags(args)
+      return get_compiler(use_cxx(src_file)) + cflags + per_file_cflags + compile_args + [src_file]
 
     def get_clang_command_asm(src_file):
       asflags = shared.get_clang_flags()
@@ -2070,6 +2064,10 @@ def compile_source_file(i, input_file):
   if specified_target and specified_target.startswith('-'):
     exit_with_error('invalid output filename: `%s`' % specified_target)
 
+  ldflags = shared.emsdk_ldflags(newargs)
+  for f in ldflags:
+    add_link_flag(sys.maxsize, f)
+
   using_lld = not (link_to_object and shared.Settings.LTO)
   link_flags = filter_link_flags(link_flags, using_lld)
 

src/struct_info.json

@@ -1454,7 +1454,7 @@
             ]
     },
     {
-        "file": "../lib/libc/musl/src/internal/pthread_impl.h",
+        "file": "system/lib/libc/musl/src/internal/pthread_impl.h",
         "structs": {
             "pthread": [
               "threadStatus",
@@ -1477,7 +1477,7 @@
         "defines": ["__ATTRP_C11_THREAD"]
     },
     {
-        "file": "../lib/libc/musl/src/internal/libc.h",
+        "file": "system/lib/libc/musl/src/internal/libc.h",
         "structs": {
             "libc": [
               "global_locale"

system/lib/fetch/asmfs.cpp

@@ -15,7 +15,7 @@
 #include <emscripten/emscripten.h>
 #include <emscripten/fetch.h>
 #include <emscripten/threading.h>
-#include <libc/fcntl.h>
+#include <fcntl.h>
 #include <math.h>
 #include <string.h>
 #include <sys/stat.h>

tests/test_other.py

@@ -480,10 +480,10 @@ def test_emcc_asm_v_wasm(self):
   def test_emcc_cflags(self):
     output = self.run_process([EMCC, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], False)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     output = self.run_process([EMXX, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], True)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     # check they work
     cmd = [CLANG_CXX, path_from_root('tests', 'hello_world.cpp')] + shlex.split(flags.replace('\\', '\\\\')) + ['-c', '-o', 'out.o']
     self.run_process(cmd)

tests/test_sanity.py

@@ -691,10 +691,10 @@ def test_embuilder_wasm_backend(self):
     # the --lto flag makes us build wasm-bc
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm-lto'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
 
   def test_binaryen_version(self):
     restore_and_set_up()

tools/cache.py

@@ -84,21 +84,32 @@ def erase(self):
   def get_path(self, name):
     return os.path.join(self.dirname, name)
 
+  def get_sysroot_dir(self, absolute):
+    if absolute:
+      return os.path.join(self.dirname, 'sysroot')
+    return 'sysroot'
+
   def get_include_dir(self):
-    return os.path.join(self.dirname, 'include')
+    return os.path.join(self.get_sysroot_dir(absolute=True), 'include')
 
-  def get_lib_dir(self):
-    subdir = 'wasm'
+  def get_lib_dir(self, absolute):
+    path = os.path.join(self.get_sysroot_dir(absolute=absolute), 'lib')
+    if shared.Settings.MEMORY64:
+      path = os.path.join(path, 'wasm64-emscripten')
+    else:
+      path = os.path.join(path, 'wasm32-emscripten')
+    # if relevant, use a subdir of the cache
+    subdir = []
     if shared.Settings.LTO:
-      subdir += '-lto'
+      subdir.append('lto')
     if shared.Settings.RELOCATABLE:
-      subdir += '-pic'
-    if shared.Settings.MEMORY64:
-      subdir += '-memory64'
-    return subdir
+      subdir.append('pic')
+    if subdir:
+      path = os.path.join(path, '-'.join(subdir))
+    return path
 
   def get_lib_name(self, name):
-    return os.path.join(self.get_lib_dir(), name)
+    return os.path.join(self.get_lib_dir(absolute=False), name)
 
   def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))

tools/gen_struct_info.py

@@ -246,6 +246,7 @@ def inspect_code(headers, cpp_opts, structs, defines):
                                     '-O0',
                                     '-Werror',
                                     '-Wno-format',
+                                    '-I', shared.path_from_root(),
                                     '-s', 'BOOTSTRAPPING_STRUCT_INFO=1',
                                     '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0',
                                     '-s', 'STRICT=1',

tools/ports/regal.py

@@ -46,7 +46,7 @@ def create():
 
     # includes
     source_path_include = os.path.join(ports.get_dir(), 'regal', 'regal-' + TAG, 'include', 'GL')
-    ports.install_header_dir(source_path_include)
+    ports.install_headers(source_path_include, target='GL')
 
     # build
     srcs_regal = ['regal/RegalShaderInstance.cpp',