From 0e98c2001858700011e74dd7bbec3c94ea8d2efd Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Wed, 18 Dec 2024 23:35:59 +0000 Subject: [PATCH 1/3] Use /emsdk/emscripten for relative path substitution When `deterministic_paths` is set, we are currently using `-ffile-prefix-map` to produce the same path in data and debug info. In the case of absolute paths, their emscripten path is replaced with a fake path `/emsdk/emscripten`, and in the case of relative paths, all path relative to the emscripten directory is removed, so `../../system/lib/somefile.c` becomes `system/lib/somefiles.c`. https://github.com/emscripten-core/emscripten/blob/f66b5d706e174d9e5cc6122c06ea29dcd2735cd0/tools/system_libs.py#L472-L477 https://github.com/emscripten-core/emscripten/blob/f66b5d706e174d9e5cc6122c06ea29dcd2735cd0/tools/system_libs.py#L495-L501 So this does not make relative paths and absolute paths the same, which can lead to different builds depending on whether the command line uses absolute paths vs. relative ones. Currently we use relative paths when `EMCC_BATCH_BUILD` is set. And Ninja builds cannot use relative paths. This is also what was suggested in https://github.com/emscripten-core/emscripten/issues/23195#issuecomment-2549784273 while discussins `__FILE__` problem in #23915. This does not change any code size tests because no code size tests happens to contain `__FILE__` at the moment. (One will be added in #22994) --- tools/system_libs.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index e6bbf1c3435b1..99e226cb9a8fc 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -42,6 +42,10 @@ # link time. USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0')) +# A fake emscripten path to use in __FILE__ macro and debug info to produce +# reproducible builds across platforms. +FAKE_EMSCRIPTEN_PATH = '/emsdk/emscripten' + def files_in_path(path, filenames): srcdir = utils.path_from_root(path) @@ -472,9 +476,9 @@ def generate_ninja(self, build_dir, libname): if self.deterministic_paths: source_dir = utils.path_from_root() relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', - f'-ffile-prefix-map={relative_source_dir}/=', - '-fdebug-compilation-dir=/emsdk/emscripten'] + cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', + f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}', + '-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] asflags = get_base_cflags(preprocess=False) input_files = self.get_files() ninja_file = os.path.join(build_dir, 'build.ninja') @@ -496,9 +500,9 @@ def build_objects(self, build_dir): source_dir = utils.path_from_root() if batch_inputs: relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={relative_source_dir}/='] - cflags += [f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', - '-fdebug-compilation-dir=/emsdk/emscripten'] + cflags += [f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}'] + cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', + '-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src) From e1d4787c5bd77b7fa283fc6fbdf9165ea8c62527 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 02:37:55 +0000 Subject: [PATCH 2/3] Fix other.test_dwarf_system_lib --- test/test_other.py | 2 +- tools/system_libs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_other.py b/test/test_other.py index 2642b9aafd82c..1ae34b5c98081 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -10409,7 +10409,7 @@ def test_dwarf_system_lib(self): [LLVM_DWARFDUMP, libc, '-debug-info', '-debug-line', '--recurse-depth=0'], stdout=PIPE).stdout # Check that the embedded location of the source file is correct. - self.assertIn('DW_AT_name\t("system/lib/emmalloc.c")', dwdump) + self.assertIn('DW_AT_name\t("/emsdk/emscripten/system/lib/emmalloc.c")', dwdump) self.assertIn('DW_AT_comp_dir\t("/emsdk/emscripten")', dwdump) @parameterized({ diff --git a/tools/system_libs.py b/tools/system_libs.py index 99e226cb9a8fc..0e835e9165856 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -478,7 +478,7 @@ def generate_ninja(self, build_dir, libname): relative_source_dir = os.path.relpath(source_dir, build_dir) cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}', - '-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] + f'-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] asflags = get_base_cflags(preprocess=False) input_files = self.get_files() ninja_file = os.path.join(build_dir, 'build.ninja') @@ -502,7 +502,7 @@ def build_objects(self, build_dir): relative_source_dir = os.path.relpath(source_dir, build_dir) cflags += [f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}'] cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', - '-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] + f'-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src) From 666256add68dbbe03588347642ba723c6a41ffcd Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 02:38:51 +0000 Subject: [PATCH 3/3] DETERMINISITIC_PREFIX --- tools/system_libs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 0e835e9165856..3b01a85fbf651 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -42,9 +42,9 @@ # link time. USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0')) -# A fake emscripten path to use in __FILE__ macro and debug info to produce -# reproducible builds across platforms. -FAKE_EMSCRIPTEN_PATH = '/emsdk/emscripten' +# A (fake) deterministic emscripten path to use in __FILE__ macro and debug info +# to produce reproducible builds across platforms. +DETERMINISITIC_PREFIX = '/emsdk/emscripten' def files_in_path(path, filenames): @@ -476,9 +476,9 @@ def generate_ninja(self, build_dir, libname): if self.deterministic_paths: source_dir = utils.path_from_root() relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', - f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}', - f'-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] + cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', + f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}', + f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] asflags = get_base_cflags(preprocess=False) input_files = self.get_files() ninja_file = os.path.join(build_dir, 'build.ninja') @@ -500,9 +500,9 @@ def build_objects(self, build_dir): source_dir = utils.path_from_root() if batch_inputs: relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={relative_source_dir}={FAKE_EMSCRIPTEN_PATH}'] - cflags += [f'-ffile-prefix-map={source_dir}={FAKE_EMSCRIPTEN_PATH}', - f'-fdebug-compilation-dir={FAKE_EMSCRIPTEN_PATH}'] + cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] + cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', + f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src)