Reapply "[runtimes] Allow building against an installed LLVM tree"

This relands #86209 which was reverted because ./bin/llvm no longer
accepted test paths in the source tree instead of the build tree. This was
happening because `add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit`
was called before all tsst suites were registered, and therefore it was
missing the source->build dir mappings.

Original commit message:

I am currently trying to test the LLVM runtimes (including compiler-rt)
against an installed LLVM tree rather than a build tree (since that is
no longer available). Currently, the runtimes build of compiler-rt assumes
that LLVM_BINARY_DIR is writable since it uses configure_file() to write
there during the CMake configure stage. Instead, generate this file inside
CMAKE_CURRENT_BINARY_DIR, which will match LLVM_BINARY_DIR when invoked
from llvm/runtimes/CMakeLists.txt.

I also needed to make a minor change to the hwasan tests: hwasan_symbolize
was previously found in the LLVM_BINARY_DIR, but since it is generated as
part of the compiler-rt build it is now inside the CMake build directory
instead. I fixed this by passing the output directory to lit as
config.compiler_rt_bindir and using llvm_config.add_tool_substitutions().

For testing that we no longer write to the LLVM install directory as
part of testing or configuration, I created a read-only bind mount and
configured the runtimes builds as follows:
```
$ sudo mount --bind --read-only ~/llvm-install /tmp/upstream-llvm-readonly
$ cmake -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_C_COMPILER=/tmp/upstream-llvm-readonly/bin/clang \
  -DCMAKE_CXX_COMPILER=/tmp/upstream-llvm-readonly/bin/clang++ \
  -DLLVM_INCLUDE_TESTS=TRUE -DLLVM_ENABLE_ASSERTIONS=TRUE \
  -DCOMPILER_RT_INCLUDE_TESTS=TRUE -DCOMPILER_RT_DEBUG=OFF \
  -DLLVM_ENABLE_RUNTIMES=compiler-rt \
  -DCMAKE_DISABLE_FIND_PACKAGE_LLVM=TRUE \
  -DCMAKE_DISABLE_FIND_PACKAGE_Clang=TRUE \
  -G Ninja -S ~/upstream-llvm-project/runtimes \
  -B ~/upstream-llvm-project/runtimes/cmake-build-debug-llvm-git
```

Pull Request: #114307

Author: arichardson

compiler-rt/cmake/Modules/AddCompilerRT.cmake

@@ -790,6 +790,7 @@ function(configure_compiler_rt_lit_site_cfg input output)
 
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_OUTPUT_DIR ${COMPILER_RT_OUTPUT_DIR})
+  string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR ${COMPILER_RT_EXEC_OUTPUT_DIR})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR ${output_dir})
 
   configure_lit_site_cfg(${input} ${output})

compiler-rt/test/hwasan/lit.cfg.py

@@ -2,6 +2,9 @@
 
 import os
 
+from lit.llvm import llvm_config
+from lit.llvm.subst import ToolSubst, FindTool
+
 # Setup config name.
 config.name = "HWAddressSanitizer" + getattr(config, "name_suffix", "default")
 
@@ -74,6 +77,12 @@ def build_invocation(compile_flags):
     ("%env_hwasan_opts=", "env HWASAN_OPTIONS=" + default_hwasan_opts_str)
 )
 
+# Ensure that we can use hwasan_symbolize from the expected location
+llvm_config.add_tool_substitutions(
+    [ToolSubst("hwasan_symbolize", unresolved="fatal")],
+    search_dirs=[config.compiler_rt_bindir],
+)
+
 # Default test suffixes.
 config.suffixes = [".c", ".cpp"]
 

compiler-rt/test/lit.common.configured.in

@@ -28,6 +28,7 @@ set_default("python_executable", "@Python3_EXECUTABLE@")
 set_default("compiler_rt_debug", @COMPILER_RT_DEBUG_PYBOOL@)
 set_default("compiler_rt_intercept_libdispatch", @COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL@)
 set_default("compiler_rt_output_dir", "@COMPILER_RT_RESOLVED_OUTPUT_DIR@")
+set_default("compiler_rt_bindir", "@COMPILER_RT_RESOLVED_EXEC_OUTPUT_DIR@")
 set_default("compiler_rt_libdir", "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@")
 set_default("emulator", "@COMPILER_RT_EMULATOR@")
 set_default("asan_shadow_scale", "@COMPILER_RT_ASAN_SHADOW_SCALE@")

runtimes/CMakeLists.txt

@@ -1,6 +1,18 @@
 # This file handles building LLVM runtime sub-projects.
 cmake_minimum_required(VERSION 3.20.0)
 
+# This file can be used in two ways: the bootstrapping build calls it from
+# llvm/runtimes/CMakeLists.txt where we reuse the build tree of the top-level
+# build or it can be directly invoked in this directory. In the latter case we
+# might be building against a LLVM install tree and might not have a valid build
+# tree set up yet. We can detect whether we are using the bootstrapping build
+# by checking for the HAVE_LLVM_LIT flag that is passed explicitly to
+# llvm_ExternalProject_Add().
+if (HAVE_LLVM_LIT)
+  message(STATUS "Performing bootstrapping runtimes build.")
+else()
+  message(STATUS "Performing standalone runtimes build.")
+endif()
 # Add path for custom and the LLVM build's modules to the CMake module path.
 set(LLVM_COMMON_CMAKE_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../cmake")
 include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
@@ -236,6 +248,25 @@ foreach(entry ${runtimes})
 endforeach()
 
 if(LLVM_INCLUDE_TESTS)
+  # If built with the runtimes build (rooted at runtimes/CMakeLists.txt), we
+  # won't have llvm-lit. If built with the bootstrapping build (rooted at
+  # llvm/CMakeLists.txt), the top-level llvm CMake invocation already generated
+  # the llvm-lit script.
+  if (NOT HAVE_LLVM_LIT)
+    # Ensure that the appropriate variables for lit are set before adding any
+    # runtimes since their CMake tests configuration might depend on lit being
+    # present. This ensures that the testsuites use a local lit from the build
+    # dir rather than ${LLVM_INSTALL_DIR}/bin/llvm-lit (which may not exist if
+    # LLVM_BINARY_DIR points at an installed LLVM tree rather than a build tree).
+    set(LLVM_LIT_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin)
+    get_llvm_lit_path(_base_dir _file_name)
+    set(LLVM_EXTERNAL_LIT "${_base_dir}/${_file_name}" CACHE STRING "Command used to spawn lit" FORCE)
+    # Avoid warning about missing llvm-lit from runtimes CMake files. This is
+    # fine since we call configure_file() to create llvm-lit at the end of this
+    # file (after recursing into all runtimes' CMake logic), so it will exist.
+    set(LLVM_EXTERNAL_LIT_MISSING_WARNED_ONCE YES CACHE INTERNAL "")
+  endif()
+
   set(LIT_ARGS_DEFAULT "-sv --show-xfail --show-unsupported")
   if (MSVC OR XCODE)
     set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
@@ -273,6 +304,8 @@ if(LLVM_INCLUDE_TESTS)
     # If built by manually invoking cmake on this directory, we don't have
     # llvm-lit. If invoked via llvm/runtimes, the toplevel llvm cmake
     # invocation already generated the llvm-lit script.
+    # NOTE: this must be called after all testsuites have been added, since
+    # otherwise the generated llvm-lit does not have all required path mappings.
     add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
                      ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
   endif()
@@ -306,10 +339,10 @@ if(SUB_COMPONENTS)
   if(LLVM_RUNTIMES_TARGET)
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
+      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Components.cmake)
   else()
     configure_file(
       ${CMAKE_CURRENT_SOURCE_DIR}/Components.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/Components.cmake)
+      ${CMAKE_CURRENT_BINARY_DIR}/runtimes/Components.cmake)
   endif()
 endif()