From 97e9a2d0afa1c1cbf5928e3408795b654be6ee8a Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sun, 28 May 2023 17:01:01 -0700 Subject: [PATCH 01/20] [CMake] Add paths for building Swift code against host toolchain When building Swift code into the compiler (e.g, the new Swift parser along with macros support), make sure we always add the appropriate paths to (1) link against the host Swift toolchain, and (2) find the host Swift libraries (such as SwiftSyntax) at runtime. The CMake code for doing this was only running for Darwin builds, so generalize it to also work on Linux. (cherry picked from commit 277619a604ee415206abe9c8d87f836c8e9b816a) --- cmake/modules/AddSwift.cmake | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index 5b8365e930ce8..8545235ecae3d 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -576,7 +576,7 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping) # Make sure we can find the early SwiftSyntax libraries. target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host") - # For the "end step" of bootstrapping configurations on Darwin, need to be + # For the "end step" of bootstrapping configurations, we need to be # able to fall back to the SDK directory for libswiftCore et al. if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*") if (NOT "${bootstrapping}" STREQUAL "1") @@ -590,6 +590,13 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping) get_filename_component(TOOLCHAIN_BIN_DIR ${SWIFT_EXEC_FOR_SWIFT_MODULES} DIRECTORY) get_filename_component(TOOLCHAIN_LIB_DIR "${TOOLCHAIN_BIN_DIR}/../lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}" ABSOLUTE) target_link_directories(${target} PUBLIC ${TOOLCHAIN_LIB_DIR}) + else() + get_filename_component(swift_bin_dir ${SWIFT_EXEC_FOR_SWIFT_MODULES} DIRECTORY) + get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY) + set(host_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}") + target_link_directories(${target} PRIVATE ${host_lib_dir}) + + set(swift_runtime_rpath "${host_lib_dir}") endif() endif() endif() @@ -926,10 +933,17 @@ function(add_swift_host_tool executable) endif() endif() - set_property( - TARGET ${executable} - APPEND PROPERTY INSTALL_RPATH - "@executable_path/../${extra_relative_rpath}lib/swift/host") + if(${SWIFT_HOST_VARIANT_SDK} IN_LIST SWIFT_DARWIN_PLATFORMS) + set_property( + TARGET ${executable} + APPEND PROPERTY INSTALL_RPATH + "@executable_path/../${extra_relative_rpath}lib/swift/host") + else() + set_property( + TARGET ${executable} + APPEND PROPERTY INSTALL_RPATH + "$ORIGIN/../${extra_relative_rpath}lib/swift/host") + endif() endif() if(ASHT_THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY) From 4690e59cad969e0c66939eb4236bf2875aa5d2bc Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sun, 28 May 2023 22:20:25 -0700 Subject: [PATCH 02/20] [CMake] Fix host library builds and rpaths for testing macros This enables running macro tests on Linux. (cherry picked from commit c573a7a3562170e83de93d1eaed9d435c8670168) --- cmake/modules/AddPureSwift.cmake | 12 +++++++++--- test/Macros/macro_swiftdeps.swift | 4 ++-- test/lit.cfg | 11 +++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmake/modules/AddPureSwift.cmake b/cmake/modules/AddPureSwift.cmake index 80a2b32ca8a0c..0c7e852883427 100644 --- a/cmake/modules/AddPureSwift.cmake +++ b/cmake/modules/AddPureSwift.cmake @@ -267,9 +267,15 @@ function(add_pure_swift_host_tool name) add_executable(${name} ${APSHT_SOURCES}) _add_host_swift_compile_options(${name}) - set_property(TARGET ${name} - APPEND PROPERTY INSTALL_RPATH - "@executable_path/../lib/swift/host") + if(${SWIFT_HOST_VARIANT_SDK} IN_LIST SWIFT_DARWIN_PLATFORMS) + set_property(TARGET ${name} + APPEND PROPERTY INSTALL_RPATH + "@executable_path/../lib/swift/host") + else() + set_property(TARGET ${name} + APPEND PROPERTY INSTALL_RPATH + "$ORIGIN/../lib/swift/host") + endif() set_property(TARGET ${name} PROPERTY BUILD_WITH_INSTALL_RPATH YES) diff --git a/test/Macros/macro_swiftdeps.swift b/test/Macros/macro_swiftdeps.swift index dd969f3d6e9a9..2b6438e7ffada 100644 --- a/test/Macros/macro_swiftdeps.swift +++ b/test/Macros/macro_swiftdeps.swift @@ -7,7 +7,7 @@ // RUN: split-file %s %t/src -//#-- Prepare the macro dylib plugin. +//#-- Prepare the macro shared library plugin. // RUN: %host-build-swift \ // RUN: -swift-version 5 \ // RUN: -emit-library -o %t/plugin/%target-library-name(MacroDefinition) \ @@ -76,7 +76,7 @@ // RUN: %FileCheck --check-prefix WITHOUT_PLUGIN %s < %t/with_macro_nonprimary.swiftdeps.processed // WITH_PLUGIN: externalDepend interface '' 'BUILD_DIR{{.*}}mock-plugin' false -// WITH_PLUGIN: externalDepend interface '' 'BUILD_DIR{{.*}}libMacroDefinition.dylib' false +// WITH_PLUGIN: externalDepend interface '' 'BUILD_DIR{{.*}}libMacroDefinition.{{(dylib|so|dll)}}' false // WITHOUT_PLUGIN-NOT: MacroDefinition // WITHOUT_PLUGIN-NOT: mock-plugin diff --git a/test/lit.cfg b/test/lit.cfg index d74963fc49c0c..39a306ae30644 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -544,9 +544,16 @@ else: "env SDKROOT=%s %r -toolchain-stdlib-rpath -Xlinker -rpath -Xlinker /usr/lib/swift %s %s %s" % (shell_quote(config.host_sdkroot), config.swiftc, mcp_opt, config.swift_test_options, config.swift_driver_test_options)) +# Parse the host triple. +(host_cpu, host_vendor, host_os, host_vers) = re.match('([^-]+)-([^-]+)-([^0-9-]+)(.*)', config.host_triple).groups() + +if platform.system() == 'Darwin': + host_build_extra_rpath="" +else: + host_build_extra_rpath="-Xlinker -rpath -Xlinker %s" % (make_path(config.swift_lib_dir, 'swift', host_os)) + config.host_build_swift = ( - "%s -sdk %s -target %s -I %s -L %s" % (config.swiftc_driver, config.host_sdkroot, config.host_triple, config.swift_host_lib_dir, config.swift_host_lib_dir) -) + "%s -sdk %s -target %s -I %s -L %s %s" % (config.swiftc_driver, config.host_sdkroot, config.host_triple, config.swift_host_lib_dir, config.swift_host_lib_dir, host_build_extra_rpath)) config.substitutions.append( ('%llvm_obj_root', config.llvm_obj_root) ) config.substitutions.append( ('%swift-lib-dir', config.swift_lib_dir) ) From b3b1b7a7b8595471760b5e547ee0b8f4e264bace Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sun, 28 May 2023 23:27:15 -0700 Subject: [PATCH 03/20] [SourceKit] Fix paths when including the new Swift compiler (cherry picked from commit defe9ebaba860c6493affb4a4e360b60d58f7e69) --- .../cmake/modules/AddSwiftSourceKit.cmake | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/SourceKit/cmake/modules/AddSwiftSourceKit.cmake b/tools/SourceKit/cmake/modules/AddSwiftSourceKit.cmake index 5c480dcd4a0ba..f44b8de53895e 100644 --- a/tools/SourceKit/cmake/modules/AddSwiftSourceKit.cmake +++ b/tools/SourceKit/cmake/modules/AddSwiftSourceKit.cmake @@ -153,12 +153,17 @@ function(add_sourcekit_swift_runtime_link_flags target path HAS_SWIFT_MODULES) message(FATAL_ERROR "Unknown ASKD_BOOTSTRAPPING_MODE '${ASKD_BOOTSTRAPPING_MODE}'") endif() endif() - set(RPATH_LIST ${RPATH_LIST} PARENT_SCOPE) if(SWIFT_SWIFT_PARSER) # Make sure we can find the early SwiftSyntax libraries. target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host") + # Add rpath to the host Swift libraries. + if (NOT ${SWIFT_HOST_VARIANT_SDK} IN_LIST SWIFT_DARWIN_PLATFORMS) + file(RELATIVE_PATH relative_hostlib_path "${path}" "${SWIFTLIB_DIR}/host") + list(APPEND RPATH_LIST "$ORIGIN/${relative_hostlib_path}") + endif() + # For the "end step" of bootstrapping configurations on Darwin, need to be # able to fall back to the SDK directory for libswiftCore et al. if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*") @@ -173,10 +178,19 @@ function(add_sourcekit_swift_runtime_link_flags target path HAS_SWIFT_MODULES) get_filename_component(TOOLCHAIN_BIN_DIR ${SWIFT_EXEC_FOR_SWIFT_MODULES} DIRECTORY) get_filename_component(TOOLCHAIN_LIB_DIR "${TOOLCHAIN_BIN_DIR}/../lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}" ABSOLUTE) target_link_directories(${target} PUBLIC ${TOOLCHAIN_LIB_DIR}) + else() + get_filename_component(swift_bin_dir ${SWIFT_EXEC_FOR_SWIFT_MODULES} DIRECTORY) + get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY) + set(host_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}") + target_link_directories(${target} PUBLIC ${host_lib_dir}) + + list(APPEND RPATH_LIST "${host_lib_dir}") endif() endif() endif() endif() + + set(RPATH_LIST ${RPATH_LIST} PARENT_SCOPE) endfunction() # Add a new SourceKit library. From 38b6f0e5e7164490268b100f6329c9ee65ed0ec3 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Sat, 19 Aug 2023 00:18:53 +0000 Subject: [PATCH 04/20] [CMake] Support Macros in Linux For compiling codes required for macro support, we now need swiftc compiler in the build machine. Unlike Darwin OSes, where swiftCore runtime is guaranteed to be present in /usr/lib, Linux doesn't have ABI stability and the stdlib of the build machine is not at the specific location. So the built compiler cannot relies on the shared object in the toolchain. (cherry picked from commit 9c9010e5b7506c4bd89c5ef0148e4741b32a1a2c) (cherry picked from commit 040d15d7e8727a192dc3944199bfc93257bef78c) --- CMakeLists.txt | 1 + SwiftCompilerSources/CMakeLists.txt | 8 ++- cmake/modules/AddPureSwift.cmake | 22 ++++++ cmake/modules/AddSwift.cmake | 22 ++++-- cmake/modules/AddSwiftUnittests.cmake | 27 ++++++- cmake/modules/SwiftImplicitImport.cmake | 2 +- cmake/modules/SwiftStripBuilderRpath.cmake | 71 +++++++++++++++++++ lib/CMakeLists.txt | 43 +++++++++-- lib/Macros/CMakeLists.txt | 25 ++++++- lib/SwiftDemangle/CMakeLists.txt | 7 ++ test/lit.cfg | 39 +++++----- .../cmake/modules/AddSwiftSourceKit.cmake | 26 +++---- .../tools/complete-test/CMakeLists.txt | 6 ++ .../tools/sourcekitd-repl/CMakeLists.txt | 6 ++ .../tools/sourcekitd-test/CMakeLists.txt | 6 ++ tools/libStaticMirror/CMakeLists.txt | 6 ++ tools/libSwiftScan/CMakeLists.txt | 29 ++++++-- tools/swift-plugin-server/CMakeLists.txt | 5 ++ unittests/SwiftDemangle/CMakeLists.txt | 2 +- unittests/Threading/CMakeLists.txt | 2 +- unittests/runtime/CMakeLists.txt | 2 +- unittests/runtime/LongTests/CMakeLists.txt | 2 +- utils/build-presets.ini | 3 +- .../products/earlyswiftsyntax.py | 20 ++---- 24 files changed, 305 insertions(+), 77 deletions(-) create mode 100644 cmake/modules/SwiftStripBuilderRpath.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 342f60855e32a..12e9684f6ca2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -799,6 +799,7 @@ include(SwiftConfigureSDK) include(SwiftComponents) include(SwiftList) include(AddPureSwift) +include(SwiftStripBuilderRpath) # Configure swift include, install, build components. swift_configure_components() diff --git a/SwiftCompilerSources/CMakeLists.txt b/SwiftCompilerSources/CMakeLists.txt index 89a78c6486078..51b575aec4b1c 100644 --- a/SwiftCompilerSources/CMakeLists.txt +++ b/SwiftCompilerSources/CMakeLists.txt @@ -231,7 +231,7 @@ else() # # step 1: generate a dummy source file, which just includes all headers # defined in include/swift/module.modulemap - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp" + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp" " #include \"Basic/BridgedSwiftObject.h\" #include \"Basic/BasicBridging.h\" @@ -247,6 +247,12 @@ else() #include \"Parse/RegexParserBridging.h\" ") + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp" + "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp" + ) # step 2: build a library containing that source file. This library depends on all the included header files. # The swift modules can now depend on that target. diff --git a/cmake/modules/AddPureSwift.cmake b/cmake/modules/AddPureSwift.cmake index 0c7e852883427..205068f27622e 100644 --- a/cmake/modules/AddPureSwift.cmake +++ b/cmake/modules/AddPureSwift.cmake @@ -65,6 +65,27 @@ function(_add_host_swift_compile_options name) _add_host_variant_swift_sanitizer_flags(${name}) endfunction() +function(_set_pure_swift_link_flags name relpath_to_lib_dir) + if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX") + # Don't add builder's stdlib RPATH automatically, because we want to *de-prioritize* it. + target_compile_options(${name} PRIVATE + $<$:-no-toolchain-stdlib-rpath> + ) + + get_filename_component(swift_bin_dir ${CMAKE_Swift_COMPILER} DIRECTORY) + get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY) + set(host_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}") + + set_property(TARGET ${name} + APPEND PROPERTY INSTALL_RPATH + # At runtime, use swiftCore in the current toolchain. + "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}" + # But before building the stdlib with the tool, use the builder libs. This should be removed in install time. + "${host_lib_dir}") + + endif() +endfunction() + # Add a new "pure" Swift host library. # # "Pure" Swift host libraries can only contain Swift code, and will be built @@ -266,6 +287,7 @@ function(add_pure_swift_host_tool name) # Create the library. add_executable(${name} ${APSHT_SOURCES}) _add_host_swift_compile_options(${name}) + _set_pure_swift_link_flags(${name} "../lib") if(${SWIFT_HOST_VARIANT_SDK} IN_LIST SWIFT_DARWIN_PLATFORMS) set_property(TARGET ${name} diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index 8545235ecae3d..8896d16c0cc3f 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -545,10 +545,14 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping) target_link_libraries(${target} PRIVATE "swiftCore") target_link_directories(${target} PRIVATE ${host_lib_dir}) + + # At runtime, use swiftCore in the current toolchain. + # FIXME: This assumes the ABI hasn't changed since the builder. + set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}") + if(ASRLF_BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS") - set(swift_runtime_rpath "${host_lib_dir}") - else() - set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}") + # But before building the stdlib with the tool, use the builder libs. This should be removed in install time. + list(APPEND swift_runtime_rpath "${host_lib_dir}") endif() elseif(ASRLF_BOOTSTRAPPING_MODE STREQUAL "BOOTSTRAPPING") @@ -573,9 +577,6 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping) endif() if(SWIFT_SWIFT_PARSER) - # Make sure we can find the early SwiftSyntax libraries. - target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host") - # For the "end step" of bootstrapping configurations, we need to be # able to fall back to the SDK directory for libswiftCore et al. if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*") @@ -964,7 +965,14 @@ function(add_swift_host_tool executable) swift_install_in_component(TARGETS ${executable} RUNTIME DESTINATION bin - COMPONENT ${ASHT_SWIFT_COMPONENT}) + COMPONENT ${ASHT_SWIFT_COMPONENT} + ) + + swift_install_strip_builder_rpath( + TARGETS ${executable} + DESTINATION bin + COMPONENT ${ASHT_SWIFT_COMPONENT} + ) swift_is_installing_component(${ASHT_SWIFT_COMPONENT} is_installing) endif() diff --git a/cmake/modules/AddSwiftUnittests.cmake b/cmake/modules/AddSwiftUnittests.cmake index a4573981c225c..45acb9bf10698 100644 --- a/cmake/modules/AddSwiftUnittests.cmake +++ b/cmake/modules/AddSwiftUnittests.cmake @@ -5,10 +5,32 @@ add_custom_target(SwiftUnitTests) set_target_properties(SwiftUnitTests PROPERTIES FOLDER "Tests") +# Add a new Swift unit test executable. +# +# Usage: +# add_swift_unittest(name +# [IS_TARGET_TEST] +# source1 [source2 source3 ...]) +# +# name +# Name of the test (e.g., SwiftASTTest). +# +# IS_TARGET_TEST +# Indicates this is a test for target libraries. Not host library. +# This avoids linking with toolchains stdlib. +# +# source1 ... +# Sources to add into this executable. function(add_swift_unittest test_dirname) + cmake_parse_arguments(ASU + "IS_TARGET_TEST" + "" + "" + ${ARGN}) + # *NOTE* Even though "add_unittest" does not have llvm in its name, it is a # function defined by AddLLVM.cmake. - add_unittest(SwiftUnitTests ${test_dirname} ${ARGN}) + add_unittest(SwiftUnitTests ${test_dirname} ${ASU_UNPARSED_ARGUMENTS}) set_target_properties(${test_dirname} PROPERTIES LINKER_LANGUAGE CXX) @@ -89,7 +111,8 @@ function(add_swift_unittest test_dirname) endif() endif() - if (SWIFT_SWIFT_PARSER) + if (SWIFT_SWIFT_PARSER AND NOT ASU_IS_TARGET_TEST) + # Link to stdlib the compiler uses. _add_swift_runtime_link_flags(${test_dirname} "../../lib" "") set_property(TARGET ${test_dirname} PROPERTY BUILD_WITH_INSTALL_RPATH OFF) endif() diff --git a/cmake/modules/SwiftImplicitImport.cmake b/cmake/modules/SwiftImplicitImport.cmake index 50c5e5415ff81..3c79357f01883 100644 --- a/cmake/modules/SwiftImplicitImport.cmake +++ b/cmake/modules/SwiftImplicitImport.cmake @@ -6,7 +6,7 @@ function(swift_supports_implicit_module module_name out_var) "${CMAKE_Swift_COMPILER}" -Xfrontend -disable-implicit-${module_name}-module-import -Xfrontend -parse-stdlib - -c - -o /dev/null + -parse - INPUT_FILE "${CMAKE_BINARY_DIR}/tmp/empty-check-${module_name}.swift" OUTPUT_QUIET ERROR_QUIET diff --git a/cmake/modules/SwiftStripBuilderRpath.cmake b/cmake/modules/SwiftStripBuilderRpath.cmake new file mode 100644 index 0000000000000..8df406b0388df --- /dev/null +++ b/cmake/modules/SwiftStripBuilderRpath.cmake @@ -0,0 +1,71 @@ +set(SWIFT_SET_RPATH_SCRIPT_FILE "${CMAKE_CURRENT_LIST_FILE}") + +function(swift_get_set_rpath_script_file out_var) + set(${out_var} "${SWIFT_SET_RPATH_SCRIPT_FILE}" PARENT_SCOPE) +endfunction() + +# Actual RPATH_SET operation to the file. +function(_swift_set_rpath_impl file new_rpath) + # NOTE: RPATH_SET is not documented, and works only for ELF and XCOFF. + file(RPATH_SET FILE "${file}" NEW_RPATH "${new_rpath}") +endfunction() + +# For 'install(SCRIPT