[LinkerWrapper] Extend with usual pass options (llvm#96704)

The goal of this patch is to enable utilizing LLVM plugin passes and
remarks for GPU offload code at link time. Specifically, this patch
extends clang-linker-wrapper's `--offload-opt` (and consequently
`-mllvm`) to accept the various LLVM pass options that tools like opt
usually accept. Those options include `--passes`, `--load-pass-plugin`,
and various remarks options.

Unlike many other LLVM options that are inherited from linked code by
clang-linker-wrapper (e.g., `-pass-remarks` is already implemented in
`llvm/lib/IR/DiagnosticHandler.cpp`), these options are implemented
separately as needed by each tool (e.g., opt, llc). Fortunately, this
patch is able to handle most of the implementation by passing the option
values to `lto::Config`.

For testing plugin support, this patch uses the simple `Bye` plugin from
LLVM core, but that requires several small Clang test suite config
extensions.

Author: jdenny-ornl

clang/test/CMakeLists.txt

@@ -11,6 +11,9 @@ llvm_canonicalize_cmake_booleans(
   CLANG_SPAWN_CC1
   CLANG_ENABLE_CIR
   ENABLE_BACKTRACES
+  LLVM_BUILD_EXAMPLES
+  LLVM_BYE_LINK_INTO_TOOLS
+  LLVM_ENABLE_PLUGINS
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_ZSTD
   LLVM_ENABLE_PER_TARGET_RUNTIME_DIR

clang/test/Driver/linker-wrapper-llvm-help.c

@@ -0,0 +1,10 @@
+// Check that these simple command lines for listing LLVM options are supported,
+// as claimed by 'clang-linker-wrapper --help'.
+
+// RUN: clang-linker-wrapper -mllvm --help 2>&1 | FileCheck %s
+// RUN: clang-linker-wrapper --offload-opt=--help 2>&1 | FileCheck %s
+
+// Look for a few options supported only after -mllvm and --offload-opt.
+//     CHECK: OPTIONS:
+// CHECK-DAG: --passes=<string>
+// CHECK-DAG: --load-pass-plugin=<string>

clang/test/Driver/linker-wrapper-passes.c

@@ -0,0 +1,72 @@
+// Check various clang-linker-wrapper pass options after -offload-opt.
+
+// REQUIRES: llvm-plugins, llvm-examples
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// Setup.
+// RUN: mkdir -p %t
+// RUN: %clang -cc1 -emit-llvm-bc -o %t/host-x86_64-unknown-linux-gnu.bc \
+// RUN:     -disable-O0-optnone -triple=x86_64-unknown-linux-gnu %s
+// RUN: %clang -cc1 -emit-llvm-bc -o %t/openmp-amdgcn-amd-amdhsa.bc \
+// RUN:     -disable-O0-optnone -triple=amdgcn-amd-amdhsa %s
+// RUN: opt %t/openmp-amdgcn-amd-amdhsa.bc -o %t/openmp-amdgcn-amd-amdhsa.bc \
+// RUN:     -passes=forceattrs -force-remove-attribute=f:noinline
+// RUN: clang-offload-packager -o %t/openmp-x86_64-unknown-linux-gnu.out \
+// RUN:     --image=file=%t/openmp-amdgcn-amd-amdhsa.bc,triple=amdgcn-amd-amdhsa
+// RUN: %clang -cc1 -S -o %t/host-x86_64-unknown-linux-gnu.s \
+// RUN:     -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+// RUN:     -fembed-offload-object=%t/openmp-x86_64-unknown-linux-gnu.out \
+// RUN:     %t/host-x86_64-unknown-linux-gnu.bc
+// RUN: %clang -cc1as -o %t/host-x86_64-unknown-linux-gnu.o \
+// RUN:     -triple x86_64-unknown-linux-gnu -filetype obj -target-cpu x86-64 \
+// RUN:     %t/host-x86_64-unknown-linux-gnu.s
+
+// Check plugin, -passes, and no remarks.
+// RUN: clang-linker-wrapper -o a.out --embed-bitcode \
+// RUN:     --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \
+// RUN:     %offload-opt-loadbye --offload-opt=-wave-goodbye \
+// RUN:     --offload-opt=-passes="function(goodbye),module(inline)" 2>&1 | \
+// RUN:   FileCheck -match-full-lines -check-prefixes=OUT %s
+
+// Check plugin, -p, and remarks.
+// RUN: clang-linker-wrapper -o a.out --embed-bitcode \
+// RUN:     --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \
+// RUN:     %offload-opt-loadbye --offload-opt=-wave-goodbye \
+// RUN:     --offload-opt=-p="function(goodbye),module(inline)" \
+// RUN:     --offload-opt=-pass-remarks=inline \
+// RUN:     --offload-opt=-pass-remarks-output=%t/remarks.yml \
+// RUN:     --offload-opt=-pass-remarks-filter=inline \
+// RUN:     --offload-opt=-pass-remarks-format=yaml 2>&1 | \
+// RUN:   FileCheck -match-full-lines -check-prefixes=OUT,REM %s
+// RUN: FileCheck -input-file=%t/remarks.yml -match-full-lines \
+// RUN:     -check-prefixes=YML %s
+
+// Check handling of bad plugin.
+// RUN: not clang-linker-wrapper \
+// RUN:     --offload-opt=-load-pass-plugin=%t/nonexistent.so 2>&1 | \
+// RUN:   FileCheck -match-full-lines -check-prefixes=BAD-PLUGIN %s
+
+//  OUT-NOT: {{.}}
+//      OUT: Bye: f
+// OUT-NEXT: Bye: test
+// REM-NEXT: remark: {{.*}} 'f' inlined into 'test' {{.*}}
+//  OUT-NOT: {{.}}
+
+//  YML-NOT: {{.}}
+//      YML: --- !Passed
+// YML-NEXT: Pass: inline
+// YML-NEXT: Name: Inlined
+// YML-NEXT: Function: test
+// YML-NEXT: Args:
+//      YML:  - Callee: f
+//      YML:  - Caller: test
+//      YML: ...
+//  YML-NOT: {{.}}
+
+// BAD-PLUGIN-NOT: {{.}}
+//     BAD-PLUGIN: {{.*}}Could not load library {{.*}}nonexistent.so{{.*}}
+// BAD-PLUGIN-NOT: {{.}}
+
+void f() {}
+void test() { f(); }

clang/test/lit.cfg.py

@@ -109,6 +109,15 @@
 if config.clang_examples:
     config.available_features.add("examples")
 
+if config.llvm_examples:
+    config.available_features.add("llvm-examples")
+
+if config.llvm_linked_bye_extension:
+    config.substitutions.append(("%offload-opt-loadbye", ""))
+else:
+    loadbye = f"-load-pass-plugin={config.llvm_shlib_dir}/Bye{config.llvm_shlib_ext}"
+    config.substitutions.append(("%offload-opt-loadbye", f"--offload-opt={loadbye}"))
+
 
 def have_host_jit_feature_support(feature_name):
     clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
@@ -213,6 +222,9 @@ def have_host_clang_repl_cuda():
 if config.has_plugins and config.llvm_plugin_ext:
     config.available_features.add("plugins")
 
+if config.llvm_has_plugins and config.llvm_plugin_ext:
+    config.available_features.add("llvm-plugins")
+
 if config.clang_default_pie_on_linux:
     config.available_features.add("default-pie-on-linux")
 

clang/test/lit.site.cfg.py.in

@@ -7,6 +7,7 @@ config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
 config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@"))
 config.llvm_libs_dir = lit_config.substitute(path(r"@LLVM_LIBS_DIR@"))
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
+config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
 config.lit_tools_dir = path(r"@LLVM_LIT_TOOLS_DIR@")
 config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@"
@@ -39,7 +40,10 @@ config.python_executable = "@Python3_EXECUTABLE@"
 config.use_z3_solver = lit_config.params.get('USE_Z3_SOLVER', "@USE_Z3_SOLVER@")
 config.has_plugins = @CLANG_PLUGIN_SUPPORT@
 config.clang_vendor_uti = "@CLANG_VENDOR_UTI@"
+config.llvm_examples = @LLVM_BUILD_EXAMPLES@
+config.llvm_linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.llvm_external_lit = path(r"@LLVM_EXTERNAL_LIT@")
+config.llvm_has_plugins = @LLVM_ENABLE_PLUGINS@
 config.standalone_build = @CLANG_BUILT_STANDALONE@
 config.ppc_linux_default_ieeelongdouble = @PPC_LINUX_DEFAULT_IEEELONGDOUBLE@
 config.have_llvm_driver = @LLVM_TOOL_LLVM_DRIVER_BUILD@

clang/tools/clang-linker-wrapper/CMakeLists.txt

@@ -41,3 +41,5 @@ target_link_libraries(clang-linker-wrapper
   PRIVATE
   ${CLANG_LINKER_WRAPPER_LIB_DEPS}
   )
+
+export_executable_symbols_for_plugins(clang-linker-wrapper)

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

@@ -37,6 +37,8 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Passes/PassPlugin.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileOutputBuffer.h"
@@ -62,6 +64,54 @@ using namespace llvm;
 using namespace llvm::opt;
 using namespace llvm::object;
 
+// Various tools (e.g., llc and opt) duplicate this series of declarations for
+// options related to passes and remarks.
+
+static cl::opt<bool> RemarksWithHotness(
+    "pass-remarks-with-hotness",
+    cl::desc("With PGO, include profile count in optimization remarks"),
+    cl::Hidden);
+
+static cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for "
+                 "an optimization remark to be output. "
+                 "Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("N or 'auto'"), cl::init(0), cl::Hidden);
+
+static cl::opt<std::string>
+    RemarksFilename("pass-remarks-output",
+                    cl::desc("Output filename for pass remarks"),
+                    cl::value_desc("filename"));
+
+static cl::opt<std::string>
+    RemarksPasses("pass-remarks-filter",
+                  cl::desc("Only record optimization remarks from passes whose "
+                           "names match the given regular expression"),
+                  cl::value_desc("regex"));
+
+static cl::opt<std::string> RemarksFormat(
+    "pass-remarks-format",
+    cl::desc("The format used for serializing remarks (default: YAML)"),
+    cl::value_desc("format"), cl::init("yaml"));
+
+static cl::list<std::string>
+    PassPlugins("load-pass-plugin",
+                cl::desc("Load passes from plugin library"));
+
+static cl::opt<std::string> PassPipeline(
+    "passes",
+    cl::desc(
+        "A textual description of the pass pipeline. To have analysis passes "
+        "available before a certain pass, add 'require<foo-analysis>'. "
+        "'-passes' overrides the pass pipeline (but not all effects) from "
+        "specifying '--opt-level=O?' (O2 is the default) to "
+        "clang-linker-wrapper.  Be sure to include the corresponding "
+        "'default<O?>' in '-passes'."));
+static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
+                               cl::desc("Alias for -passes"));
+
 /// Path of the current binary.
 static const char *LinkerExecutable;
 
@@ -628,6 +678,12 @@ std::unique_ptr<lto::LTO> createLTO(
   Conf.CPU = Arch.str();
   Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple);
 
+  Conf.RemarksFilename = RemarksFilename;
+  Conf.RemarksPasses = RemarksPasses;
+  Conf.RemarksWithHotness = RemarksWithHotness;
+  Conf.RemarksHotnessThreshold = RemarksHotnessThreshold;
+  Conf.RemarksFormat = RemarksFormat;
+
   StringRef OptLevel = Args.getLastArgValue(OPT_opt_level, "O2");
   Conf.MAttrs = Features;
   std::optional<CodeGenOptLevel> CGOptLevelOrNone =
@@ -637,6 +693,17 @@ std::unique_ptr<lto::LTO> createLTO(
   Conf.OptLevel = OptLevel[1] - '0';
   Conf.DefaultTriple = Triple.getTriple();
 
+  // TODO: Should we complain about combining --opt-level and -passes, as opt
+  // does?  That might be too limiting in clang-linker-wrapper, so for now we
+  // just warn in the help entry for -passes that the default<O?> corresponding
+  // to --opt-level=O? should be included there.  The problem is that
+  // --opt-level produces effects in clang-linker-wrapper beyond what -passes
+  // appears to be able to achieve, so rejecting the combination of --opt-level
+  // and -passes would apparently make it impossible to combine those effects
+  // with a custom pass pipeline.
+  Conf.OptPipeline = PassPipeline;
+  Conf.PassPlugins = PassPlugins;
+
   LTOError = false;
   Conf.DiagHandler = diagnosticHandler;
 
@@ -1660,6 +1727,13 @@ int main(int Argc, char **Argv) {
     NewArgv.push_back(Arg->getValue());
   for (const opt::Arg *Arg : Args.filtered(OPT_offload_opt_eq_minus))
     NewArgv.push_back(Args.MakeArgString(StringRef("-") + Arg->getValue()));
+  SmallVector<PassPlugin, 1> PluginList;
+  PassPlugins.setCallback([&](const std::string &PluginPath) {
+    auto Plugin = PassPlugin::Load(PluginPath);
+    if (!Plugin)
+      report_fatal_error(Plugin.takeError(), /*gen_crash_diag=*/false);
+    PluginList.emplace_back(Plugin.get());
+  });
   cl::ParseCommandLineOptions(NewArgv.size(), &NewArgv[0]);
 
   Verbose = Args.hasArg(OPT_verbose);

clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td

@@ -94,9 +94,13 @@ def linker_arg_EQ : Joined<["--"], "linker-arg=">,
 
 // Arguments for the LLVM backend.
 def mllvm : Separate<["-"], "mllvm">, Flags<[WrapperOnlyOption]>,
-  MetaVarName<"<arg>">, HelpText<"Arguments passed to the LLVM invocation">;
+  MetaVarName<"<arg>">,
+  HelpText<"Arguments passed to LLVM, including Clang invocations, for which "
+           "the '-mllvm' prefix is preserved. Use '-mllvm --help' for a list "
+           "of options.">;
 def offload_opt_eq_minus : Joined<["--", "-"], "offload-opt=-">, Flags<[HelpHidden, WrapperOnlyOption]>,
-  HelpText<"Options passed to LLVM">;
+  HelpText<"Options passed to LLVM, not including the Clang invocation. Use "
+           "'--offload-opt=--help' for a list of options.">;
 
 // Standard linker flags also used by the linker wrapper.
 def sysroot_EQ : Joined<["--"], "sysroot=">, HelpText<"Set the system root">;