[KernelInfo][LinkerWrapper] Extend with usual pass options

The goal of this patch is to enable utilizing LLVM plugin passes and
remarks for GPU offload code at link time.  Specifically, this patch
extends clang-linker-wrapper's `--offload-opt` (and consequently
`-mllvm`) to accept the various LLVM pass options that tools like opt
usually accept.  Those options include `--passes`,
`--load-pass-plugin`, and various remarks options.

Unlike many other LLVM options that are inherited from linked code by
clang-linker-wrapper (e.g., `-pass-remarks` is already implemented in
`llvm/lib/IR/DiagnosticHandler.cpp`), these options are implemented
separately as needed by each tool (e.g., opt, llc).  Fortuantely, this
patch is able to handle most of the implementation by passing the
option values to `lto::Config`.

For testing plugin support, this patch uses the simple `Bye` plugin
from LLVM core, but that requires several small Clang test suite
config extensions.

We will propose this patch upstream in the future.

Author: jdenny-ornl

clang/test/CMakeLists.txt

@@ -11,6 +11,9 @@ llvm_canonicalize_cmake_booleans(
   CLANG_SPAWN_CC1
   CLANG_ENABLE_CIR
   ENABLE_BACKTRACES
+  LLVM_BUILD_EXAMPLES
+  LLVM_BYE_LINK_INTO_TOOLS
+  LLVM_ENABLE_PLUGINS
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_ZSTD
   LLVM_ENABLE_PER_TARGET_RUNTIME_DIR

clang/test/Driver/linker-wrapper-llvm-help.c

@@ -0,0 +1,10 @@
+// Check that these simple command lines for listing LLVM options are supported,
+// as claimed by 'clang-linker-wrapper --help'.
+
+// RUN: clang-linker-wrapper -mllvm --help 2>&1 | FileCheck %s
+// RUN: clang-linker-wrapper --offload-opt=--help 2>&1 | FileCheck %s
+
+// Look for a few options supported only after -mllvm and --offload-opt.
+//     CHECK: OPTIONS:
+// CHECK-DAG: --passes=<string>
+// CHECK-DAG: --load-pass-plugin=<string>

clang/test/Driver/linker-wrapper-passes.ll

@@ -0,0 +1,86 @@
+; Check various clang-linker-wrapper pass options after -offload-opt.
+
+; REQUIRES: llvm-plugins, llvm-examples
+; REQUIRES: x86-registered-target
+; REQUIRES: amdgpu-registered-target
+
+; Setup.
+; RUN: split-file %s %t
+; RUN: opt -o %t/host-x86_64-unknown-linux-gnu.bc \
+; RUN:     %t/host-x86_64-unknown-linux-gnu.ll
+; RUN: opt -o %t/openmp-amdgcn-amd-amdhsa.bc \
+; RUN:     %t/openmp-amdgcn-amd-amdhsa.ll
+; RUN: clang-offload-packager -o %t/openmp-x86_64-unknown-linux-gnu.out \
+; RUN:     --image=file=%t/openmp-amdgcn-amd-amdhsa.bc,triple=amdgcn-amd-amdhsa
+; RUN: %clang -cc1 -S -o %t/host-x86_64-unknown-linux-gnu.s \
+; RUN:     -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+; RUN:     -fembed-offload-object=%t/openmp-x86_64-unknown-linux-gnu.out \
+; RUN:     %t/host-x86_64-unknown-linux-gnu.bc
+; RUN: %clang -cc1as -o %t/host-x86_64-unknown-linux-gnu.o \
+; RUN:     -triple x86_64-unknown-linux-gnu -filetype obj -target-cpu x86-64 \
+; RUN:     %t/host-x86_64-unknown-linux-gnu.s
+
+; Check plugin, -passes, and no remarks.
+; RUN: clang-linker-wrapper -o a.out --embed-bitcode \
+; RUN:     --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \
+; RUN:     %offload-opt-loadbye --offload-opt=-wave-goodbye \
+; RUN:     --offload-opt=-passes="function(goodbye),module(inline)" 2>&1 | \
+; RUN:   FileCheck -match-full-lines -check-prefixes=OUT %s
+
+; Check plugin, -p, and remarks.
+; RUN: clang-linker-wrapper -o a.out --embed-bitcode \
+; RUN:     --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \
+; RUN:     %offload-opt-loadbye --offload-opt=-wave-goodbye \
+; RUN:     --offload-opt=-p="function(goodbye),module(inline)" \
+; RUN:     --offload-opt=-pass-remarks=inline \
+; RUN:     --offload-opt=-pass-remarks-output=%t/remarks.yml \
+; RUN:     --offload-opt=-pass-remarks-filter=inline \
+; RUN:     --offload-opt=-pass-remarks-format=yaml 2>&1 | \
+; RUN:   FileCheck -match-full-lines -check-prefixes=OUT,REM %s
+; RUN: FileCheck -input-file=%t/remarks.yml -match-full-lines \
+; RUN:     -check-prefixes=YML %s
+
+; Check handling of bad plugin.
+; RUN: not clang-linker-wrapper \
+; RUN:     --offload-opt=-load-pass-plugin=%t/nonexistent.so 2>&1 | \
+; RUN:   FileCheck -match-full-lines -check-prefixes=BAD-PLUGIN %s
+
+;  OUT-NOT: {{.}}
+;      OUT: Bye: f
+; OUT-NEXT: Bye: test
+; REM-NEXT: remark: {{.*}} 'f' inlined into 'test' {{.*}}
+;  OUT-NOT: {{.}}
+
+;  YML-NOT: {{.}}
+;      YML: --- !Passed
+; YML-NEXT: Pass: inline
+; YML-NEXT: Name: Inlined
+; YML-NEXT: Function: test
+; YML-NEXT: Args:
+;      YML:  - Callee: f
+;      YML:  - Caller: test
+;      YML: ...
+;  YML-NOT: {{.}}
+
+; BAD-PLUGIN-NOT: {{.}}
+;     BAD-PLUGIN: {{.*}}Could not load library {{.*}}nonexistent.so{{.*}}
+; BAD-PLUGIN-NOT: {{.}}
+
+;--- host-x86_64-unknown-linux-gnu.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;--- openmp-amdgcn-amd-amdhsa.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define void @f() {
+entry:
+  ret void
+}
+
+define amdgpu_kernel void @test() {
+entry:
+  call void @f()
+  ret void
+}

clang/test/Driver/lit.local.cfg

@@ -19,6 +19,7 @@ config.suffixes = [
     ".hip",
     ".hipi",
     ".hlsl",
+    ".ll",
     ".yaml",
     ".test",
 ]

clang/test/lit.cfg.py

@@ -109,6 +109,15 @@
 if config.clang_examples:
     config.available_features.add("examples")
 
+if config.llvm_examples:
+    config.available_features.add("llvm-examples")
+
+if config.llvm_linked_bye_extension:
+    config.substitutions.append(("%offload-opt-loadbye", ""))
+else:
+    loadbye = f"-load-pass-plugin={config.llvm_shlib_dir}/Bye{config.llvm_shlib_ext}"
+    config.substitutions.append(("%offload-opt-loadbye", f"--offload-opt={loadbye}"))
+
 
 def have_host_jit_feature_support(feature_name):
     clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
@@ -213,6 +222,9 @@ def have_host_clang_repl_cuda():
 if config.has_plugins and config.llvm_plugin_ext:
     config.available_features.add("plugins")
 
+if config.llvm_has_plugins and config.llvm_plugin_ext:
+    config.available_features.add("llvm-plugins")
+
 if config.clang_default_pie_on_linux:
     config.available_features.add("default-pie-on-linux")
 

clang/test/lit.site.cfg.py.in

@@ -7,6 +7,7 @@ config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
 config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@"))
 config.llvm_libs_dir = lit_config.substitute(path(r"@LLVM_LIBS_DIR@"))
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
+config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
 config.lit_tools_dir = path(r"@LLVM_LIT_TOOLS_DIR@")
 config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@"
@@ -39,7 +40,10 @@ config.python_executable = "@Python3_EXECUTABLE@"
 config.use_z3_solver = lit_config.params.get('USE_Z3_SOLVER', "@USE_Z3_SOLVER@")
 config.has_plugins = @CLANG_PLUGIN_SUPPORT@
 config.clang_vendor_uti = "@CLANG_VENDOR_UTI@"
+config.llvm_examples = @LLVM_BUILD_EXAMPLES@
+config.llvm_linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 config.llvm_external_lit = path(r"@LLVM_EXTERNAL_LIT@")
+config.llvm_has_plugins = @LLVM_ENABLE_PLUGINS@
 config.standalone_build = @CLANG_BUILT_STANDALONE@
 config.ppc_linux_default_ieeelongdouble = @PPC_LINUX_DEFAULT_IEEELONGDOUBLE@
 config.have_llvm_driver = @LLVM_TOOL_LLVM_DRIVER_BUILD@

clang/tools/clang-linker-wrapper/CMakeLists.txt

@@ -41,3 +41,5 @@ target_link_libraries(clang-linker-wrapper
   PRIVATE
   ${CLANG_LINKER_WRAPPER_LIB_DEPS}
   )
+
+export_executable_symbols_for_plugins(clang-linker-wrapper)

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

@@ -36,6 +36,8 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Passes/PassPlugin.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileOutputBuffer.h"
@@ -60,6 +62,54 @@ using namespace llvm;
 using namespace llvm::opt;
 using namespace llvm::object;
 
+// Various tools (e.g., llc and opt) duplicate this series of declarations for
+// options related to passes and remarks.
+
+static cl::opt<bool> RemarksWithHotness(
+    "pass-remarks-with-hotness",
+    cl::desc("With PGO, include profile count in optimization remarks"),
+    cl::Hidden);
+
+static cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for "
+                 "an optimization remark to be output. "
+                 "Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("N or 'auto'"), cl::init(0), cl::Hidden);
+
+static cl::opt<std::string>
+    RemarksFilename("pass-remarks-output",
+                    cl::desc("Output filename for pass remarks"),
+                    cl::value_desc("filename"));
+
+static cl::opt<std::string>
+    RemarksPasses("pass-remarks-filter",
+                  cl::desc("Only record optimization remarks from passes whose "
+                           "names match the given regular expression"),
+                  cl::value_desc("regex"));
+
+static cl::opt<std::string> RemarksFormat(
+    "pass-remarks-format",
+    cl::desc("The format used for serializing remarks (default: YAML)"),
+    cl::value_desc("format"), cl::init("yaml"));
+
+static cl::list<std::string>
+    PassPlugins("load-pass-plugin",
+                cl::desc("Load passes from plugin library"));
+
+static cl::opt<std::string> PassPipeline(
+    "passes",
+    cl::desc(
+        "A textual description of the pass pipeline. To have analysis passes "
+        "available before a certain pass, add 'require<foo-analysis>'. "
+        "'-passes' overrides the pass pipeline (but not all effects) from "
+        "specifying '--opt-level=O?' (O2 is the default) to "
+        "clang-linker-wrapper.  Be sure to include the corresponding "
+        "'default<O?>' in '-passes'."));
+static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline),
+                               cl::desc("Alias for -passes"));
+
 /// Path of the current binary.
 static const char *LinkerExecutable;
 
@@ -631,6 +681,12 @@ std::unique_ptr<lto::LTO> createLTO(
   Conf.CPU = Arch.str();
   Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple);
 
+  Conf.RemarksFilename = RemarksFilename;
+  Conf.RemarksPasses = RemarksPasses;
+  Conf.RemarksWithHotness = RemarksWithHotness;
+  Conf.RemarksHotnessThreshold = RemarksHotnessThreshold;
+  Conf.RemarksFormat = RemarksFormat;
+
   StringRef OptLevel = Args.getLastArgValue(OPT_opt_level, "O2");
   Conf.MAttrs = Features;
   std::optional<CodeGenOptLevel> CGOptLevelOrNone =
@@ -640,6 +696,17 @@ std::unique_ptr<lto::LTO> createLTO(
   Conf.OptLevel = OptLevel[1] - '0';
   Conf.DefaultTriple = Triple.getTriple();
 
+  // TODO: Should we complain about combining --opt-level and -passes, as opt
+  // does?  That might be too limiting in clang-linker-wrapper, so for now we
+  // just warn in the help entry for -passes that the default<O?> corresponding
+  // to --opt-level=O? should be included there.  The problem is that
+  // --opt-level produces effects in clang-linker-wrapper beyond what -passes
+  // appears to be able to achieve, so rejecting the combination of --opt-level
+  // and -passes would apparently make it impossible to combine those effects
+  // with a custom pass pipeline.
+  Conf.OptPipeline = PassPipeline;
+  Conf.PassPlugins = PassPlugins;
+
   LTOError = false;
   Conf.DiagHandler = diagnosticHandler;
 
@@ -1669,6 +1736,13 @@ int main(int Argc, char **Argv) {
     NewArgv.push_back(Arg->getValue());
   for (const opt::Arg *Arg : Args.filtered(OPT_offload_opt_eq_minus))
     NewArgv.push_back(Args.MakeArgString(StringRef("-") + Arg->getValue()));
+  SmallVector<PassPlugin, 1> PluginList;
+  PassPlugins.setCallback([&](const std::string &PluginPath) {
+    auto Plugin = PassPlugin::Load(PluginPath);
+    if (!Plugin)
+      report_fatal_error(Plugin.takeError(), /*gen_crash_diag=*/false);
+    PluginList.emplace_back(Plugin.get());
+  });
   cl::ParseCommandLineOptions(NewArgv.size(), &NewArgv[0]);
 
   Verbose = Args.hasArg(OPT_verbose);

clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td

@@ -94,9 +94,13 @@ def linker_arg_EQ : Joined<["--"], "linker-arg=">,
 
 // Arguments for the LLVM backend.
 def mllvm : Separate<["-"], "mllvm">, Flags<[WrapperOnlyOption]>,
-  MetaVarName<"<arg>">, HelpText<"Arguments passed to the LLVM invocation">;
+  MetaVarName<"<arg>">,
+  HelpText<"Arguments passed to LLVM, including Clang invocations, for which "
+           "the '-mllvm' prefix is preserved. Use '-mllvm --help' for a list "
+           "of options.">;
 def offload_opt_eq_minus : Joined<["--", "-"], "offload-opt=-">, Flags<[HelpHidden, WrapperOnlyOption]>,
-  HelpText<"Options passed to LLVM">;
+  HelpText<"Options passed to LLVM, not including the Clang invocation. Use "
+           "'--offload-opt=--help' for a list of options.">;
 
 // Standard linker flags also used by the linker wrapper.
 def sysroot_EQ : Joined<["--"], "sysroot=">, HelpText<"Set the system root">;