[Driver] Save PTX files for SYCL kernels in the user input directory. (#12422)

Save `PTX` files generated (for `SYCL` kernels) during PTX target
processing for CUDA backend using the `-fsycl-dump-device-code` option.

Example usage:
`clang++ -fsycl -fsycl-targets=nvptx64-nvidia-cuda
-fsycl-dmp-device-code=/path/to/ptx syclfile.cpp`

The `PTX` files (`.s` files for each kernel) will be saved under
`/path/to/ptx`

Author: srividya-sundaram

clang/lib/Driver/Compilation.cpp

@@ -185,17 +185,22 @@ bool Compilation::CleanupFile(const char *File, bool IssueErrors) const {
   // able to remove), or non-regular files. Underlying tools may have
   // intentionally not overwritten them.
 
-  // Save the device code files(spv files) only if -fsycl-dump-device-code
-  // option is enabled.
+  // Save the device code files if -fsycl-dump-device-code option is enabled.
   if (TheDriver.isDumpDeviceCodeEnabled()) {
     Arg *DumpDeviceCodeArg =
         getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ);
     std::string ExpectedDir =
         DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : "";
     std::string ActualFile(File);
-    if (ActualFile.find(ExpectedDir) != std::string::npos &&
-        llvm::sys::path::extension(ActualFile).equals(".spv"))
-      return false;
+
+    if (ActualFile.find(ExpectedDir) != std::string::npos) {
+      // Save PTX files generated by LLVM NVPTX Back-End,
+      // when the nvptx*-nvidia-cuda is passed to -fsycl-targets.
+      if (DefaultToolChain.getTriple().isNVPTX())
+        return false;
+      if (llvm::sys::path::extension(ActualFile).equals(".spv"))
+        return false;
+    }
   }
 
   if (!llvm::sys::fs::can_write(File) || !llvm::sys::fs::is_regular_file(File))

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -111,6 +111,26 @@ void SYCL::constructLLVMForeachCommand(Compilation &C, const JobAction &JA,
         C.getArgs().MakeArgString("--out-dir=" + OutputDirName));
   }
 
+  // If fsycl-dump-device-code is passed, put the PTX files
+  // into the path provided in fsycl-dump-device-code.
+  if (T->getToolChain().getTriple().isNVPTX() &&
+      C.getDriver().isDumpDeviceCodeEnabled() && Ext.equals("s")) {
+    SmallString<128> OutputDir;
+
+    Arg *DumpDeviceCodeArg =
+        C.getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ);
+
+    OutputDir = (DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : "");
+
+    // If the output directory path is empty, put the PTX files in the
+    // current directory.
+    if (OutputDir.empty())
+      llvm::sys::path::native(OutputDir = "./");
+    else
+      OutputDir.append(llvm::sys::path::get_separator());
+    ForeachArgs.push_back(C.getArgs().MakeArgString("--out-dir=" + OutputDir));
+  }
+
   ForeachArgs.push_back(C.getArgs().MakeArgString("--"));
   ForeachArgs.push_back(
       C.getArgs().MakeArgString(InputCommand->getExecutable()));

clang/test/Driver/save-ptx-files.cpp

@@ -0,0 +1,43 @@
+// Save PTX files during PTX target processing using -fsycl-dump-device-code option.
+
+// Verify that -fsycl-dump-device-code saves PTX files in the user provided directory
+// while targeting CUDA enabled GPUs.
+
+// Linux
+// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES,CHECK-SPIRV-FILES
+
+// clang --driver-mode=g++
+// RUN: %clangxx -### -fsycl  -fsycl-targets=nvptx64-nvidia-cuda -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES
+
+// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES-CWD,CHECK-SPIRV-FILES-CWD
+
+// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}}
+// CHECK-SPIRV-FILES: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"
+// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device"
+// CHECK-SPIRV-FILES-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"
+
+// Windows - Check if PTX files are saved in the user provided path.
+// RUN: %clang_cl -### -fsycl \
+// RUN: -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN: -fsycl-dump-device-code=/user/input/path %s 2>&1 \
+// RUN: | FileCheck -check-prefix=CHECK-PTX-WIN %s
+
+// Windows - Check if PTX and SPV files are saved in user provided path.
+// RUN: %clang_cl -### -fsycl \
+// RUN: -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN: -fsycl-dump-device-code=/user/input/path %s 2>&1 \
+// RUN: | FileCheck -check-prefixes=CHECK-PTX-WIN,CHECK-SPV-WIN %s
+
+// Windows - Check PTX files saved in current working directory when -fsycl-dump-device-code
+// is empty. 
+// RUN: %clang_cl -### -fsycl \
+// RUN: -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN: -fsycl-dump-device-code= %s 2>&1 \
+// RUN: | FileCheck -check-prefix=CHECK-PTX-WIN-CWD %s
+
+// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}}
+// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}}
+// CHECK-SPV-WIN:  llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"