Expand Kineto profiler support (part 1) (#57333)

Summary:
Pull Request resolved: pytorch/pytorch#57333

Pull Request resolved: #193

Expanding Kineto support to more platforms

Reviewed By: gdankel

Differential Revision: D27873669

fbshipit-source-id: 4a72a589f958440cbfff247751b7f4e1910a10c7

Author: Ilia Cherniavskii

libkineto/CMakeLists.txt

@@ -48,13 +48,13 @@ endif()
 
 # Define file lists
 if (LIBKINETO_NOCUPTI)
-    get_filelist("get_libkineto_cpu_only_srcs()" LIBKINETO_SRCS)
+    get_filelist("get_libkineto_cpu_only_srcs(with_api=False)" LIBKINETO_SRCS)
     message(INFO " CUPTI unavailable or disabled - not building GPU profilers")
 else()
-    get_filelist("get_libkineto_srcs()" LIBKINETO_SRCS)
+    get_filelist("get_libkineto_srcs(with_api=False)" LIBKINETO_SRCS)
 endif()
 get_filelist("get_libkineto_public_headers()" LIBKINETO_PUBLIC_HEADERS)
-set(LIBKINETO_API_SRCS "${LIBKINETO_SOURCE_DIR}/libkineto_api.cpp")
+get_filelist("get_libkineto_api_srcs()" LIBKINETO_API_SRCS)
 
 add_library(kineto_base OBJECT ${LIBKINETO_SRCS})
 add_library(kineto_api OBJECT ${LIBKINETO_API_SRCS})
@@ -69,13 +69,22 @@ set_target_properties(kineto_base kineto_api PROPERTIES
       CXX_EXTENSIONS NO
       CXX_VISIBILITY_PRESET hidden)
 
-target_compile_options(kineto_base PRIVATE "-DKINETO_NAMESPACE=libkineto"
-"-std=gnu++14")
-target_compile_options(kineto_api PRIVATE "-std=gnu++14")
+set(KINETO_COMPILE_OPTIONS "-DKINETO_NAMESPACE=libkineto")
+list(APPEND KINETO_COMPILE_OPTIONS "-DFMT_HEADER_ONLY")
+if(NOT MSVC)
+  list(APPEND KINETO_COMPILE_OPTIONS "-std=c++14")
+else()
+  list(APPEND KINETO_COMPILE_OPTIONS "/std:c++14")
+  list(APPEND KINETO_COMPILE_OPTIONS "-DWIN32_LEAN_AND_MEAN")
+  list(APPEND KINETO_COMPILE_OPTIONS "-DNOGDI")
+endif()
 if (NOT LIBKINETO_NOCUPTI)
-    target_compile_options(kineto_base PRIVATE "-DHAS_CUPTI")
+    list(APPEND KINETO_COMPILE_OPTIONS "-DHAS_CUPTI")
 endif()
 
+target_compile_options(kineto_base PRIVATE "${KINETO_COMPILE_OPTIONS}")
+target_compile_options(kineto_api PRIVATE "${KINETO_COMPILE_OPTIONS}")
+
 if(NOT TARGET fmt)
   if(NOT FMT_SOURCE_DIR)
     set(FMT_SOURCE_DIR "${LIBKINETO_THIRDPARTY_DIR}/fmt"
@@ -95,6 +104,8 @@ if(NOT TARGET fmt)
 endif()
 
 set(FMT_INCLUDE_DIR "${FMT_SOURCE_DIR}/include")
+message(STATUS "Kineto: FMT_SOURCE_DIR = ${FMT_SOURCE_DIR}")
+message(STATUS "Kineto: FMT_INCLUDE_DIR = ${FMT_INCLUDE_DIR}")
 if (NOT CUPTI_INCLUDE_DIR)
     set(CUPTI_INCLUDE_DIR "${CUDA_SOURCE_DIR}/extras/CUPTI/include")
 endif()
@@ -134,9 +145,8 @@ else()
 endif()
 
 target_link_libraries(kineto "${CUDA_cupti_LIBRARY}")
-
-target_link_libraries(kineto $<BUILD_INTERFACE:fmt>)
-add_dependencies(kineto fmt)
+target_link_libraries(kineto $<BUILD_INTERFACE:fmt::fmt-header-only>)
+add_dependencies(kineto fmt::fmt-header-only)
 
 install(TARGETS kineto EXPORT kinetoLibraryConfig
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}

libkineto/include/ActivityProfilerInterface.h

@@ -18,7 +18,7 @@
 namespace libkineto {
 
 class ActivityProfilerController;
-class CpuTraceBuffer;
+struct CpuTraceBuffer;
 class Config;
 
 class ActivityProfilerInterface {
@@ -78,9 +78,9 @@ class ActivityProfilerInterface {
     return true;
   }
 
-  // Maps kernel thread id -> pthread id for CPU ops.
+  // Saves information for the current thread to be used in profiler output
   // Client must record any new kernel thread where the activity has occured.
-  virtual void recordThreadInfo(pid_t tid, pthread_t pthreadId) {}
+  virtual void recordThreadInfo() {}
 
   // Record trace metadata, currently supporting only string key and values,
   // values with the same key are overwritten

libkineto/include/GenericTraceActivity.h

@@ -12,6 +12,7 @@
 #include <thread>
 #include <vector>
 
+#include "ThreadUtil.h"
 #include "TraceActivity.h"
 
 namespace libkineto {
@@ -21,7 +22,7 @@ namespace libkineto {
 struct GenericTraceActivity : TraceActivity {
 
   int64_t deviceId() const override {
-    return cachedPid();
+    return processId();
   }
 
   int64_t resourceId() const override {
@@ -60,15 +61,14 @@ struct GenericTraceActivity : TraceActivity {
     metadata_.push_back(std::move(kv));
   }
 
-   const std::string getMetadata() const {
+  const std::string getMetadata() const {
     return fmt::format("{}", fmt::join(metadata_, ", "));
   }
 
   int64_t startTime{0};
   int64_t endTime{0};
   int64_t correlation{0};
   int device{-1};
-  // TODO: Add OS abstraction
   int32_t sysThreadId{0};
   std::string activityName;
   ActivityType activityType;

libkineto/include/ThreadUtil.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace libkineto {
+
+int32_t systemThreadId();
+int32_t threadId();
+bool setThreadName(const std::string& name);
+std::string getThreadName();
+
+int32_t processId();
+std::string processName(int32_t pid);
+
+struct ProcessInfo {
+  int32_t pid;
+  const std::string name;
+  const std::string label;
+};
+
+struct ThreadInfo {
+  ThreadInfo(int32_t tid, const std::string& name) :
+    tid(tid), name(name) {}
+  int32_t tid;
+  const std::string name;
+};
+
+// Return a list of pids and process names for the current process
+// and its parents.
+std::vector<std::pair<int32_t, std::string>> pidCommandPairsOfAncestors();
+
+} // namespace libkineto

libkineto/include/TraceActivity.h

@@ -8,8 +8,6 @@
 #pragma once
 
 #include <string>
-#include <sys/types.h>
-#include <unistd.h>
 
 #include "ActivityType.h"
 
@@ -37,21 +35,12 @@ struct TraceActivity {
   virtual const TraceActivity* linkedActivity() const = 0;
   // Log activity
   virtual void log(ActivityLogger& logger) const = 0;
-};
-
-namespace {
-  // Caching pid is not safe across forks and clones but we currently
-  // don't support an active profiler in a forked process.
-  static inline pid_t cachedPid() {
-    static pid_t pid = getpid();
-    return pid;
-  }
 
-  static inline int64_t nsToUs(int64_t ns) {
+  static int64_t nsToUs(int64_t ns) {
     // It's important that this conversion is the same everywhere.
     // No rounding!
     return ns / 1000;
   }
-}
+};
 
 } // namespace libkineto

libkineto/include/libkineto.h

@@ -9,7 +9,6 @@
 
 #pragma once
 
-#include <signal.h>
 #include <atomic>
 #include <chrono>
 #include <functional>
@@ -27,6 +26,8 @@
 #include "GenericTraceActivity.h"
 #include "TraceSpan.h"
 
+#include "ThreadUtil.h"
+
 extern "C" {
   void suppressLibkinetoLogMessages();
   bool libkineto_init(bool cpuOnly, bool logOnError);
@@ -98,7 +99,7 @@ class LibkinetoApi {
 
   std::unique_ptr<ActivityProfilerInterface> activityProfiler_{};
   ClientInterface* client_{};
-  pthread_t clientRegisterThread_{0};
+  int32_t clientRegisterThread_{0};
 
   bool isLoaded_{false};
   std::atomic_int netSizeThreshold_{};
@@ -108,4 +109,3 @@ class LibkinetoApi {
 LibkinetoApi& api();
 
 } // namespace libkineto
-

libkineto/include/time_since_epoch.h

@@ -12,7 +12,7 @@
 namespace libkineto {
 
 inline int64_t timeSinceEpoch(
-      const std::chrono::time_point<std::chrono::high_resolution_clock>& t) {
+      const std::chrono::time_point<std::chrono::system_clock>& t) {
     return std::chrono::duration_cast<std::chrono::microseconds>(
                t.time_since_epoch())
         .count();

libkineto/libkineto_defs.bzl

@@ -3,7 +3,13 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-def get_libkineto_srcs():
+def get_libkineto_api_srcs():
+    return [
+        "src/ThreadUtil.cpp",
+        "src/libkineto_api.cpp",
+    ]
+
+def get_libkineto_srcs(with_api = True):
     return [
         "src/AbstractConfig.cpp",
         "src/ActivityProfiler.cpp",
@@ -19,17 +25,14 @@ def get_libkineto_srcs():
         "src/EventProfilerController.cpp",
         "src/GenericTraceActivity.cpp",
         "src/Logger.cpp",
-        "src/ProcessInfo.cpp",
-        "src/ThreadName.cpp",
         "src/WeakSymbols.cpp",
         "src/cupti_strings.cpp",
         "src/init.cpp",
-        "src/libkineto_api.cpp",
         "src/output_csv.cpp",
         "src/output_json.cpp",
-    ]
+    ] + (get_libkineto_api_srcs() if with_api else [])
 
-def get_libkineto_cpu_only_srcs():
+def get_libkineto_cpu_only_srcs(with_api = True):
     return [
         "src/AbstractConfig.cpp",
         "src/ActivityProfiler.cpp",
@@ -41,22 +44,20 @@ def get_libkineto_cpu_only_srcs():
         "src/Demangle.cpp",
         "src/GenericTraceActivity.cpp",
         "src/Logger.cpp",
-        "src/ProcessInfo.cpp",
-        "src/ThreadName.cpp",
         "src/init.cpp",
-        "src/libkineto_api.cpp",
         "src/output_csv.cpp",
         "src/output_json.cpp",
-    ]
+    ] + (get_libkineto_api_srcs() if with_api else [])
 
 def get_libkineto_public_headers():
     return [
         "include/ActivityProfilerInterface.h",
         "include/ActivityType.h",
-        "include/ClientTraceActivity.h",
         "include/ClientInterface.h",
+        "include/GenericTraceActivity.h",
         "include/TraceActivity.h",
         "include/TraceSpan.h",
+        "include/ThreadUtil.h",
         "include/libkineto.h",
         "include/time_since_epoch.h",
     ]

libkineto/src/AbstractConfig.cpp

@@ -7,6 +7,7 @@
 
 #include "AbstractConfig.h"
 
+#include <array>
 #include <fmt/format.h>
 #include <sstream>
 

libkineto/src/ActivityProfiler.cpp

@@ -8,10 +8,7 @@
 #include "ActivityProfiler.h"
 
 #include <fmt/format.h>
-#include <libgen.h>
-#include <sys/types.h>
 #include <time.h>
-#include <unistd.h>
 #include <atomic>
 #include <iomanip>
 #include <string>
@@ -32,6 +29,7 @@
 #include "output_base.h"
 
 #include "Logger.h"
+#include "ThreadUtil.h"
 
 using namespace std::chrono;
 using namespace libkineto;
@@ -617,7 +615,7 @@ const time_point<system_clock> ActivityProfiler::performRunLoopStep(
       // FIXME: Is this a good idea for synced start?
       {
         std::lock_guard<std::mutex> guard(mutex_);
-        profileEndTime_ = time_point<high_resolution_clock>(
+        profileEndTime_ = time_point<system_clock>(
                               microseconds(captureWindowStartTime_)) +
             config_->activitiesOnDemandDuration();
       }
@@ -655,23 +653,6 @@ const time_point<system_clock> ActivityProfiler::performRunLoopStep(
   return new_wakeup_time;
 }
 
-// Extract process name from /proc/pid/cmdline. This does not have
-// the 16 character limit that /proc/pid/status and /prod/pid/comm has.
-const string processName(pid_t pid) {
-  FILE* cmdfile = fopen(fmt::format("/proc/{}/cmdline", pid).c_str(), "r");
-  if (cmdfile != nullptr) {
-    char* command = nullptr;
-    int scanned = fscanf(cmdfile, "%ms", &command);
-    if (scanned > 0 && command) {
-      string ret(basename(command));
-      free(command);
-      return ret;
-    }
-  }
-  VLOG(1) << "Failed to read process name for pid " << pid;
-  return "";
-}
-
 void ActivityProfiler::finalizeTrace(const Config& config, ActivityLogger& logger) {
   LOG(INFO) << "Recorded nets:";
   {
@@ -682,9 +663,9 @@ void ActivityProfiler::finalizeTrace(const Config& config, ActivityLogger& logge
   }
 
   // Process names
-  string process_name = processName(getpid());
+  string process_name = processName(processId());
   if (!process_name.empty()) {
-    pid_t pid = getpid();
+    int32_t pid = processId();
     logger.handleProcessInfo(
         {pid, process_name, "CPU"}, captureWindowStartTime_);
     if (!cpuOnly_) {