Productize VMStructs-based stack walker

.github/workflows/test_workflow.yml

@@ -84,7 +84,7 @@ jobs:
           export LIBC=glibc
           export SANITIZER=${{ matrix.config }}
 
-          ./gradlew -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
+          ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
           EXIT_CODE=$?
 
           if [ $EXIT_CODE -ne 0 ]; then
@@ -174,7 +174,7 @@ jobs:
           }')
           export JAVA_VERSION
 
-          ./gradlew -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
+          ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
           EXIT_CODE=$?
 
           if [ $EXIT_CODE -ne 0 ]; then
@@ -274,7 +274,7 @@ jobs:
           export LIBC=glibc
           export SANITIZER=${{ matrix.config }}
 
-          ./gradlew -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
+          ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
           EXIT_CODE=$?
 
           if [ $EXIT_CODE -ne 0 ]; then

ddprof-lib/gtest/build.gradle

@@ -42,7 +42,7 @@ def buildResourcesTask = tasks.register("buildResources", Exec) {
   description = "Build the resources for the Google Tests"
 
   onlyIf {
-    hasGtest && !project.hasProperty('skip-native') && os().isLinux()
+    hasGtest && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest') && os().isLinux()
   }
 
   def targetDir = project(':ddprof-lib').file('build/test/resources/unresolved-functions')
@@ -58,7 +58,7 @@ def buildSmallLibTask = tasks.register("buildSmallLib", Exec) {
   description = "Build the small-lib shared library for the Google Tests"
 
   onlyIf {
-    hasGtest && !project.hasProperty('skip-native') && os().isLinux()
+    hasGtest && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest') && os().isLinux()
   }
 
   def sourceDir = project(':ddprof-lib').file('src/test/resources/small-lib')
@@ -77,7 +77,7 @@ def buildSmallLibTask = tasks.register("buildSmallLib", Exec) {
 
 def gtestAll = tasks.register("gtest") {
   onlyIf {
-    hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native')
+    hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest')
   }
   group = 'verification'
   description = "Run all Google Tests for all build configurations of the library"
@@ -99,6 +99,7 @@ tasks.whenTaskAdded { task ->
             def gtestCompileTask = tasks.register("compileGtest${config.name.capitalize()}_${testName}", CppCompile) {
               onlyIf {
                 config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native')
+                && !project.hasProperty('skip-gtest')
               }
               group = 'build'
               description = "Compile the Google Test ${testName} for the ${config.name} build of the library"
@@ -156,6 +157,7 @@ tasks.whenTaskAdded { task ->
             def gtestLinkTask = tasks.register("linkGtest${config.name.capitalize()}_${testName}", LinkExecutable) {
               onlyIf {
                 config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native')
+                && !project.hasProperty('skip-gtest')
               }
               group = 'build'
               description = "Link the Google Test for the ${config.name} build of the library"
@@ -180,6 +182,7 @@ tasks.whenTaskAdded { task ->
             def gtestExecuteTask = tasks.register("gtest${config.name.capitalize()}_${testName}", Exec) {
               onlyIf {
                 config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native')
+                && !project.hasProperty('skip-gtest')
               }
               group = 'verification'
               description = "Run the Google Test ${testName} for the ${config.name} build of the library"

ddprof-lib/src/main/cpp/arguments.cpp

@@ -277,21 +277,18 @@ Error Arguments::parse(const char *args) {
 
       CASE("cstack")
       if (value != NULL) {
-        switch (value[0]) {
-        case 'n':
-          _cstack = CSTACK_NO;
-          break;
-        case 'd':
+        if (strcmp(value, "fp") == 0) {
+          _cstack = CSTACK_FP;
+        } else if (strcmp(value, "dwarf") == 0) {
           _cstack = CSTACK_DWARF;
-          break;
-        case 'l':
+        } else if (strcmp(value, "lbr") == 0) {
           _cstack = CSTACK_LBR;
-          break;
-        case 'v':
+        } else if (strcmp(value, "vm") == 0) {
           _cstack = CSTACK_VM;
-          break;
-        default:
-          _cstack = CSTACK_FP;
+        } else if (strcmp(value, "vmx") == 0) {
+          _cstack = CSTACK_VMX;
+        } else {
+          _cstack = CSTACK_NO;
         }
       }
 

ddprof-lib/src/main/cpp/arguments.h

@@ -61,12 +61,13 @@ enum Style {
 };
 
 enum CStack {
-  CSTACK_DEFAULT,
-  CSTACK_NO,
-  CSTACK_FP,
-  CSTACK_DWARF,
-  CSTACK_LBR,
-  CSTACK_VM
+  CSTACK_DEFAULT,  // use perf_event_open stack if available or Frame Pointer links otherwise
+  CSTACK_NO,       // do not collect native frames
+  CSTACK_FP,       // walk stack using Frame Pointer links
+  CSTACK_DWARF,    // use DWARF unwinding info from .eh_frame section
+  CSTACK_LBR,      // Last Branch Record hardware capability
+  CSTACK_VM,       // unwind using HotSpot VMStructs
+  CSTACK_VMX       // same as CSTACK_VM but with intermediate native frames
 };
 
 enum Output { OUTPUT_NONE, OUTPUT_COLLAPSED, OUTPUT_JFR };

ddprof-lib/src/main/cpp/dwarf.h

@@ -55,7 +55,13 @@ const int DW_REG_FP = 29;
 const int DW_REG_SP = 31;
 const int DW_REG_PC = 30;
 const int EMPTY_FRAME_SIZE = 0;
-const int LINKED_FRAME_SIZE = 0;
+
+// aarch64 function prologue looks like this (if frame pointer is used):
+// stp x29, x30, [sp, -16]!   // Save FP (x29) and LR (x30)
+// mov x29, sp                // Set FP to SP
+// ---
+// LINKED_FRAME_SIZE should be 16
+const int LINKED_FRAME_SIZE = 2 * DW_STACK_SLOT;
 
 #else
 

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -751,13 +751,15 @@ void Recording::writeSettings(Buffer *buf, Arguments &args) {
   writeBoolSetting(buf, T_HEAP_LIVE_OBJECT, "enabled", args._record_liveness);
 
   writeBoolSetting(buf, T_ACTIVE_RECORDING, "debugSymbols",
-                   VMStructs::hasDebugSymbols());
+                   VMStructs::libjvm()->hasDebugSymbols());
   writeBoolSetting(buf, T_ACTIVE_RECORDING, "kernelSymbols",
                    Symbols::haveKernelSymbols());
   writeStringSetting(buf, T_ACTIVE_RECORDING, "cpuEngine",
                      Profiler::instance()->cpuEngine()->name());
   writeStringSetting(buf, T_ACTIVE_RECORDING, "wallEngine",
                      Profiler::instance()->wallEngine()->name());
+  writeStringSetting(buf, T_ACTIVE_RECORDING, "cstack", 
+                     Profiler::instance()->cstack());
   flushIfNeeded(buf);
 }
 

ddprof-lib/src/main/cpp/profiler.cpp

@@ -17,7 +17,6 @@
 
 #include "profiler.h"
 #include "asyncSampleMutex.h"
-#include "common.h"
 #include "context.h"
 #include "counters.h"
 #include "ctimer.h"
@@ -289,7 +288,7 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
         PerfEvents::walkKernel(tid, callchain + native_frames,
                                MAX_NATIVE_FRAMES - native_frames, java_ctx);
   }
-  if (_cstack == CSTACK_VM) {
+  if (_cstack >= CSTACK_VM) {
     return 0;
   } else if (_cstack == CSTACK_DWARF) {
     native_frames += StackWalker::walkDwarf(ucontext, callchain + native_frames,
@@ -406,7 +405,7 @@ int Profiler::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames,
   if (in_java && java_ctx->sp != 0) {
     // skip ahead to the Java frames before calling AGCT
     frame.restore((uintptr_t)java_ctx->pc, java_ctx->sp, java_ctx->fp);
-  } else if (state != 0 && vm_thread->lastJavaSP() == 0) {
+  } else if (state != 0 && (vm_thread->anchor() == nullptr || vm_thread->anchor()->lastJavaSP() == 0)) {
     // we haven't found the top Java frame ourselves, and the lastJavaSP wasn't
     // recorded either when not in the Java state, lastJava ucontext will be
     // used by AGCT
@@ -485,14 +484,16 @@ int Profiler::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames,
     }
   } else if (trace.num_frames == ticks_unknown_not_Java &&
              !(_safe_mode & LAST_JAVA_PC)) {
-    uintptr_t &sp = vm_thread->lastJavaSP();
-    uintptr_t &pc = vm_thread->lastJavaPC();
-    if (sp != 0 && pc == 0) {
+    JavaFrameAnchor* anchor = vm_thread->anchor();
+    uintptr_t sp = anchor->lastJavaSP();
+    const void* pc = anchor->lastJavaPC();
+    if (sp != 0 && pc == NULL) {
       // We have the last Java frame anchor, but it is not marked as walkable.
       // Make it walkable here
-      pc = ((uintptr_t *)sp)[-1];
+      pc = ((const void**)sp)[-1];
+      anchor->setLastJavaPC(pc);
 
-      NMethod *m = CodeHeap::findNMethod((const void *)pc);
+      NMethod *m = CodeHeap::findNMethod(pc);
       if (m != NULL) {
         // AGCT fails if the last Java frame is a Runtime Stub with an invalid
         // _frame_complete_offset. In this case we patch _frame_complete_offset
@@ -502,28 +503,29 @@ int Profiler::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames,
           m->setFrameCompleteOffset(0);
         }
         VM::_asyncGetCallTrace(&trace, max_depth, ucontext);
-      } else if (_libs->findLibraryByAddress((const void *)pc) != NULL) {
+      } else if (_libs->findLibraryByAddress(pc) != NULL) {
         VM::_asyncGetCallTrace(&trace, max_depth, ucontext);
       }
 
-      pc = 0;
+      anchor->setLastJavaPC(nullptr);
     }
   } else if (trace.num_frames == ticks_not_walkable_not_Java &&
              !(_safe_mode & LAST_JAVA_PC)) {
-    uintptr_t &sp = vm_thread->lastJavaSP();
-    uintptr_t &pc = vm_thread->lastJavaPC();
-    if (sp != 0 && pc != 0) {
+    JavaFrameAnchor* anchor = vm_thread->anchor();
+    uintptr_t sp = anchor->lastJavaSP();
+    const void* pc = anchor->lastJavaPC();
+    if (sp != 0 && pc != NULL) {
       // Similar to the above: last Java frame is set,
       // but points to a Runtime Stub with an invalid _frame_complete_offset
-      NMethod *m = CodeHeap::findNMethod((const void *)pc);
+      NMethod *m = CodeHeap::findNMethod(pc);
       if (m != NULL && !m->isNMethod() && m->frameSize() > 0 &&
           m->frameCompleteOffset() == -1) {
         m->setFrameCompleteOffset(0);
         VM::_asyncGetCallTrace(&trace, max_depth, ucontext);
       }
     }
   } else if (trace.num_frames == ticks_GC_active && !(_safe_mode & GC_TRACES)) {
-    if (vm_thread->lastJavaSP() == 0) {
+    if (vm_thread->anchor()->lastJavaSP() == 0) {
       // Do not add 'GC_active' for threads with no Java frames, e.g. Compiler
       // threads
       frame.restore(saved_pc, saved_sp, saved_fp);
@@ -682,28 +684,32 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
     ASGCT_CallFrame *native_stop = frames + num_frames;
     num_frames += getNativeTrace(ucontext, native_stop, event_type, tid,
                                  &java_ctx, &truncated);
-    if (_cstack == CSTACK_VM) {
-      num_frames +=
-          StackWalker::walkVM(ucontext, frames + num_frames, _max_stack_depth,
-                              _call_stub_begin, _call_stub_end);
+    if (_cstack == CSTACK_VMX) {
+      num_frames += StackWalker::walkVM(ucontext, frames + num_frames, _max_stack_depth, VM_EXPERT, &truncated);
     } else if (event_type == BCI_CPU || event_type == BCI_WALL) {
-      int java_frames = 0;
-      {
+      if (_cstack == CSTACK_VM) {
+        num_frames += StackWalker::walkVM(ucontext, frames + num_frames, _max_stack_depth, VM_NORMAL, &truncated);
+      } else {
         // Async events
         AsyncSampleMutex mutex(ProfiledThread::current());
+        int java_frames = 0;
         if (mutex.acquired()) {
-          java_frames =
-              getJavaTraceAsync(ucontext, frames + num_frames, _max_stack_depth,
-                                &java_ctx, &truncated);
+          java_frames = getJavaTraceAsync(ucontext, frames + num_frames, _max_stack_depth, &java_ctx, &truncated);
+          if (java_frames > 0 && java_ctx.pc != NULL && VMStructs::hasMethodStructs()) {
+              NMethod* nmethod = CodeHeap::findNMethod(java_ctx.pc);
+              if (nmethod != NULL) {
+                  fillFrameTypes(frames + num_frames, java_frames, nmethod);
+              }
+          }
         }
-      }
-      if (java_frames > 0 && java_ctx.pc != NULL) {
-        NMethod *nmethod = CodeHeap::findNMethod(java_ctx.pc);
-        if (nmethod != NULL) {
-          fillFrameTypes(frames + num_frames, java_frames, nmethod);
+        if (java_frames > 0 && java_ctx.pc != NULL) {
+          NMethod *nmethod = CodeHeap::findNMethod(java_ctx.pc);
+          if (nmethod != NULL) {
+            fillFrameTypes(frames + num_frames, java_frames, nmethod);
+          }
         }
+        num_frames += java_frames;
       }
-      num_frames += java_frames;
     }
 
     if (num_frames == 0) {
@@ -1069,7 +1075,7 @@ Error Profiler::checkJvmCapabilities() {
     }
   }
 
-  if (!VMStructs::hasDebugSymbols() && !VM::isOpenJ9()) {
+  if (!VMStructs::libjvm()->hasDebugSymbols() && !VM::isOpenJ9()) {
     Log::warn("Install JVM debug symbols to improve profile accuracy");
   }
 
@@ -1162,7 +1168,6 @@ Error Profiler::start(Arguments &args, bool reset) {
       return Error(
           "VMStructs stack walking is not supported on this JVM/platform");
     }
-    Log::info("cstack=vm is an experimental option, use with care");
   }
 
   // Kernel symbols are useful only for perf_events without --all-user
@@ -1292,7 +1297,7 @@ Error Profiler::check(Arguments &args) {
       return Error("DWARF unwinding is not supported on this platform");
     } else if (args._cstack == CSTACK_LBR && _cpu_engine != &perf_events) {
       return Error("Branch stack is supported only with PMU events");
-    } else if (args._cstack == CSTACK_VM && !VMStructs::hasStackStructs()) {
+    } else if (_cstack >= CSTACK_VM && !(VMStructs::hasStackStructs() && OS::isLinux())) {
       return Error(
           "VMStructs stack walking is not supported on this JVM/platform");
     }

ddprof-lib/src/main/cpp/profiler.h

@@ -1,17 +1,6 @@
 /*
- * Copyright 2016 Andrei Pangin
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Copyright The async-profiler authors
+ * SPDX-License-Identifier: Apache-2.0
  */
 
 #ifndef _PROFILER_H
@@ -180,6 +169,19 @@ class Profiler {
     return _instance;
   }
 
+  const char* cstack() {
+    switch (_cstack) {
+      case CSTACK_DEFAULT: return "default";
+      case CSTACK_NO: return "no";
+      case CSTACK_FP: return "fp";
+      case CSTACK_DWARF: return "dwarf";
+      case CSTACK_LBR: return "lbr";
+      case CSTACK_VM: return "vm";
+      case CSTACK_VMX: return "vmx";
+      default: return "default";
+    }
+  }
+
   u64 total_samples() { return _total_samples; }
   int max_stack_depth() { return _max_stack_depth; }
   time_t uptime() { return time(NULL) - _start_time; }
@@ -213,7 +215,8 @@ class Profiler {
   Error stop();
   Error flushJfr();
   Error dump(const char *path, const int length);
-  void switchThreadEvents(jvmtiEventMode mode);
+  void logStats();
+    void switchThreadEvents(jvmtiEventMode mode);
   int convertNativeTrace(int native_frames, const void **callchain,
                          ASGCT_CallFrame *frames);
   void recordSample(void *ucontext, u64 weight, int tid, jint event_type,