Merge pull request #2431 from igchor/ze_trace_tests

[Tests] replace zeCallsMap with zelTracer

Author: pbalcer

source/adapters/level_zero/common.cpp

@@ -86,12 +86,7 @@ bool setEnvVar(const char *name, const char *value) {
 
 ZeUSMImportExtension ZeUSMImport;
 
-// This will count the calls to Level-Zero
-// TODO: remove the ifdef once
-// https://github.com/oneapi-src/unified-runtime/issues/1454 is implemented
-#ifndef UR_L0_CALL_COUNT_IN_TESTS
 std::map<std::string, int> *ZeCallCount = nullptr;
-#endif
 
 inline void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
   switch (ZeError) {

test/adapters/level_zero/CMakeLists.txt

@@ -43,35 +43,27 @@ if(UR_BUILD_ADAPTER_L0)
     endif()
 
     if(NOT WIN32 AND NOT UR_STATIC_ADAPTER_L0)
-        # Make L0 use CallMap from a seprate shared lib so that we can access the map
-        # from the tests. This only seems to work on linux
-        add_library(zeCallMap SHARED zeCallMap.cpp)
-        install_ur_library(zeCallMap)
-        target_compile_definitions(ur_adapter_level_zero PRIVATE UR_L0_CALL_COUNT_IN_TESTS)
-        # TODO: stop exporting internals like this for tests...
-        target_link_libraries(ur_adapter_level_zero PRIVATE zeCallMap)
-
         add_adapter_test(level_zero_ze_calls
             FIXTURE DEVICES
             SOURCES
                 event_cache_tests.cpp
             ENVIRONMENT
                 "UR_ADAPTERS_FORCE_LOAD=\"$<TARGET_FILE:ur_adapter_level_zero>\""
-                "UR_L0_LEAKS_DEBUG=1"
+                "ZE_ENABLE_TRACING_LAYER=1"
         )
 
-        target_link_libraries(test-adapter-level_zero_ze_calls PRIVATE zeCallMap)
+        target_link_libraries(test-adapter-level_zero_ze_calls PRIVATE LevelZeroLoader LevelZeroLoader-Headers)
 
         add_adapter_test(level_zero_multi_queue
             FIXTURE DEVICES
             SOURCES
                 multi_device_event_cache_tests.cpp
             ENVIRONMENT
                 "UR_ADAPTERS_FORCE_LOAD=\"$<TARGET_FILE:ur_adapter_level_zero>\""
-                "UR_L0_LEAKS_DEBUG=1"
+                "ZE_ENABLE_TRACING_LAYER=1"
         )
 
-        target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE zeCallMap)
+        target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE LevelZeroLoader LevelZeroLoader-Headers)
     endif()
 
     add_adapter_test(level_zero_ipc

test/adapters/level_zero/event_cache_tests.cpp

@@ -10,12 +10,32 @@
 #include <map>
 #include <string>
 
+#include "ze_tracer_common.hpp"
+
+std::size_t eventCreateCount = 0;
+std::size_t eventDestroyCount = 0;
+
+void OnEnterEventCreate(ze_event_create_params_t *, ze_result_t, void *,
+                        void **) {
+    eventCreateCount++;
+}
+
+void OnEnterEventDestroy(ze_event_destroy_params_t *, ze_result_t, void *,
+                         void **) {
+    eventDestroyCount++;
+}
+
+static std::shared_ptr<_zel_tracer_handle_t> tracer = [] {
+    zel_core_callbacks_t prologue_callbacks{};
+    prologue_callbacks.Event.pfnCreateCb = OnEnterEventCreate;
+    prologue_callbacks.Event.pfnDestroyCb = OnEnterEventDestroy;
+    return enableTracing(prologue_callbacks, {});
+}();
+
 template <typename... Args> auto combineFlags(std::tuple<Args...> tuple) {
     return std::apply([](auto... args) { return (... |= args); }, tuple);
 }
 
-extern std::map<std::string, int> *ZeCallCount;
-
 using FlagsTupleType = std::tuple<ur_queue_flags_t, ur_queue_flags_t,
                                   ur_queue_flags_t, ur_queue_flags_t>;
 
@@ -43,8 +63,8 @@ struct urEventCacheTest : uur::urContextTestWithParam<FlagsTupleType> {
         ASSERT_SUCCESS(urMemBufferCreate(context, UR_MEM_FLAG_WRITE_ONLY, size,
                                          nullptr, &buffer));
 
-        (*ZeCallCount)["zeEventCreate"] = 0;
-        (*ZeCallCount)["zeEventDestroy"] = 0;
+        eventCreateCount = 0;
+        eventDestroyCount = 0;
     }
 
     void TearDown() override {
@@ -96,9 +116,9 @@ TEST_P(urEventCacheTest, eventsReuseNoVisibleEvent) {
     // TODO: why events are not reused for UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE?
     if ((flags & UR_QUEUE_FLAG_DISCARD_EVENTS) &&
         !(flags & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE)) {
-        ASSERT_EQ((*ZeCallCount)["zeEventCreate"], 2);
+        ASSERT_EQ(eventCreateCount, 2);
     } else {
-        ASSERT_GE((*ZeCallCount)["zeEventCreate"], numIters * numEnqueues);
+        ASSERT_GE(eventCreateCount, numIters * numEnqueues);
     }
 }
 
@@ -115,7 +135,7 @@ TEST_P(urEventCacheTest, eventsReuseWithVisibleEvent) {
         verifyData();
     }
 
-    ASSERT_LT((*ZeCallCount)["zeEventCreate"], numIters * numEnqueues);
+    ASSERT_LT(eventCreateCount, numIters * numEnqueues);
 }
 
 TEST_P(urEventCacheTest, eventsReuseWithVisibleEventAndWait) {
@@ -139,9 +159,9 @@ TEST_P(urEventCacheTest, eventsReuseWithVisibleEventAndWait) {
         UUR_ASSERT_SUCCESS_OR_EXIT_IF_UNSUPPORTED(urQueueFinish(queue));
     }
 
-    ASSERT_GE((*ZeCallCount)["zeEventCreate"], waitEveryN);
+    ASSERT_GE(eventCreateCount, waitEveryN);
     // TODO: why there are more events than this?
-    // ASSERT_LE((*ZeCallCount)["zeEventCreate"],  waitEveryN * 2 + 2);
+    // ASSERT_LE(eventCreateCount,  waitEveryN * 2 + 2);
 }
 
 template <typename T>

test/adapters/level_zero/multi_device_event_cache_tests.cpp

@@ -10,7 +10,21 @@
 #include <map>
 #include <string>
 
-extern std::map<std::string, int> *ZeCallCount;
+#include "ze_tracer_common.hpp"
+
+size_t zeCommandListAppendWaitOnEventsCount = 0;
+
+void OnAppendWaitOnEventsCb(ze_command_list_append_wait_on_events_params_t *,
+                            ze_result_t, void *, void **) {
+    zeCommandListAppendWaitOnEventsCount++;
+}
+
+static std::shared_ptr<_zel_tracer_handle_t> tracer = [] {
+    zel_core_callbacks_t prologue_callbacks{};
+    prologue_callbacks.CommandList.pfnAppendWaitOnEventsCb =
+        OnAppendWaitOnEventsCb;
+    return enableTracing(prologue_callbacks, {});
+}();
 
 using urMultiQueueMultiDeviceEventCacheTest = uur::urAllDevicesTest;
 TEST_F(urMultiQueueMultiDeviceEventCacheTest,
@@ -40,12 +54,12 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
     ASSERT_SUCCESS(urDevicePartition(devices[0], &properties, numSubDevices,
                                      sub_devices.data(), nullptr));
     uur::raii::Context context1 = nullptr;
-    ASSERT_SUCCESS(
-        urContextCreate(1, &sub_devices[0], nullptr, context1.ptr()));
+    ASSERT_SUCCESS(urContextCreate(sub_devices.size(), &sub_devices[0], nullptr,
+                                   context1.ptr()));
     ASSERT_NE(nullptr, context1);
     uur::raii::Context context2 = nullptr;
-    ASSERT_SUCCESS(
-        urContextCreate(1, &sub_devices[1], nullptr, context2.ptr()));
+    ASSERT_SUCCESS(urContextCreate(sub_devices.size(), &sub_devices[0], nullptr,
+                                   context2.ptr()));
     ASSERT_NE(nullptr, context2);
     ur_queue_handle_t queue1 = nullptr;
     ASSERT_SUCCESS(urQueueCreate(context1, sub_devices[0], 0, &queue1));
@@ -54,16 +68,14 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
     uur::raii::Event event = nullptr;
     uur::raii::Event eventWait = nullptr;
     uur::raii::Event eventWaitDummy = nullptr;
-    (*ZeCallCount)["zeCommandListAppendWaitOnEvents"] = 0;
-    EXPECT_SUCCESS(
-        urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr()));
+    zeCommandListAppendWaitOnEventsCount = 0;
     EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr,
                                                  eventWaitDummy.ptr()));
     EXPECT_SUCCESS(
         urEnqueueEventsWait(queue1, 1, eventWaitDummy.ptr(), eventWait.ptr()));
     EXPECT_SUCCESS(
         urEnqueueEventsWait(queue2, 1, eventWait.ptr(), event.ptr()));
-    EXPECT_EQ((*ZeCallCount)["zeCommandListAppendWaitOnEvents"], 2);
+    EXPECT_EQ(zeCommandListAppendWaitOnEventsCount, 2);
     ASSERT_SUCCESS(urEventRelease(eventWaitDummy.get()));
     ASSERT_SUCCESS(urEventRelease(eventWait.get()));
     ASSERT_SUCCESS(urEventRelease(event.get()));
@@ -77,10 +89,12 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
         GTEST_SKIP();
     }
     uur::raii::Context context1 = nullptr;
-    ASSERT_SUCCESS(urContextCreate(1, &devices[0], nullptr, context1.ptr()));
+    ASSERT_SUCCESS(
+        urContextCreate(devices.size(), &devices[0], nullptr, context1.ptr()));
     ASSERT_NE(nullptr, context1);
     uur::raii::Context context2 = nullptr;
-    ASSERT_SUCCESS(urContextCreate(1, &devices[1], nullptr, context2.ptr()));
+    ASSERT_SUCCESS(
+        urContextCreate(devices.size(), &devices[0], nullptr, context2.ptr()));
     ASSERT_NE(nullptr, context2);
     ur_queue_handle_t queue1 = nullptr;
     ASSERT_SUCCESS(urQueueCreate(context1, devices[0], 0, &queue1));
@@ -89,16 +103,14 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
     uur::raii::Event event = nullptr;
     uur::raii::Event eventWait = nullptr;
     uur::raii::Event eventWaitDummy = nullptr;
-    (*ZeCallCount)["zeCommandListAppendWaitOnEvents"] = 0;
-    EXPECT_SUCCESS(
-        urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr()));
+    zeCommandListAppendWaitOnEventsCount = 0;
     EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr,
                                                  eventWaitDummy.ptr()));
     EXPECT_SUCCESS(
         urEnqueueEventsWait(queue1, 1, eventWaitDummy.ptr(), eventWait.ptr()));
     EXPECT_SUCCESS(
         urEnqueueEventsWait(queue2, 1, eventWait.ptr(), event.ptr()));
-    EXPECT_EQ((*ZeCallCount)["zeCommandListAppendWaitOnEvents"], 3);
+    EXPECT_EQ(zeCommandListAppendWaitOnEventsCount, 3);
     ASSERT_SUCCESS(urEventRelease(eventWaitDummy.get()));
     ASSERT_SUCCESS(urEventRelease(eventWait.get()));
     ASSERT_SUCCESS(urEventRelease(event.get()));

test/adapters/level_zero/zeCallMap.cpp

@@ -0,0 +1,31 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "uur/fixtures.h"
+
+#include <level_zero/layers/zel_tracing_api.h>
+#include <loader/ze_loader.h>
+
+#include <memory>
+
+std::shared_ptr<_zel_tracer_handle_t>
+enableTracing(zel_core_callbacks_t prologueCallbacks,
+              zel_core_callbacks_t epilogueCallbacks) {
+    EXPECT_EQ(zeInit(ZE_INIT_FLAG_GPU_ONLY), ZE_RESULT_SUCCESS);
+
+    zel_tracer_desc_t tracer_desc = {ZEL_STRUCTURE_TYPE_TRACER_EXP_DESC,
+                                     nullptr, nullptr};
+    zel_tracer_handle_t tracer = nullptr;
+    EXPECT_EQ(zelTracerCreate(&tracer_desc, &tracer), ZE_RESULT_SUCCESS);
+
+    EXPECT_EQ(zelTracerSetPrologues(tracer, &prologueCallbacks),
+              ZE_RESULT_SUCCESS);
+    EXPECT_EQ(zelTracerSetEpilogues(tracer, &epilogueCallbacks),
+              ZE_RESULT_SUCCESS);
+    EXPECT_EQ(zelTracerSetEnabled(tracer, true), ZE_RESULT_SUCCESS);
+
+    return std::shared_ptr<_zel_tracer_handle_t>(
+        tracer, [](zel_tracer_handle_t tracer) { zelTracerDestroy(tracer); });
+}