Skip to content

[SYCL][L0] allow use of sub-devices with their root-device context #6538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 10, 2022
Merged

[SYCL][L0] allow use of sub-devices with their root-device context #6538

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
620fdbe
[SYCL[L0] allow use of sub-devices with their root-device context
smaslov-intel Aug 4, 2022
f35220a
clang-format
smaslov-intel Aug 8, 2022
424cda5
fix sub-sub-device index
smaslov-intel Aug 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 50 additions & 22 deletions sycl/plugins/level_zero/pi_level_zero.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,50 @@ pi_device _pi_context::getRootDevice() const {
}

pi_result _pi_context::initialize() {

// Helper lambda to create various USM allocators for a device.
auto createUSMAllocators = [this](pi_device Device) {
SharedMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMSharedMemoryAlloc(this, Device))));
SharedReadOnlyMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMSharedReadOnlyMemoryAlloc(this, Device))));
DeviceMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMDeviceMemoryAlloc(this, Device))));
};

// Recursive helper to call createUSMAllocators for all sub-devices
std::function<void(pi_device)> createUSMAllocatorsRecursive;
createUSMAllocatorsRecursive =
[this, createUSMAllocators,
&createUSMAllocatorsRecursive](pi_device Device) -> void {
createUSMAllocators(Device);
for (auto &SubDevice : Device->SubDevices)
createUSMAllocatorsRecursive(SubDevice);
};

// Create USM allocator context for each pair (device, context).
//
for (auto &Device : Devices) {
createUSMAllocatorsRecursive(Device);
}
// Create USM allocator context for host. Device and Shared USM allocations
// are device-specific. Host allocations are not device-dependent therefore
// we don't need a map with device as key.
HostMemAllocContext = std::make_unique<USMAllocContext>(
std::unique_ptr<SystemMemory>(new USMHostMemoryAlloc(this)));

// We may allocate memory to this root device so create allocators.
if (SingleRootDevice && DeviceMemAllocContexts.find(SingleRootDevice) ==
DeviceMemAllocContexts.end()) {
createUSMAllocators(SingleRootDevice);
}

// Create the immediate command list to be used for initializations
// Created as synchronous so level-zero performs implicit synchronization and
// there is no need to query for completion in the plugin
Expand Down Expand Up @@ -3410,11 +3454,7 @@ pi_result piQueueCreate(pi_context Context, pi_device Device,
PI_ASSERT(Context, PI_ERROR_INVALID_CONTEXT);
PI_ASSERT(Queue, PI_ERROR_INVALID_QUEUE);
PI_ASSERT(Device, PI_ERROR_INVALID_DEVICE);

if (std::find(Context->Devices.begin(), Context->Devices.end(), Device) ==
Context->Devices.end()) {
return PI_ERROR_INVALID_DEVICE;
}
PI_ASSERT(Context->isValidDevice(Device), PI_ERROR_INVALID_DEVICE);

// Create placeholder queues in the compute queue group.
// Actual L0 queues will be created at first use.
Expand Down Expand Up @@ -4196,11 +4236,7 @@ pi_result piextMemCreateWithNativeHandle(pi_native_handle NativeHandle,
pi_device Device = nullptr;
if (ZeDevice) {
Device = Context->getPlatform()->getDeviceFromNativeHandle(ZeDevice);
// Check that the device is present in this context.
if (std::find(Context->Devices.begin(), Context->Devices.end(), Device) ==
Context->Devices.end()) {
return PI_ERROR_INVALID_CONTEXT;
}
PI_ASSERT(Context->isValidDevice(Device), PI_ERROR_INVALID_CONTEXT);
}

try {
Expand Down Expand Up @@ -4469,12 +4505,7 @@ pi_result piProgramLink(pi_context Context, pi_uint32 NumDevices,

// Validate input parameters.
PI_ASSERT(DeviceList, PI_ERROR_INVALID_DEVICE);
{
auto DeviceEntry =
find(Context->Devices.begin(), Context->Devices.end(), DeviceList[0]);
if (DeviceEntry == Context->Devices.end())
return PI_ERROR_INVALID_DEVICE;
}
PI_ASSERT(Context->isValidDevice(DeviceList[0]), PI_ERROR_INVALID_DEVICE);
PI_ASSERT(!PFnNotify && !UserData, PI_ERROR_INVALID_VALUE);
if (NumInputPrograms == 0 || InputPrograms == nullptr)
return PI_ERROR_INVALID_VALUE;
Expand Down Expand Up @@ -4679,12 +4710,9 @@ pi_result piProgramBuild(pi_program Program, pi_uint32 NumDevices,
std::scoped_lock Guard(Program->Mutex);
// Check if device belongs to associated context.
PI_ASSERT(Program->Context, PI_ERROR_INVALID_PROGRAM);
{
auto DeviceEntry = find(Program->Context->Devices.begin(),
Program->Context->Devices.end(), DeviceList[0]);
if (DeviceEntry == Program->Context->Devices.end())
return PI_ERROR_INVALID_VALUE;
}
PI_ASSERT(Program->Context->isValidDevice(DeviceList[0]),
PI_ERROR_INVALID_VALUE);

// It is legal to build a program created from either IL or from native
// device code.
if (Program->State != _pi_program::IL &&
Expand Down
46 changes: 11 additions & 35 deletions sycl/plugins/level_zero/pi_level_zero.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,41 +596,6 @@ struct _pi_context : _pi_object {
SingleRootDevice(getRootDevice()), ZeCommandListInit{nullptr} {
// NOTE: one must additionally call initialize() to complete
// PI context creation.

// Create USM allocator context for each pair (device, context).
for (uint32_t I = 0; I < NumDevices; I++) {
pi_device Device = Devs[I];
SharedMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMSharedMemoryAlloc(this, Device))));
SharedReadOnlyMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMSharedReadOnlyMemoryAlloc(this, Device))));
DeviceMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(Device),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMDeviceMemoryAlloc(this, Device))));
}
// Create USM allocator context for host. Device and Shared USM allocations
// are device-specific. Host allocations are not device-dependent therefore
// we don't need a map with device as key.
HostMemAllocContext = std::make_unique<USMAllocContext>(
std::unique_ptr<SystemMemory>(new USMHostMemoryAlloc(this)));

// We may allocate memory to this root device so create allocators.
if (SingleRootDevice && DeviceMemAllocContexts.find(SingleRootDevice) ==
DeviceMemAllocContexts.end()) {
SharedMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(SingleRootDevice),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMSharedMemoryAlloc(this, SingleRootDevice))));
DeviceMemAllocContexts.emplace(
std::piecewise_construct, std::make_tuple(SingleRootDevice),
std::make_tuple(std::unique_ptr<SystemMemory>(
new USMDeviceMemoryAlloc(this, SingleRootDevice))));
}
}

// Initialize the PI context.
Expand All @@ -657,6 +622,17 @@ struct _pi_context : _pi_object {
// Therefore it can be accessed without holding a lock on this _pi_context.
const std::vector<pi_device> Devices;

// Checks if Device is covered by this context.
// For that the Device or its root devices need to be in the context.
bool isValidDevice(pi_device Device) const {
while (Device) {
if (std::find(Devices.begin(), Devices.end(), Device) != Devices.end())
return true;
Device = Device->RootDevice;
}
return false;
}

// If context contains one device or sub-devices of the same device, we want
// to save this device.
// This field is only set at _pi_context creation time, and cannot change.
Expand Down