Skip to content

Commit 42c5d3c

Browse files
CUDA: enable peer access between devices
1 parent 0728c5a commit 42c5d3c

File tree

1 file changed

+22
-0
lines changed

1 file changed

+22
-0
lines changed

ggml-cuda.cu

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3939,6 +3939,28 @@ void ggml_init_cublas() {
39393939
CUBLAS_CHECK(cublasSetMathMode(g_cublas_handles[id], CUBLAS_TF32_TENSOR_OP_MATH));
39403940
}
39413941

3942+
#ifdef NDEBUG
3943+
for (int id = 0; id < g_device_count; ++id) {
3944+
CUDA_CHECK(cudaSetDevice(id));
3945+
3946+
for (int id_other = 0; id_other < g_device_count; ++id_other) {
3947+
if (id == id_other) {
3948+
continue;
3949+
}
3950+
if (id != g_main_device && id_other != g_main_device) {
3951+
continue;
3952+
}
3953+
3954+
int canAccessPeer;
3955+
CUDA_CHECK(cudaDeviceCanAccessPeer(&canAccessPeer, id, id_other));
3956+
if (canAccessPeer) {
3957+
// FIXME for some reason enabling peer access makes prompt processing slightly slower
3958+
CUDA_CHECK(cudaDeviceEnablePeerAccess(id_other, 0));
3959+
}
3960+
}
3961+
}
3962+
#endif // NDEBUG
3963+
39423964
// configure logging to stdout
39433965
// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
39443966

0 commit comments

Comments
 (0)