CUDA: fix scratch malloced on non-main device (ggml-org#3220)

JohannesGaessler · pkrmf · commit 3488793b1570 · 2023-09-26T17:19:23.000-05:00
diff --git a/ggml-cuda.cu b/ggml-cuda.cu
@@ -6970,6 +6970,7 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)
         return;
     }
     if (g_scratch_buffer == nullptr) {
+        ggml_cuda_set_device(g_main_device);
         CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));
     }
 

Original file line number	Diff line number	Diff line change
`@@ -6970,6 +6970,7 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)`
`6970`	`6970`	`return;`
`6971`	`6971`	`}`
`6972`	`6972`	`if (g_scratch_buffer == nullptr) {`
	`6973`	`+ ggml_cuda_set_device(g_main_device);`
`6973`	`6974`	`CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));`
`6974`	`6975`	`}`
`6975`	`6976`