diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index dbe53ceec..248cb2c42 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -6970,6 +6970,7 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)
         return;
     }
     if (g_scratch_buffer == nullptr) {
+        ggml_cuda_set_device(g_main_device);
         CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));
     }