diff --git a/paddle/fluid/operators/cholesky_op.cu b/paddle/fluid/operators/cholesky_op.cu
index dabb2d2567eadb17e6c4be62af5ba80e73dbeaa6..c44299686516e968692fe146a5c324c7f1fa83d2 100644
--- a/paddle/fluid/operators/cholesky_op.cu
+++ b/paddle/fluid/operators/cholesky_op.cu
@@ -67,7 +67,7 @@ class CholeskyGPUKernel : public framework::OpKernel<T> {
     auto info = memory::Alloc(dev_ctx, sizeof(int) * batch_count);
     auto* info_ptr = reinterpret_cast<int*>(info->ptr());
 
-#if CUDA_VERSION >= 9020
+#if CUDA_VERSION >= 9020 && !defined(_WIN32)
     if (batch_count > 1) {
       std::vector<T*> output_ptrs;
       for (int i = 0; i < batch_count; i++) {
@@ -93,7 +93,7 @@ class CholeskyGPUKernel : public framework::OpKernel<T> {
         Potrf(dev_ctx, uplo, m, out_data + i * m * m, m, info_ptr + i);
       }
 
-#if CUDA_VERSION >= 9020
+#if CUDA_VERSION >= 9020 && !defined(_WIN32)
     }
 #endif
   }
@@ -126,7 +126,7 @@ class CholeskyGPUKernel : public framework::OpKernel<T> {
 
 FUNC_WITH_TYPES(POTRF_INSTANCE);
 
-#if CUDA_VERSION >= 9020
+#if CUDA_VERSION >= 9020 && !defined(_WIN32)
 #define POTRF_BATCH_INSTANCE(T, C)                                          \
   template <>                                                               \
   void CholeskyGPUKernel<T>::PotrfBatched(                                  \
diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc
index e2e45dc7e8fb227bd852e5f539eb96b93fa75f9d..dd62265e333c9832eb6b1f7bed7cffe726a17c70 100644
--- a/paddle/fluid/platform/dynload/dynamic_loader.cc
+++ b/paddle/fluid/platform/dynload/dynamic_loader.cc
@@ -230,10 +230,7 @@ void* GetCusolverDsoHandle() {
 #if defined(__APPLE__) || defined(__OSX__)
   return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.dylib");
 #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA)
-  // TODO(guosheng): Fix cusolver support on windows.
-  // return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib);
-  PADDLE_THROW(platform::errors::Unavailable(
-      "Cusolver loader cannot support Windows temporarily."));
+  return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib);
 #else
   return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.so");
 #endif