diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake
index 170a8667e0b7c3cd752de5cf637a8b4a5763545d..6bd9dcbdb984dc502acfbba157e7a12f650d0248 100644
--- a/cmake/cuda.cmake
+++ b/cmake/cuda.cmake
@@ -189,7 +189,7 @@ add_definitions("-DPADDLE_CUDA_BINVER=\"${CUDA_VERSION_MAJOR}${CUDA_VERSION_MINO
 include_directories(${CUDA_INCLUDE_DIRS})
 if(NOT WITH_DSO)
     if(WIN32)
-      set_property(GLOBAL PROPERTY CUDA_MODULES ${CUDNN_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY})
+      set_property(GLOBAL PROPERTY CUDA_MODULES ${CUDNN_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY} ${CUDA_cusolver_LIBRARY})
     endif(WIN32)
 endif(NOT WITH_DSO)
 
diff --git a/paddle/fluid/platform/dynload/cusolver.cc b/paddle/fluid/platform/dynload/cusolver.cc
index 84aecabf6e2033173108bf59e5adcb2101511b74..5d841c55073b1fb2bf01c3545c44cffcb2b92fc5 100644
--- a/paddle/fluid/platform/dynload/cusolver.cc
+++ b/paddle/fluid/platform/dynload/cusolver.cc
@@ -25,6 +25,9 @@ void *cusolver_dso_handle;
 
 CUSOLVER_ROUTINE_EACH(DEFINE_WRAP);
 
+#ifdef CUSOLVER_ROUTINE_EACH_R1
+CUSOLVER_ROUTINE_EACH_R1(DEFINE_WRAP);
+#endif
 }  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/dynload/cusolver.h b/paddle/fluid/platform/dynload/cusolver.h
index 226e53369e8568620f3c0f0ba71ab2831a9303a4..379bf78d0a716f7b28031f4fffb1903ea6e3fb95 100644
--- a/paddle/fluid/platform/dynload/cusolver.h
+++ b/paddle/fluid/platform/dynload/cusolver.h
@@ -70,6 +70,7 @@ CUSOLVER_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUSOLVER_WRAP);
 CUSOLVER_ROUTINE_EACH_R1(DECLARE_DYNAMIC_LOAD_CUSOLVER_WRAP)
 #endif
 
+#undef DECLARE_DYNAMIC_LOAD_CUSOLVER_WRAP
 }  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc
index 3eb2b21fcc7a4a09cabfffdc65dada4a230efc72..e2e45dc7e8fb227bd852e5f539eb96b93fa75f9d 100644
--- a/paddle/fluid/platform/dynload/dynamic_loader.cc
+++ b/paddle/fluid/platform/dynload/dynamic_loader.cc
@@ -30,12 +30,12 @@ DEFINE_string(cudnn_dir, "",
 
 DEFINE_string(cuda_dir, "",
               "Specify path for loading cuda library, such as libcublas, "
-              "libcurand. For instance, /usr/local/cuda/lib64. If default, "
-              "dlopen will search cuda from LD_LIBRARY_PATH");
+              "libcurand, libcusolver. For instance, /usr/local/cuda/lib64. "
+              "If default, dlopen will search cuda from LD_LIBRARY_PATH");
 
 DEFINE_string(nccl_dir, "",
-              "Specify path for loading nccl library, such as libcublas, "
-              "libcurand. For instance, /usr/local/cuda/lib64. If default, "
+              "Specify path for loading nccl library, such as libnccl.so. "
+              "For instance, /usr/local/cuda/lib64. If default, "
               "dlopen will search cuda from LD_LIBRARY_PATH");
 
 DEFINE_string(cupti_dir, "", "Specify path for loading cupti.so.");
@@ -65,6 +65,8 @@ static PathNode s_py_site_pkg_path;
 static constexpr char* win_cublas_lib = "cublas64_" PADDLE_CUDA_BINVER ".dll";
 static constexpr char* win_curand_lib = "curand64_" PADDLE_CUDA_BINVER ".dll";
 static constexpr char* win_cudnn_lib = "cudnn64_" PADDLE_CUDNN_BINVER ".dll";
+static constexpr char* win_cusolver_lib =
+    "cusolver64_" PADDLE_CUDA_BINVER ".dll";
 #endif
 
 static inline std::string join(const std::string& part1,
@@ -228,7 +230,10 @@ void* GetCusolverDsoHandle() {
 #if defined(__APPLE__) || defined(__OSX__)
   return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.dylib");
 #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA)
-  return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib);
+  // TODO(guosheng): Fix cusolver support on windows.
+  // return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib);
+  PADDLE_THROW(platform::errors::Unavailable(
+      "Cusolver loader cannot support Windows temporarily."));
 #else
   return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.so");
 #endif