From dc53ba879f38c8265cb61ee5c4284eef6598da72 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Wed, 21 Oct 2020 11:32:01 +0800 Subject: [PATCH] fix dynamic_loader more safe and error message on windows (#28144) --- CMakeLists.txt | 27 +++--- cmake/cuda.cmake | 5 +- cmake/cudnn.cmake | 11 ++- .../fluid/platform/dynload/dynamic_loader.cc | 92 ++++++++++++++----- paddle/fluid/platform/port.h | 6 +- 5 files changed, 96 insertions(+), 45 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 06d687fc9c4..1a8eef5e66b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,10 +193,19 @@ if(WITH_BRPC_RDMA) endif() endif() -# lite subgraph compilation depends on CUDNN_ROOT, -# so include(cudnn) needs to be in front of include(third_party/lite) -include(cudnn) # set cudnn libraries, must before configure -include(third_party) # download, build, install third_party +if(WITH_GPU) + include(cuda) + # lite subgraph compilation depends on CUDNN_ROOT, + # so include(cudnn) needs to be in front of include(third_party/lite) + include(cudnn) # set cudnn libraries, must before configure + include(tensorrt) + # there is no official support of nccl, cupti in windows + if(NOT WIN32) + include(cupti) + endif() +endif() + +include(third_party) # download, build, install third_party, Contains about 20+ dependencies if(WITH_DISTRIBUTE) if(WITH_GRPC) @@ -209,18 +218,8 @@ if(WITH_DISTRIBUTE) endif() endif() -# there is no official support of nccl, cupti in windows -if(NOT WIN32) - include(cupti) -endif() - include(flags) # set paddle compile flags -if(WITH_GPU) - include(cuda) - include(tensorrt) -endif() - if(WITH_PROFILER) find_package(Gperftools REQUIRED) include_directories(${GPERFTOOLS_INCLUDE_DIR}) diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index c78fe5f6c7f..146cbee1c6a 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -198,7 +198,9 @@ elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0) # CUDA 11.x set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__") endif() -add_definitions("-DPADDLE_CUDA_BINVER=\"${CUDA_VERSION_MAJOR}${CUDA_VERSION_MINOR}\"") +add_definitions("-DCUDA_VERSION_MAJOR=\"${CUDA_VERSION_MAJOR}\"") +add_definitions("-DCUDA_VERSION_MINOR=\"${CUDA_VERSION_MINOR}\"") +add_definitions("-DCUDA_TOOLKIT_ROOT_DIR=\"${CUDA_TOOLKIT_ROOT_DIR}\"") # setting nvcc arch flags select_nvcc_arch_flags(NVCC_FLAGS_EXTRA) @@ -249,3 +251,4 @@ endif() mark_as_advanced(CUDA_BUILD_CUBIN CUDA_BUILD_EMULATION CUDA_VERBOSE_BUILD) mark_as_advanced(CUDA_SDK_ROOT_DIR CUDA_SEPARABLE_COMPILATION) + diff --git a/cmake/cudnn.cmake b/cmake/cudnn.cmake index b68e1b4070c..d8d8f634e76 100644 --- a/cmake/cudnn.cmake +++ b/cmake/cudnn.cmake @@ -35,17 +35,18 @@ list(APPEND CUDNN_CHECK_LIBRARY_DIRS ${CUDA_TOOLKIT_ROOT_DIR}/lib/x64 ) set(CUDNN_LIB_NAME "") + if (LINUX) -set(CUDNN_LIB_NAME "libcudnn.so") + set(CUDNN_LIB_NAME "libcudnn.so") endif(LINUX) if(WIN32) -# only support cudnn7 -set(CUDNN_LIB_NAME "cudnn.lib" "cudnn64_7.dll") + # only support cudnn7 + set(CUDNN_LIB_NAME "cudnn.lib" "cudnn64_7.dll") endif(WIN32) if(APPLE) -set(CUDNN_LIB_NAME "libcudnn.dylib" "libcudnn.so") + set(CUDNN_LIB_NAME "libcudnn.dylib" "libcudnn.so") endif(APPLE) find_library(CUDNN_LIBRARY NAMES ${CUDNN_LIB_NAME} # libcudnn_static.a @@ -88,7 +89,7 @@ macro(find_cudnn_version cudnn_header_file) if(NOT CUDNN_MAJOR_VERSION) set(CUDNN_VERSION "???") else() - add_definitions("-DPADDLE_CUDNN_BINVER=\"${CUDNN_MAJOR_VERSION}\"") + add_definitions("-DCUDNN_MAJOR_VERSION=\"${CUDNN_MAJOR_VERSION}\"") math(EXPR CUDNN_VERSION "${CUDNN_MAJOR_VERSION} * 1000 + ${CUDNN_MINOR_VERSION} * 100 + ${CUDNN_PATCHLEVEL_VERSION}") diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 0c8a64ccf69..4d911d12e55 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -57,17 +57,26 @@ struct PathNode { static constexpr char cupti_lib_path[] = CUPTI_LIB_PATH; -// NOTE: In order to adapt to the default installation path of cuda on linux -static constexpr char linux_cudnn_lib_path[] = "/usr/local/cuda/lib64"; +// NOTE: In order to adapt to the default installation path of cuda +#if defined(_WIN32) && defined(PADDLE_WITH_CUDA) +static constexpr char cuda_lib_path[] = CUDA_TOOLKIT_ROOT_DIR "/bin"; +#else +static constexpr char cuda_lib_path[] = "/usr/local/cuda/lib64"; +#endif static PathNode s_py_site_pkg_path; #if defined(_WIN32) && defined(PADDLE_WITH_CUDA) -static constexpr char* win_cublas_lib = "cublas64_" PADDLE_CUDA_BINVER ".dll"; -static constexpr char* win_curand_lib = "curand64_" PADDLE_CUDA_BINVER ".dll"; -static constexpr char* win_cudnn_lib = "cudnn64_" PADDLE_CUDNN_BINVER ".dll"; +static constexpr char* win_cublas_lib = + "cublas64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR + ".dll;cublas64_" CUDA_VERSION_MAJOR ".dll"; +static constexpr char* win_curand_lib = + "curand64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR + ".dll;curand64_" CUDA_VERSION_MAJOR ".dll"; +static constexpr char* win_cudnn_lib = "cudnn64_" CUDNN_MAJOR_VERSION ".dll"; static constexpr char* win_cusolver_lib = - "cusolver64_" PADDLE_CUDA_BINVER ".dll"; + "cusolver64_" CUDA_VERSION_MAJOR CUDA_VERSION_MINOR + ".dll;cusolver64_" CUDA_VERSION_MAJOR ".dll"; #endif static inline std::string join(const std::string& part1, @@ -87,6 +96,24 @@ static inline std::string join(const std::string& part1, return ret; } +static inline std::vector split( + const std::string& str, const std::string separator = " ") { + std::vector str_list; + std::string::size_type firstPos; + firstPos = str.find_first_not_of(separator, 0); + std::string::size_type lastPos; + lastPos = str.find_first_of(separator, firstPos); + while (std::string::npos != firstPos && std::string::npos != lastPos) { + str_list.push_back(str.substr(firstPos, lastPos - firstPos)); + firstPos = str.find_first_not_of(separator, lastPos); + lastPos = str.find_first_of(separator, firstPos); + } + if (std::string::npos == lastPos) { + str_list.push_back(str.substr(firstPos, lastPos - firstPos)); + } + return str_list; +} + void SetPaddleLibPath(const std::string& py_site_pkg_path) { s_py_site_pkg_path.path = py_site_pkg_path; VLOG(3) << "Set paddle lib path : " << py_site_pkg_path; @@ -147,26 +174,31 @@ static inline void* GetDsoHandleFromSearchPath( #else int dynload_flags = 0; #endif // !_WIN32 - // 1. search in user config path by FLAGS - void* dso_handle = - GetDsoHandleFromSpecificPath(config_path, dso_name, dynload_flags); - // 2. search in system default path - if (nullptr == dso_handle) { - dso_handle = GetDsoHandleFromDefaultPath(dso_name, dynload_flags); - } - // 3. search in extra paths - if (nullptr == dso_handle) { - for (auto path : extra_paths) { - dso_handle = GetDsoHandleFromSpecificPath(path, dso_name, dynload_flags); + std::vector dso_names = split(dso_name, ";"); + void* dso_handle = nullptr; + for (auto dso : dso_names) { + // 1. search in user config path by FLAGS + dso_handle = GetDsoHandleFromSpecificPath(config_path, dso, dynload_flags); + // 2. search in extra paths + if (nullptr == dso_handle) { + for (auto path : extra_paths) { + VLOG(3) << "extra_paths: " << path; + dso_handle = GetDsoHandleFromSpecificPath(path, dso, dynload_flags); + } + } + // 3. search in system default path + if (nullptr == dso_handle) { + dso_handle = GetDsoHandleFromDefaultPath(dso, dynload_flags); } + if (nullptr != dso_handle) break; } - // 4. [If Failed] logging warning if exists + // 4. [If Failed for All dso_names] logging warning if exists if (nullptr == dso_handle && !warning_msg.empty()) { LOG(WARNING) << warning_msg; } - // 5. [If Failed] logging or throw error info + // 5. [If Failed for All dso_names] logging or throw error info if (nullptr == dso_handle) { auto error_msg = "The third-party dynamic library (%s) that Paddle depends on is not " @@ -203,7 +235,8 @@ void* GetCublasDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib"); #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) - return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cublas_lib); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cublas_lib, true, + {cuda_lib_path}); #else return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so"); #endif @@ -220,10 +253,19 @@ void* GetCUDNNDsoHandle() { return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", false, {}, mac_warn_meg); #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) - return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, win_cudnn_lib); + std::string win_warn_meg( + "Note: [Recommend] copy cudnn into CUDA installation directory. \n " + "For instance, download cudnn-10.0-windows10-x64-v7.6.5.32.zip from " + "NVIDIA's official website, \n" + "then, unzip it and copy it into C:\\Program Files\\NVIDIA GPU Computing " + "Toolkit\\CUDA/v10.0\n" + "You should do this according to your CUDA installation directory and " + "CUDNN version."); + return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, win_cudnn_lib, true, + {cuda_lib_path}, win_warn_meg); #else return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", false, - {linux_cudnn_lib_path}); + {cuda_lib_path}); #endif } @@ -241,7 +283,8 @@ void* GetCurandDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib"); #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) - return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_curand_lib); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_curand_lib, true, + {cuda_lib_path}); #else return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so"); #endif @@ -251,7 +294,8 @@ void* GetCusolverDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.dylib"); #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) - return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, win_cusolver_lib, true, + {cuda_lib_path}); #else return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcusolver.so"); #endif diff --git a/paddle/fluid/platform/port.h b/paddle/fluid/platform/port.h index c5e8ff807a2..b2f26ba9581 100644 --- a/paddle/fluid/platform/port.h +++ b/paddle/fluid/platform/port.h @@ -56,7 +56,11 @@ static void *dlopen(const char *filename, int flag) { std::string file_name(filename); HMODULE hModule = LoadLibrary(file_name.c_str()); if (!hModule) { - throw std::runtime_error(file_name + " not found."); + if (flag) { + throw std::runtime_error(file_name + " not found."); + } else { + return nullptr; + } } return reinterpret_cast(hModule); } -- GitLab