diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 9acfab19869e63c64aca4463ce3f525d75d3dbc5..1e80fb886cc2daeb1b50b5fef6cbad07c4c70b95 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include #include // NOLINT #include +#include #include "gflags/gflags.h" #include "glog/logging.h" @@ -58,6 +59,8 @@ struct PathNode { }; static constexpr char cupti_lib_path[] = CUPTI_LIB_PATH; + +// NOTE: In order to adapt to the default installation path of cuda on linux static constexpr char linux_cudnn_lib_path[] = "/usr/local/cuda/lib64"; static PathNode s_py_site_pkg_path; @@ -92,14 +95,29 @@ void SetPaddleLibPath(const std::string& py_site_pkg_path) { VLOG(3) << "Set paddle lib path : " << py_site_pkg_path; } +static inline void* GetDsoHandleFromSpecificPath(const std::string& spec_path, + const std::string& dso_name, + int dynload_flags) { + void* dso_handle = nullptr; + if (!spec_path.empty()) { + // search xxx.so from custom path + VLOG(3) << "Try to find library: " << dso_name + << " from specific path: " << spec_path; + std::string dso_path = join(spec_path, dso_name); + dso_handle = dlopen(dso_path.c_str(), dynload_flags); + } + return dso_handle; +} + static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, int dynload_flags) { - VLOG(3) << "Try to find library: " << dso_path - << " from default system path."; // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH // and /usr/local/lib path void* dso_handle = dlopen(dso_path.c_str(), dynload_flags); + VLOG(3) << "Try to find library: " << dso_path + << " from default system path."; +// TODO(chenweihang): This path is used to search which libs? // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to // bring System Integrity Projection (SIP), if dso_handle // is null, search from default package path in Mac OS. @@ -107,80 +125,72 @@ static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, if (nullptr == dso_handle) { dso_handle = dlopen(join("/usr/local/cuda/lib/", dso_path).c_str(), dynload_flags); - if (nullptr == dso_handle) { - if (dso_path == "libcudnn.dylib") { - LOG(WARNING) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n " - "For instance, sudo tar -xzf " - "cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local \n sudo " - "chmod a+r /usr/local/cuda/include/cudnn.h " - "/usr/local/cuda/lib/libcudnn*"; - } - } } #endif - if (nullptr == dso_handle) { - LOG(WARNING) << "Can not find library: " << dso_path - << ". The process maybe hang. Please try to add the lib path " - "to LD_LIBRARY_PATH."; - } return dso_handle; } -static inline void* GetDsoHandleFromSearchPath(const std::string& search_root, - const std::string& dso_name, - bool throw_on_error = true) { +/* + * We define three priorities for dynamic library search: + * + * First: Search for the path specified by the user + * Second: Search the system default path + * Third: Search for a special path corresponding to + * a specific library to adapt to changes and easy to expand. + */ + +static inline void* GetDsoHandleFromSearchPath( + const std::string& config_path, const std::string& dso_name, + bool throw_on_error = true, + const std::vector& extra_paths = std::vector(), + const std::string& warning_msg = std::string()) { #if !defined(_WIN32) int dynload_flags = RTLD_LAZY | RTLD_LOCAL; #else int dynload_flags = 0; #endif // !_WIN32 - void* dso_handle = nullptr; + // 1. search in user config path by FLAGS + void* dso_handle = + GetDsoHandleFromSpecificPath(config_path, dso_name, dynload_flags); + // 2. search in system default path + if (nullptr == dso_handle) { + dso_handle = GetDsoHandleFromDefaultPath(dso_name, dynload_flags); + } + // 3. search in extra paths + if (nullptr == dso_handle) { + for (auto path : extra_paths) { + dso_handle = GetDsoHandleFromSpecificPath(path, dso_name, dynload_flags); + } + } - std::string dlPath = dso_name; - if (search_root.empty()) { - dso_handle = GetDsoHandleFromDefaultPath(dlPath, dynload_flags); - } else { - // search xxx.so from custom path - dlPath = join(search_root, dso_name); - dso_handle = dlopen(dlPath.c_str(), dynload_flags); + // 4. [If Failed] logging warning if exists + if (nullptr == dso_handle && !warning_msg.empty()) { + LOG(WARNING) << warning_msg; + } + + // 5. [If Failed] logging or throw error info + if (nullptr == dso_handle) { + auto error_msg = + "Failed to find dynamic library: %s ( %s ) \n" + "Please specify its path correctly using following ways: \n" + " set environment variable LD_LIBRARY_PATH on Linux or " + "DYLD_LIBRARY_PATH on Mac OS. \n" + " For instance, issue command: export LD_LIBRARY_PATH=... \n" + " Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is " + "impossible unless System Integrity Protection (SIP) is disabled."; #if !defined(_WIN32) auto errorno = dlerror(); #else auto errorno = GetLastError(); #endif // !_WIN32 - // if not found, search from default path - if (nullptr == dso_handle) { - LOG(WARNING) << "Failed to find dynamic library: " << dlPath << " (" - << errorno << ")"; - if (dlPath.find("nccl") != std::string::npos) { - std::cout - << "You may need to install 'nccl2' from NVIDIA official website: " - << "https://developer.nvidia.com/nccl/nccl-download" - << "before install PaddlePaddle" << std::endl; - } - dlPath = dso_name; - dso_handle = GetDsoHandleFromDefaultPath(dlPath, dynload_flags); + if (throw_on_error) { + // NOTE: Special error report case, no need to change its format + PADDLE_THROW(platform::errors::NotFound(error_msg, dso_name, errorno)); + } else { + LOG(WARNING) << string::Sprintf(error_msg, dso_name, errorno); } } - auto error_msg = - "Failed to find dynamic library: %s ( %s ) \n Please specify " - "its path correctly using following ways: \n set " - "environment variable LD_LIBRARY_PATH on Linux or " - "DYLD_LIBRARY_PATH on Mac OS. \n For instance, issue command: " - "export LD_LIBRARY_PATH=... \n Note: After Mac OS 10.11, " - "using the DYLD_LIBRARY_PATH is impossible unless System " - "Integrity Protection (SIP) is disabled."; -#if !defined(_WIN32) - auto errorno = dlerror(); -#else - auto errorno = GetLastError(); -#endif // !_WIN32 - if (throw_on_error) { - PADDLE_ENFORCE(nullptr != dso_handle, error_msg, dlPath, errorno); - } else if (nullptr == dso_handle) { - LOG(WARNING) << string::Sprintf(error_msg, dlPath, errorno); - } return dso_handle; } @@ -197,27 +207,29 @@ void* GetCublasDsoHandle() { void* GetCUDNNDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", false); + std::string mac_warn_meg( + "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n " + "For instance, sudo tar -xzf " + "cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local \n sudo " + "chmod a+r /usr/local/cuda/include/cudnn.h " + "/usr/local/cuda/lib/libcudnn*"); + return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", false, + {}, mac_warn_meg); #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA) return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, win_cudnn_lib); #else - std::string linux_cudnn_path = linux_cudnn_lib_path; - if (!FLAGS_cudnn_dir.empty()) { - linux_cudnn_path = FLAGS_cudnn_dir; - } - return GetDsoHandleFromSearchPath(linux_cudnn_path, "libcudnn.so", false); + return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", false, + {linux_cudnn_lib_path}); #endif } void* GetCUPTIDsoHandle() { - std::string cupti_path = cupti_lib_path; - if (!FLAGS_cupti_dir.empty()) { - cupti_path = FLAGS_cupti_dir; - } #if defined(__APPLE__) || defined(__OSX__) - return GetDsoHandleFromSearchPath(cupti_path, "libcupti.dylib", false); + return GetDsoHandleFromSearchPath(FLAGS_cupti_dir, "libcupti.dylib", false, + {cupti_lib_path}); #else - return GetDsoHandleFromSearchPath(cupti_path, "libcupti.so", false); + return GetDsoHandleFromSearchPath(FLAGS_cupti_dir, "libcupti.so", false, + {cupti_lib_path}); #endif } @@ -272,10 +284,16 @@ void* GetWarpCTCDsoHandle() { } void* GetNCCLDsoHandle() { + std::string warning_msg( + "You may need to install 'nccl2' from NVIDIA official website: " + "https://developer.nvidia.com/nccl/nccl-download" + "before install PaddlePaddle."); #if defined(__APPLE__) || defined(__OSX__) - return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.dylib"); + return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.dylib", true, {}, + warning_msg); #else - return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.so"); + return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.so", true, {}, + warning_msg); #endif }