diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc
index 8ed998b0a464eb2e732b3dec7b1bd82c84163e3f..9acfab19869e63c64aca4463ce3f525d75d3dbc5 100644
--- a/paddle/fluid/platform/dynload/dynamic_loader.cc
+++ b/paddle/fluid/platform/dynload/dynamic_loader.cc
@@ -58,6 +58,7 @@ struct PathNode {
 };
 
 static constexpr char cupti_lib_path[] = CUPTI_LIB_PATH;
+static constexpr char linux_cudnn_lib_path[] = "/usr/local/cuda/lib64";
 
 static PathNode s_py_site_pkg_path;
 
@@ -200,7 +201,11 @@ void* GetCUDNNDsoHandle() {
 #elif defined(_WIN32) && defined(PADDLE_WITH_CUDA)
   return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, win_cudnn_lib);
 #else
-  return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", false);
+  std::string linux_cudnn_path = linux_cudnn_lib_path;
+  if (!FLAGS_cudnn_dir.empty()) {
+    linux_cudnn_path = FLAGS_cudnn_dir;
+  }
+  return GetDsoHandleFromSearchPath(linux_cudnn_path, "libcudnn.so", false);
 #endif
 }