diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt
index 4f6381b8afda687412f1e4b61dab8663afcf8d5f..cc6b52e9271ff00e91f2ca172815c543eb99261d 100644
--- a/paddle/platform/CMakeLists.txt
+++ b/paddle/platform/CMakeLists.txt
@@ -1,6 +1,6 @@
+add_subdirectory(dynload)
+
 nv_test(cuda_test SRCS cuda_test.cu)
 
 cc_library(place SRCS place.cc)
 cc_test(place_test SRCS place_test.cc DEPS place glog gflags)
-
-cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags)
diff --git a/paddle/platform/dynload/CMakeLists.txt b/paddle/platform/dynload/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9f829b70128655f018c59db32b95d3f1789da5fc
--- /dev/null
+++ b/paddle/platform/dynload/CMakeLists.txt
@@ -0,0 +1 @@
+cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags)
diff --git a/paddle/platform/cublas.h b/paddle/platform/dynload/cublas.h
similarity index 95%
rename from paddle/platform/cublas.h
rename to paddle/platform/dynload/cublas.h
index 90704f37e643a3bfe318a949762fc4075666e707..c9150ac5738f3292f530f053d7c0bb8a1203f13d 100644
--- a/paddle/platform/cublas.h
+++ b/paddle/platform/dynload/cublas.h
@@ -19,7 +19,7 @@ limitations under the License. */
 
 namespace paddle {
 namespace platform {
-namespace dyload {
+namespace dynload {
 
 std::once_flag cublas_dso_flag;
 void *cublas_dso_handle = nullptr;
@@ -32,17 +32,17 @@ void *cublas_dso_handle = nullptr;
  * note: default dynamic linked libs
  */
 #ifdef PADDLE_USE_DSO
-#define DYNAMIC_LOAD_CUBLAS_WRAP(__name)                           \
-  struct DynLoad__##__name {                                       \
-    template <typename... Args>                                    \
-    cublasStatus_t operator()(Args... args) {                      \
-      typedef cublasStatus_t (*cublasFunc)(Args...);               \
-      std::call_once(cublas_dso_flag,                              \
-                     paddle::platform::dyload::GetCublasDsoHandle, \
-                     &cublas_dso_handle);                          \
-      void *p_##__name = dlsym(cublas_dso_handle, #__name);        \
-      return reinterpret_cast<cublasFunc>(p_##__name)(args...);    \
-    }                                                              \
+#define DYNAMIC_LOAD_CUBLAS_WRAP(__name)                            \
+  struct DynLoad__##__name {                                        \
+    template <typename... Args>                                     \
+    cublasStatus_t operator()(Args... args) {                       \
+      typedef cublasStatus_t (*cublasFunc)(Args...);                \
+      std::call_once(cublas_dso_flag,                               \
+                     paddle::platform::dynload::GetCublasDsoHandle, \
+                     &cublas_dso_handle);                           \
+      void *p_##__name = dlsym(cublas_dso_handle, #__name);         \
+      return reinterpret_cast<cublasFunc>(p_##__name)(args...);     \
+    }                                                               \
   } __name;  // struct DynLoad__##__name
 #else
 #define DYNAMIC_LOAD_CUBLAS_WRAP(__name)      \
@@ -99,6 +99,6 @@ CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP)
 #define CUBLAS_GETRF paddle::platform::dynload::cublasDgetrfBatched
 #define CUBLAS_GETRI paddle::platform::dynload::cublasDgetriBatched
 #endif
-}  // namespace dyload
+}  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/platform/cudnn.h b/paddle/platform/dynload/cudnn.h
similarity index 97%
rename from paddle/platform/cudnn.h
rename to paddle/platform/dynload/cudnn.h
index 06e2a05d862620fae9d61a35d84530dd8f34018f..c03424b375e5dc80610eb4ca8146069b7300e016 100644
--- a/paddle/platform/cudnn.h
+++ b/paddle/platform/dynload/cudnn.h
@@ -19,24 +19,24 @@ limitations under the License. */
 
 namespace paddle {
 namespace platform {
-namespace dyload {
+namespace dynload {
 
 std::once_flag cudnn_dso_flag;
 void* cudnn_dso_handle = nullptr;
 
 #ifdef PADDLE_USE_DSO
 
-#define DYNAMIC_LOAD_CUDNN_WRAP(__name)                           \
-  struct DynLoad__##__name {                                      \
-    template <typename... Args>                                   \
-    auto operator()(Args... args) -> decltype(__name(args...)) {  \
-      using cudnn_func = decltype(__name(args...)) (*)(Args...);  \
-      std::call_once(cudnn_dso_flag,                              \
-                     paddle::platform::dyload::GetCudnnDsoHandle, \
-                     &cudnn_dso_handle);                          \
-      void* p_##__name = dlsym(cudnn_dso_handle, #__name);        \
-      return reinterpret_cast<cudnn_func>(p_##__name)(args...);   \
-    }                                                             \
+#define DYNAMIC_LOAD_CUDNN_WRAP(__name)                            \
+  struct DynLoad__##__name {                                       \
+    template <typename... Args>                                    \
+    auto operator()(Args... args) -> decltype(__name(args...)) {   \
+      using cudnn_func = decltype(__name(args...)) (*)(Args...);   \
+      std::call_once(cudnn_dso_flag,                               \
+                     paddle::platform::dynload::GetCudnnDsoHandle, \
+                     &cudnn_dso_handle);                           \
+      void* p_##__name = dlsym(cudnn_dso_handle, #__name);         \
+      return reinterpret_cast<cudnn_func>(p_##__name)(args...);    \
+    }                                                              \
   } __name; /* struct DynLoad__##__name */
 
 #else
@@ -129,6 +129,6 @@ CUDNN_DNN_ROUTINE_EACH_R5(DYNAMIC_LOAD_CUDNN_WRAP)
 
 #undef CUDNN_DNN_ROUTINE_EACH
 // clang-format on
-}  // namespace dyload
+}  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/platform/curand.h b/paddle/platform/dynload/curand.h
similarity index 93%
rename from paddle/platform/curand.h
rename to paddle/platform/dynload/curand.h
index a9cbe48ef81808e5a71ee819ad87c803938affc1..1ef7a8c833d7488ec18f7c8df55b16006e94cc72 100644
--- a/paddle/platform/curand.h
+++ b/paddle/platform/dynload/curand.h
@@ -19,21 +19,21 @@ limitations under the License. */
 
 namespace paddle {
 namespace platform {
-namespace dyload {
+namespace dynload {
 std::once_flag curand_dso_flag;
 void *curand_dso_handle = nullptr;
 #ifdef PADDLE_USE_DSO
-#define DYNAMIC_LOAD_CURAND_WRAP(__name)                           \
-  struct DynLoad__##__name {                                       \
-    template <typename... Args>                                    \
-    curandStatus_t operator()(Args... args) {                      \
-      typedef curandStatus_t (*curandFunc)(Args...);               \
-      std::call_once(curand_dso_flag,                              \
-                     paddle::platform::dyload::GetCurandDsoHandle, \
-                     &curand_dso_handle);                          \
-      void *p_##__name = dlsym(curand_dso_handle, #__name);        \
-      return reinterpret_cast<curandFunc>(p_##__name)(args...);    \
-    }                                                              \
+#define DYNAMIC_LOAD_CURAND_WRAP(__name)                            \
+  struct DynLoad__##__name {                                        \
+    template <typename... Args>                                     \
+    curandStatus_t operator()(Args... args) {                       \
+      typedef curandStatus_t (*curandFunc)(Args...);                \
+      std::call_once(curand_dso_flag,                               \
+                     paddle::platform::dynload::GetCurandDsoHandle, \
+                     &curand_dso_handle);                           \
+      void *p_##__name = dlsym(curand_dso_handle, #__name);         \
+      return reinterpret_cast<curandFunc>(p_##__name)(args...);     \
+    }                                                               \
   } __name; /* struct DynLoad__##__name */
 #else
 #define DYNAMIC_LOAD_CURAND_WRAP(__name)      \
@@ -60,6 +60,6 @@ CURAND_RAND_ROUTINE_EACH(DYNAMIC_LOAD_CURAND_WRAP)
 
 #undef CURAND_RAND_ROUTINE_EACH
 #undef DYNAMIC_LOAD_CURAND_WRAP
-}  // namespace dyload
+}  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/platform/dynamic_loader.cc b/paddle/platform/dynload/dynamic_loader.cc
similarity index 99%
rename from paddle/platform/dynamic_loader.cc
rename to paddle/platform/dynload/dynamic_loader.cc
index 9e0aadf8e29398d93be148c2b050ce36bdbe687c..8ef67bad8c63cd1dfff967dbf1627ed84645e0d0 100644
--- a/paddle/platform/dynamic_loader.cc
+++ b/paddle/platform/dynload/dynamic_loader.cc
@@ -36,7 +36,7 @@ DEFINE_string(lapack_dir, "", "Specify path for loading liblapack.so.");
 
 namespace paddle {
 namespace platform {
-namespace dyload {
+namespace dynload {
 
 static inline std::string join(const std::string& part1,
                                const std::string& part2) {
@@ -164,6 +164,6 @@ void GetLapackDsoHandle(void** dso_handle) {
 #endif
 }
 
-}  // namespace dyload
+}  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/platform/dynamic_loader.h b/paddle/platform/dynload/dynamic_loader.h
similarity index 96%
rename from paddle/platform/dynamic_loader.h
rename to paddle/platform/dynload/dynamic_loader.h
index bb58fcba17326a42df33cf12599360e3d9a8c9ce..a99b05443feb909f10b2c56f4d8bdf3c6fa11e3f 100644
--- a/paddle/platform/dynamic_loader.h
+++ b/paddle/platform/dynload/dynamic_loader.h
@@ -16,7 +16,7 @@ limitations under the License. */
 
 namespace paddle {
 namespace platform {
-namespace dyload {
+namespace dynload {
 
 /**
  * @brief    load the DSO of CUBLAS
@@ -58,6 +58,6 @@ void GetWarpCTCDsoHandle(void** dso_handle);
  */
 void GetLapackDsoHandle(void** dso_handle);
 
-}  // namespace dyload
+}  // namespace dynload
 }  // namespace platform
 }  // namespace paddle