diff --git a/.gitmodules b/.gitmodules
index 70cef6c95693936cee80ba7f4dfd88f33d0c5d9f..9128f9233f0b8085479ec74f6a06f3c0d21192a2 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -2,6 +2,10 @@
 	path = third_party/protobuf
 	url = https://github.com/protocolbuffers/protobuf.git
 	ignore = dirty
+[submodule "third_party/pocketfft"]
+	path = third_party/pocketfft
+	url = https://gitlab.mpcdf.mpg.de/mtr/pocketfft.git
+	ignore = dirty
 [submodule "third_party/gflags"]
 	path = third_party/gflags
 	url = https://github.com/gflags/gflags.git
@@ -62,6 +66,10 @@
 	path = third_party/cutlass
 	url = https://github.com/NVIDIA/cutlass.git
 	ignore = dirty
+[submodule "third_party/xbyak"]
+	path = third_party/xbyak
+	url = https://github.com/herumi/xbyak.git
+	ignore = dirty
 [submodule "third_party/mkldnn"]
 	path = third_party/mkldnn
 	url = https://github.com/oneapi-src/oneDNN.git
diff --git a/cmake/external/pocketfft.cmake b/cmake/external/pocketfft.cmake
index f56b92d467b2a88f92cb4d491c0d3f557db46209..3599d2956026d11b2cc24ea7cb008ef5a3fe289d 100644
--- a/cmake/external/pocketfft.cmake
+++ b/cmake/external/pocketfft.cmake
@@ -19,32 +19,36 @@ set(POCKETFFT_PATH
     CACHE STRING "A path setting for external_pocketfft path.")
 set(POCKETFFT_PREFIX_DIR ${POCKETFFT_PATH})
 
-set(POCKETFFT_REPOSITORY https://gitlab.mpcdf.mpg.de/mtr/pocketfft.git)
-set(POCKETFFT_TAG release_for_eigen)
-
 set(POCKETFFT_INCLUDE_DIR ${POCKETFFT_PREFIX_DIR}/src)
+set(POCKETFFT_SOURCE_DIR ${POCKETFFT_PREFIX_DIR}/src/extern_pocketfft)
 message("POCKETFFT_INCLUDE_DIR is ${POCKETFFT_INCLUDE_DIR}")
 include_directories(${POCKETFFT_INCLUDE_DIR})
 
+set(POCKETFFT_TAG release_for_eigen)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/pocketfft)
+
 if(APPLE)
   file(TO_NATIVE_PATH
        ${PADDLE_SOURCE_DIR}/patches/pocketfft/pocketfft_hdronly.h.patch
        native_dst)
   set(POCKETFFT_PATCH_COMMAND
-      git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
-      ${POCKETFFT_INCLUDE_DIR}/extern_pocketfft < ${native_dst})
+      git checkout -- . && git checkout ${POCKETFFT_TAG} && patch -Nd
+      ${SOURCE_DIR} < ${native_dst})
 endif()
 
 ExternalProject_Add(
   extern_pocketfft
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${POCKETFFT_REPOSITORY}
-  GIT_TAG ${POCKETFFT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${POCKETFFT_PREFIX_DIR}
   PATCH_COMMAND ${POCKETFFT_PATCH_COMMAND}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
+  BUILD_COMMAND
+  COMMAND ${CMAKE_COMMAND} -E remove_directory ${POCKETFFT_SOURCE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${POCKETFFT_SOURCE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E copy_directory ${SOURCE_DIR}
+          ${POCKETFFT_SOURCE_DIR}
   INSTALL_COMMAND ""
   TEST_COMMAND "")
 
diff --git a/cmake/external/xbyak.cmake b/cmake/external/xbyak.cmake
index 589056458c1f0065814dd46eb66088071c6218d5..a146d53d606e95e7ea0d0a10de64540ebf3820c9 100644
--- a/cmake/external/xbyak.cmake
+++ b/cmake/external/xbyak.cmake
@@ -20,6 +20,7 @@ set(XBYAK_INSTALL_ROOT ${THIRD_PARTY_PATH}/install/xbyak)
 set(XBYAK_INC_DIR ${XBYAK_INSTALL_ROOT}/include)
 set(XBYAK_REPOSITORY ${GIT_URL}/herumi/xbyak.git)
 set(XBYAK_TAG v5.81) # Dec 19, 2019
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/xbyak)
 
 include_directories(${XBYAK_INC_DIR})
 include_directories(${XBYAK_INC_DIR}/xbyak)
@@ -33,8 +34,7 @@ add_definitions(-DXBYAK_NO_OP_NAMES)
 ExternalProject_Add(
   ${XBYAK_PROJECT}
   ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${XBYAK_REPOSITORY}
-  GIT_TAG ${XBYAK_TAG}
+  SOURCE_DIR ${SOURCE_DIR}
   DEPENDS ""
   PREFIX ${XBYAK_PREFIX_DIR}
   UPDATE_COMMAND ""
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index 8587d025dbb42d940181c502446770c4df1b26be..2187721269502ebaba3a607fe2fa26613701d6a7 100755
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -441,7 +441,7 @@ if(WITH_DISTRIBUTE
 endif()
 
 if(WITH_XBYAK)
-  include(external/xbyak) # download, build, install xbyak
+  include(external/xbyak) # prepare submodule xbyak
   list(APPEND third_party_deps extern_xbyak)
 endif()
 
diff --git a/third_party/pocketfft b/third_party/pocketfft
new file mode 160000
index 0000000000000000000000000000000000000000..ea778e37710c07723435b1be58235996d1d43a5a
--- /dev/null
+++ b/third_party/pocketfft
@@ -0,0 +1 @@
+Subproject commit ea778e37710c07723435b1be58235996d1d43a5a
diff --git a/third_party/xbyak b/third_party/xbyak
new file mode 160000
index 0000000000000000000000000000000000000000..4ca0434b4e78c05e3f3e68bda70e8713668e87d3
--- /dev/null
+++ b/third_party/xbyak
@@ -0,0 +1 @@
+Subproject commit 4ca0434b4e78c05e3f3e68bda70e8713668e87d3