diff --git a/.gitmodules b/.gitmodules
index 3ea3f4a0903a0827c99103d6edb8ce18e96c122e..52776b68ef22ca02dd325d89eb47c552fc403db3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -82,3 +82,23 @@
 	path = third_party/gtest
 	url = https://github.com/google/googletest.git
 	ignore = dirty
+[submodule "third_party/leveldb"]
+	path = third_party/leveldb
+	url = https://github.com/google/leveldb.git
+	ignore = dirty
+[submodule "third_party/brpc"]
+	path = third_party/brpc
+	url = https://github.com/apache/brpc.git
+	ignore = dirty
+[submodule "third_party/rocksdb"]
+	path = third_party/rocksdb
+	url = https://github.com/Thunderbrook/rocksdb
+	ignore = dirty
+[submodule "third_party/absl"]
+	path = third_party/absl
+	url = https://github.com/abseil/abseil-cpp.git
+	ignore = dirty
+[submodule "third_party/jitify"]
+	path = third_party/jitify
+	url = https://github.com/NVIDIA/jitify.git
+	ignore = dirty
diff --git a/cmake/cinn/external/absl.cmake b/cmake/cinn/external/absl.cmake
index 051a83c956882048aa7cc2998df80721d72e8fbc..56befafecea2146daa552fe8a4dc9db5b7154a61 100644
--- a/cmake/cinn/external/absl.cmake
+++ b/cmake/cinn/external/absl.cmake
@@ -8,24 +8,6 @@ set(ABSL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 set(ABSL_REPOSITORY "https://github.com/abseil/abseil-cpp.git")
 set(ABSL_TAG "20210324.2")
 
-if(NOT EXISTS ${ABSL_SOURCES_DIR})
-  message(
-    STATUS "Download absl source from ${ABSL_REPOSITORY} to ABSL_SOURCES_DIR")
-  execute_process(COMMAND ${GIT_EXECUTABLE} clone -b ${ABSL_TAG}
-                          ${ABSL_REPOSITORY} ${ABSL_SOURCES_DIR})
-else()
-  # check git tag
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} -C ${ABSL_SOURCES_DIR} describe --tags
-    OUTPUT_VARIABLE CURRENT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if(NOT ${CURRENT_TAG} STREQUAL ${ABSL_TAG})
-    message(STATUS "Checkout absl to ${ABSL_TAG}")
-    execute_process(COMMAND ${GIT_EXECUTABLE} -C ${ABSL_SOURCES_DIR} checkout
-                            -q ${ABSL_TAG})
-  endif()
-endif()
-
 set(OPTIONAL_ARGS
     "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
     "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
diff --git a/cmake/cinn/external/jitify.cmake b/cmake/cinn/external/jitify.cmake
index cb49f4261dfb18bec0546917d630a3d38343b80f..7750934d8056c7c903425015d8d7648b3af1966d 100644
--- a/cmake/cinn/external/jitify.cmake
+++ b/cmake/cinn/external/jitify.cmake
@@ -10,24 +10,6 @@ set(JITIFY_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/jitify)
 set(JITIFY_URL https://github.com/NVIDIA/jitify.git)
 set(JITIFY_TAG 57de649139c866eb83acacfe50c92ad7c6278776)
 
-if(NOT EXISTS ${JITIFY_SOURCE_DIR})
-  execute_process(COMMAND ${GIT_EXECUTABLE} clone ${JITIFY_URL}
-                          ${JITIFY_SOURCE_DIR})
-  execute_process(COMMAND ${GIT_EXECUTABLE} -C ${JITIFY_SOURCE_DIR} checkout -q
-                          ${JITIFY_TAG})
-else()
-  # check git tag
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} -C ${JITIFY_SOURCE_DIR} describe --tags
-    OUTPUT_VARIABLE CURRENT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if(NOT ${CURRENT_TAG} STREQUAL ${JITIFY_TAG})
-    message(STATUS "Checkout JITIFY to ${JITIFY_TAG}")
-    execute_process(COMMAND ${GIT_EXECUTABLE} -C ${JITIFY_SOURCE_DIR} checkout
-                            -q ${JITIFY_TAG})
-  endif()
-endif()
-
 ExternalProject_Add(
   external_jitify
   ${EXTERNAL_PROJECT_LOG_ARGS}
diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake
index 3c44168921a7cd434ee63ab250ddf2d6da103064..ff33e142addbd6a2df79e6a90c50ccdeb1fda332 100755
--- a/cmake/external/brpc.cmake
+++ b/cmake/external/brpc.cmake
@@ -41,22 +41,6 @@ set(BRPC_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/brpc)
 set(BRPC_URL https://github.com/apache/brpc.git)
 set(BRPC_TAG 1.4.0)
 
-if(NOT EXISTS ${BRPC_SOURCE_DIR})
-  execute_process(COMMAND ${GIT_EXECUTABLE} clone -b ${BRPC_TAG} ${BRPC_URL}
-                          ${BRPC_SOURCE_DIR})
-else()
-  # check git tag
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} -C ${BRPC_SOURCE_DIR} describe --tags
-    OUTPUT_VARIABLE CURRENT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if(NOT ${CURRENT_TAG} STREQUAL ${BRPC_TAG})
-    message(STATUS "Checkout brpc to ${BRPC_TAG}")
-    execute_process(COMMAND ${GIT_EXECUTABLE} -C ${BRPC_SOURCE_DIR} checkout -q
-                            ${BRPC_TAG})
-  endif()
-endif()
-
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
 set(prefix_path
     "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog"
diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake
index a35b6f429fa30f96f1eed96a11e625e423203a69..bf8387321df4a9a1af49627ae1d9261fa5ce5bc6 100644
--- a/cmake/external/leveldb.cmake
+++ b/cmake/external/leveldb.cmake
@@ -30,29 +30,6 @@ if(WITH_HETERPS AND WITH_PSLIB)
   set(LEVELDN_CXXFLAGS "${LEVELDN_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
 endif()
 
-file(GLOB LEVELDB_SOURCE_FILE_LIST ${LEVELDB_SOURCE_DIR})
-list(LENGTH LEVELDB_SOURCE_FILE_LIST RES_LEN)
-if(RES_LEN EQUAL 0)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} clone -b ${LEVELDB_TAG}
-            https://github.com/google/leveldb.git ${LEVELDB_SOURCE_DIR})
-else()
-  # check git tag
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --abbrev=6 --always --tags
-    OUTPUT_VARIABLE VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET
-    WORKING_DIRECTORY ${LEVELDB_SOURCE_DIR})
-  if(NOT ${VERSION} STREQUAL ${LEVELDB_TAG})
-    file(REMOVE_RECURSE ${LEVELDB_PREFIX_DIR})
-    file(REMOVE_RECURSE ${LEVELDB_INSTALL_DIR})
-    message(
-      WARNING "leveldb version is not ${VERSION}, checkout to ${LEVELDB_TAG}")
-    execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${LEVELDB_TAG}
-                    WORKING_DIRECTORY ${LEVELDB_SOURCE_DIR})
-  endif()
-endif()
-
 ExternalProject_Add(
   extern_leveldb
   ${EXTERNAL_PROJECT_LOG_ARGS}
diff --git a/cmake/external/rocksdb.cmake b/cmake/external/rocksdb.cmake
index 98e9972e2706d2bba4aefa35ae4d12da472ba66b..5bf2a896c47d3f124242a29b957b95c735c3378e 100644
--- a/cmake/external/rocksdb.cmake
+++ b/cmake/external/rocksdb.cmake
@@ -51,29 +51,6 @@ set(ROCKSDB_CMAKE_C_FLAGS
 )
 include_directories(${ROCKSDB_INCLUDE_DIR})
 
-file(GLOB ROCKSDB_SOURCE_FILE_LIST ${ROCKSDB_SOURCE_DIR})
-list(LENGTH ROCKSDB_SOURCE_FILE_LIST RES_LEN)
-if(RES_LEN EQUAL 0)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} clone -b ${ROCKSDB_TAG}
-            "https://github.com/Thunderbrook/rocksdb" ${ROCKSDB_SOURCE_DIR})
-else()
-  # check git tag
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --abbrev=6 --always --tags
-    OUTPUT_VARIABLE VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET
-    WORKING_DIRECTORY ${ROCKSDB_SOURCE_DIR})
-  if(NOT ${VERSION} STREQUAL ${ROCKSDB_TAG})
-    file(REMOVE_RECURSE ${ROCKSDB_PREFIX_DIR})
-    file(REMOVE_RECURSE ${ROCKSDB_INSTALL_DIR})
-    message(
-      WARNING "rocksdb version is not ${VERSION}, checkout to ${ROCKSDB_TAG}")
-    execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${ROCKSDB_TAG}
-                    WORKING_DIRECTORY ${ROCKSDB_SOURCE_DIR})
-  endif()
-endif()
-
 set(CMAKE_CXX_LINK_EXECUTABLE
     "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -Wl,--no-as-needed -ldl -lrt -lz")
 if(WITH_ARM)
diff --git a/third_party/absl b/third_party/absl
new file mode 160000
index 0000000000000000000000000000000000000000..278e0a071885a22dcd2fd1b5576cc44757299343
--- /dev/null
+++ b/third_party/absl
@@ -0,0 +1 @@
+Subproject commit 278e0a071885a22dcd2fd1b5576cc44757299343
diff --git a/third_party/brpc b/third_party/brpc
new file mode 160000
index 0000000000000000000000000000000000000000..cf2a22000584f30309e49e01e634128b193abb3e
--- /dev/null
+++ b/third_party/brpc
@@ -0,0 +1 @@
+Subproject commit cf2a22000584f30309e49e01e634128b193abb3e
diff --git a/third_party/jitify b/third_party/jitify
new file mode 160000
index 0000000000000000000000000000000000000000..57de649139c866eb83acacfe50c92ad7c6278776
--- /dev/null
+++ b/third_party/jitify
@@ -0,0 +1 @@
+Subproject commit 57de649139c866eb83acacfe50c92ad7c6278776
diff --git a/third_party/leveldb b/third_party/leveldb
new file mode 160000
index 0000000000000000000000000000000000000000..803d69203a62faf50f1b77897310a3a1fcae712b
--- /dev/null
+++ b/third_party/leveldb
@@ -0,0 +1 @@
+Subproject commit 803d69203a62faf50f1b77897310a3a1fcae712b
diff --git a/third_party/rocksdb b/third_party/rocksdb
new file mode 160000
index 0000000000000000000000000000000000000000..9e18bf0e273b081de54ef1227e6f1db9e02a472a
--- /dev/null
+++ b/third_party/rocksdb
@@ -0,0 +1 @@
+Subproject commit 9e18bf0e273b081de54ef1227e6f1db9e02a472a