diff --git a/.gitmodules b/.gitmodules
index 9128f9233f0b8085479ec74f6a06f3c0d21192a2..345e82c9bd0ef14421206788d0d6079720d642d7 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -38,6 +38,10 @@
 	path = third_party/leveldb
 	url = https://github.com/google/leveldb
 	ignore = dirty
+[submodule "third_party/pybind"]
+	path = third_party/pybind
+	url = https://github.com/pybind/pybind11.git
+	ignore = dirty
 [submodule "third_party/threadpool"]
 	path = third_party/threadpool
 	url = https://github.com/progschj/ThreadPool.git
diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake
index c2782af255c59f27647ccef94f1262e8dcace4d7..6ce8290d72f4247eca7b3cc7b59faa63aaa44486 100644
--- a/cmake/external/pybind11.cmake
+++ b/cmake/external/pybind11.cmake
@@ -15,10 +15,12 @@
 include(ExternalProject)
 
 set(PYBIND_PREFIX_DIR ${THIRD_PARTY_PATH}/pybind)
-set(PYBIND_REPOSITORY ${GIT_URL}/pybind/pybind11.git)
+set(PYBIND_SOURCE_DIR ${PYBIND_PREFIX_DIR}/src/extern_pybind)
+set(PYBIND_INCLUDE_DIR ${PYBIND_SOURCE_DIR}/include)
 set(PYBIND_TAG v2.10.3)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/pybind)
+set(SOURCE_INCLUDE_DIR ${SOURCE_DIR}/include)
 
-set(PYBIND_INCLUDE_DIR ${THIRD_PARTY_PATH}/pybind/src/extern_pybind/include)
 include_directories(${PYBIND_INCLUDE_DIR})
 
 set(PYBIND_PATCH_COMMAND "")
@@ -31,14 +33,13 @@ if(NOT WIN32)
   # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache.
   set(PYBIND_PATCH_COMMAND
       git checkout -- . && git checkout ${PYBIND_TAG} && patch -Nd
-      ${PYBIND_INCLUDE_DIR}/pybind11 < ${native_dst})
+      ${SOURCE_INCLUDE_DIR}/pybind11 < ${native_dst})
 endif()
 
 ExternalProject_Add(
   extern_pybind
   ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${PYBIND_REPOSITORY}
-  GIT_TAG ${PYBIND_TAG}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${PYBIND_PREFIX_DIR}
   # If we explicitly leave the `UPDATE_COMMAND` of the ExternalProject_Add
   # function in CMakeLists blank, it will cause another parameter GIT_TAG
@@ -48,7 +49,14 @@ ExternalProject_Add(
   UPDATE_COMMAND ""
   PATCH_COMMAND ${PYBIND_PATCH_COMMAND}
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
+  # I intentionally preserved an extern_pybind/include/pybind11 directory
+  # to site-packages, so that you could discern that you intended to
+  # employ not only python, but also CPP and were ready to incorporate header files.
+  BUILD_COMMAND
+  COMMAND ${CMAKE_COMMAND} -E remove_directory ${PYBIND_SOURCE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${PYBIND_SOURCE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E copy_directory ${SOURCE_INCLUDE_DIR}
+          ${PYBIND_INCLUDE_DIR}
   INSTALL_COMMAND ""
   TEST_COMMAND "")
 
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index 2187721269502ebaba3a607fe2fa26613701d6a7..884492d5bb7cf56260adc108bd6992703b89c8b5 100755
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -305,7 +305,7 @@ endif()
 
 if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
   include(external/python) # find python and python_module
-  include(external/pybind11) # download pybind11
+  include(external/pybind11) # prepare submodule pybind11
   list(APPEND third_party_deps extern_pybind)
 endif()
 
diff --git a/third_party/pybind b/third_party/pybind
new file mode 160000
index 0000000000000000000000000000000000000000..0bd8896a4010f2d91b2340570c24fa08606ec406
--- /dev/null
+++ b/third_party/pybind
@@ -0,0 +1 @@
+Subproject commit 0bd8896a4010f2d91b2340570c24fa08606ec406