diff --git a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
index 48ddb92213a8a7597599f5074cf7a318f7794b73..792daf1479896c9bd7818c8f7e800fa2ac704222 100644
--- a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
@@ -430,4 +430,3 @@ if((WITH_ROCM OR WITH_GPU) AND (LINUX))
   set_tests_properties(test_orthogonal_strategy PROPERTIES TIMEOUT "120")
 endif()
 add_subdirectory(fleet)
-add_subdirectory(multinode)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 60d05fbdf5715670bd16b64721a11d94dbb35c08..660443ad2eff3884439df27dde36c8bdab499a1f 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -5,12 +5,59 @@ set(PYTHON_TESTS_DIR
     ${PADDLE_BINARY_DIR}/test
     CACHE INTERNAL "python tests directory")
 
+function(bash_test_modules TARGET_NAME)
+  if(NOT WITH_TESTING)
+    return()
+  endif()
+
+  set(options SERIAL)
+  set(oneValueArgs TIMEOUT START_BASH)
+  set(multiValueArgs DEPS ENVS LABELS)
+  cmake_parse_arguments(bash_test_modules "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  set(timeout 350)
+  if(${bash_test_modules_TIMEOUT})
+    set(timeout ${bash_test_modules_TIMEOUT})
+  endif()
+
+  if(WITH_COVERAGE)
+    add_test(
+      NAME ${TARGET_NAME}
+      COMMAND
+        ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
+        TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout}
+        ${bash_test_modules_ENVS} WITH_COVERAGE=ON
+        COVERAGE_FILE=${PADDLE_BINARY_DIR}/python-coverage.data bash
+        ${CMAKE_CURRENT_BINARY_DIR}/${bash_test_modules_START_BASH}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  else()
+    add_test(
+      NAME ${TARGET_NAME}
+      COMMAND
+        ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
+        TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout}
+        ${bash_test_modules_ENVS} bash
+        ${CMAKE_CURRENT_BINARY_DIR}/${bash_test_modules_START_BASH}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  endif()
+
+  if(bash_test_modules_SERIAL)
+    set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
+  endif()
+
+  if(bash_test_modules_LABELS)
+    set_tests_properties(${TARGET_NAME} PROPERTIES LABELS
+                                                   ${bash_test_modules_LABELS})
+  endif()
+endfunction()
+
 if(WITH_TESTING)
   # add_subdirectory(asp)
   # add_subdirectory(auto_parallel)
   # add_subdirectory(autograd)
   add_subdirectory(book)
-  # add_subdirectory(collective)
+  add_subdirectory(collective)
   # add_subdirectory(composite_ops)
   add_subdirectory(contrib)
   add_subdirectory(cpp)
diff --git a/test/collective/CMakeLists.txt b/test/collective/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1951a4fd0a25f2cd156e929922ca467f201b2275
--- /dev/null
+++ b/test/collective/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(multinode)
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/CMakeLists.txt b/test/collective/multinode/CMakeLists.txt
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/CMakeLists.txt
rename to test/collective/multinode/CMakeLists.txt
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/common.py b/test/collective/multinode/common.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/common.py
rename to test/collective/multinode/common.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py b/test/collective/multinode/dygraph_hybrid_dp.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py
rename to test/collective/multinode/dygraph_hybrid_dp.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py b/test/collective/multinode/dygraph_hybrid_dpppmp.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py
rename to test/collective/multinode/dygraph_hybrid_dpppmp.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py b/test/collective/multinode/dygraph_hybrid_fp16.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py
rename to test/collective/multinode/dygraph_hybrid_fp16.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py b/test/collective/multinode/dygraph_hybrid_recompute.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py
rename to test/collective/multinode/dygraph_hybrid_recompute.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py b/test/collective/multinode/mn_dygraph_group_sharded_stage3.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py
rename to test/collective/multinode/mn_dygraph_group_sharded_stage3.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/multinode_dist_test.sh b/test/collective/multinode/multinode_dist_test.sh
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/multinode_dist_test.sh
rename to test/collective/multinode/multinode_dist_test.sh
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py b/test/collective/multinode/test_collective_multi_nodes.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py
rename to test/collective/multinode/test_collective_multi_nodes.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py b/test/collective/multinode/test_multinode_dygraph_hybrid_dp.py
old mode 100755
new mode 100644
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py
rename to test/collective/multinode/test_multinode_dygraph_hybrid_dp.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py b/test/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py
old mode 100755
new mode 100644
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py
rename to test/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py b/test/collective/multinode/test_multinode_dygraph_sharding.py
old mode 100755
new mode 100644
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py
rename to test/collective/multinode/test_multinode_dygraph_sharding.py
diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/testslist.csv b/test/collective/multinode/testslist.csv
similarity index 100%
rename from python/paddle/fluid/tests/unittests/collective/multinode/testslist.csv
rename to test/collective/multinode/testslist.csv