disable some distribute test case when in CPU test env (#39682)

* disable some distribute test case when in CPU test env, test=develop

* refine, test=develop

* refine, test=develop

* refine, test=develop

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -9,7 +9,16 @@ list(REMOVE_ITEM DIST_TEST_OPS "test_dist_op")
 if ((NOT WITH_NCCL) AND (NOT WITH_RCCL))
     list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")
 endif()
+
 string(REPLACE ".py" "" DIST_TEST_OPS "${DIST_TEST_OPS}")
+
+if ((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
+    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_grad_clip")
+    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_heter_ctr")
+    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_ps_gpu_ctr")
+    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_batch_merge")
+endif()
+
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_mnist)
 list(APPEND DIST_TEST_OPS test_pipeline)
 list(APPEND DIST_TEST_OPS test_ir_pass_pipeline)
@@ -18,12 +27,14 @@ list(APPEND DIST_TEST_OPS test_parallel_dygraph_se_resnext)
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding)
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding_over_height)
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_transformer)
-list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer)
-list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer_with_recompute)
-list(APPEND DIST_TEST_OPS test_fleet_raw_program_meta_optimizer)
-list(APPEND DIST_TEST_OPS test_rnn_dp)
-list(APPEND DIST_TEST_OPS test_fleet_graph_execution_meta_optimizer)
-list(APPEND DIST_TEST_OPS test_gen_nccl_id_op)
+if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
+    list(APPEND DIST_TEST_OPS test_fleet_graph_execution_meta_optimizer)
+    list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer)
+    list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer_with_recompute)
+    list(APPEND DIST_TEST_OPS test_fleet_raw_program_meta_optimizer)
+    list(APPEND DIST_TEST_OPS test_gen_nccl_id_op)
+    list(APPEND DIST_TEST_OPS test_rnn_dp)
+endif()
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_unused_variables)
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_control_flow)
 list(APPEND DIST_TEST_OPS test_parallel_dygraph_no_sync)
@@ -564,6 +575,15 @@ foreach(TEST_OP ${TEST_OPS_WITH_GC})
   py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS ${GC_ENVS})
 endforeach()
 
+if ((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
+    list(REMOVE_ITEM TEST_OPS "test_fleet_graph_execution_meta_optimizer")
+    list(REMOVE_ITEM TEST_OPS "test_gen_nccl_id_op")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_grad_clip")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_heter_ctr")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_ps_gpu_ctr")
+    list(REMOVE_ITEM TEST_OPS "test_dist_mnist_batch_merge")
+endif()
+
 foreach(TEST_OP ${TEST_OPS})
     py_test_modules(${TEST_OP} MODULES ${TEST_OP})
 endforeach(TEST_OP)
@@ -638,12 +658,6 @@ if(WITH_DISTRIBUTE)
         py_test_modules(test_fleet_base MODULES test_fleet_base ENVS ${dist_ENVS})
         py_test_modules(test_fleet_base_2 MODULES test_fleet_base_2 ENVS ${dist_ENVS})
         py_test_modules(test_fleet_base_3 MODULES test_fleet_base_3 ENVS ${dist_ENVS})
-    	   py_test_modules(test_fleet_recompute_meta_optimizer MODULES test_fleet_recompute_meta_optimizer ENVS ${dist_ENVS})
-	       py_test_modules(test_fleet_graph_executor MODULES test_fleet_graph_executor ENVS ${dist_ENVS})
-           py_test_modules(test_fleet_gradient_merge_meta_optimizer MODULES test_fleet_gradient_merge_meta_optimizer ENVS ${dist_ENVS})
-           py_test_modules(test_fleet_sharding_meta_optimizer MODULES test_fleet_sharding_meta_optimizer ENVS ${dist_ENVS})
-           py_test_modules(test_fleet_hybrid_meta_optimizer MODULES test_fleet_hybrid_meta_optimizer ENVS ${dist_ENVS})
-           py_test_modules(test_fleet_amp_meta_optimizer MODULES test_fleet_amp_meta_optimizer ENVS ${dist_ENVS})
         py_test_modules(test_fleet_amp_init MODULES test_fleet_amp_init ENVS ${dist_ENVS})
         py_test_modules(test_fleet_fp16_allreduce_meta_optimizer MODULES test_fleet_fp16_allreduce_meta_optimizer ENVS ${dist_ENVS})
         py_test_modules(test_fleet_private_function MODULES test_fleet_private_function ENVS ${dist_ENVS})
@@ -651,10 +665,15 @@ if(WITH_DISTRIBUTE)
         py_test_modules(test_fleet_distributed_strategy MODULES test_fleet_distributed_strategy)
         py_test_modules(test_fleet_static_mp_layers MODULES test_fleet_static_mp_layers)
         #py_test_modules(test_fleet_auto MODULES test_fleet_auto ENVS ${dist_ENVS})
+        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
+            py_test_modules(test_fleet_amp_meta_optimizer MODULES test_fleet_amp_meta_optimizer ENVS ${dist_ENVS})
+            py_test_modules(test_fleet_gradient_merge_meta_optimizer MODULES test_fleet_gradient_merge_meta_optimizer ENVS ${dist_ENVS})
+            py_test_modules(test_fleet_graph_executor MODULES test_fleet_graph_executor ENVS ${dist_ENVS})
+            py_test_modules(test_fleet_hybrid_meta_optimizer MODULES test_fleet_hybrid_meta_optimizer ENVS ${dist_ENVS})
+            py_test_modules(test_fleet_recompute_meta_optimizer MODULES test_fleet_recompute_meta_optimizer ENVS ${dist_ENVS})
+            py_test_modules(test_fleet_sharding_meta_optimizer MODULES test_fleet_sharding_meta_optimizer ENVS ${dist_ENVS})
+        endif()
         if(NOT WIN32)
-            py_test_modules(test_fleet_localsgd_meta_optimizer MODULES test_fleet_localsgd_meta_optimizer ENVS ${dist_ENVS})
-            py_test_modules(test_fleet_lars_meta_optimizer MODULES test_fleet_lars_meta_optimizer ENVS ${dist_ENVS})
-            py_test_modules(test_fleet_lamb_meta_optimizer MODULES test_fleet_lamb_meta_optimizer ENVS ${dist_ENVS})
             py_test_modules(test_auto_parallel_partitioner MODULES test_auto_parallel_partitioner ENVS ${dist_ENVS})
             py_test_modules(test_auto_parallel_partitioner_gpt MODULES test_auto_parallel_partitioner_gpt ENVS ${dist_ENVS})
             py_test_modules(test_auto_parallel_searcher MODULES test_auto_parallel_searcher ENVS ${dist_ENVS})
@@ -664,6 +683,14 @@ if(WITH_DISTRIBUTE)
             py_test_modules(test_auto_parallel_reshard_mppp MODULES test_auto_parallel_reshard_mppp ENVS ${dist_ENVS})
             py_test_modules(test_auto_parallel_reshard_dpmppp MODULES test_auto_parallel_reshard_dpmppp ENVS ${dist_ENVS})
             py_test_modules(test_auto_parallel_cost_model MODULES test_auto_parallel_cost_model ENVS ${dist_ENVS})
+            if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
+                py_test_modules(test_fleet_lamb_meta_optimizer MODULES test_fleet_lamb_meta_optimizer ENVS ${dist_ENVS})
+                py_test_modules(test_fleet_lars_meta_optimizer MODULES test_fleet_lars_meta_optimizer ENVS ${dist_ENVS})
+                py_test_modules(test_fleet_localsgd_meta_optimizer MODULES test_fleet_localsgd_meta_optimizer ENVS ${dist_ENVS})
+
+
+
+            endif()
         endif(NOT WIN32)
     endif(NOT APPLE)
     if(WITH_DGC)
@@ -688,7 +715,9 @@ if(WITH_DISTRIBUTE)
         bash_test_modules(test_fleet_run_random_port START_BASH test_fleet_run_random_port.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
         bash_test_modules(test_fleet_launch_async START_BASH test_fleet_launch_async.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
         bash_test_modules(test_fleet_launch_cloud START_BASH test_fleet_launch_cloud.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
+        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
             bash_test_modules(test_fleet_launch_nproc START_BASH test_fleet_launch_nproc.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
+        endif()
         if(WITH_ASCEND OR WITH_ASCEND_CL)
             bash_test_modules(test_fleet_launch_ascend START_BASH test_fleet_launch_ascend.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
             bash_test_modules(test_ascend_group START_BASH test_ascend_group.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
@@ -708,7 +737,9 @@ if(WITH_DISTRIBUTE)
         if (WITH_GLOO)
             bash_test_modules(test_cpuonly_launch START_BASH test_cpuonly_launch.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}" PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} )
         endif()
+        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
             bash_test_modules(test_new_group START_BASH test_new_group.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}+20" PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} )
+        endif()
     endif(NOT APPLE)
 endif()
 

python/paddle/fluid/tests/unittests/asp/CMakeLists.txt

@@ -11,9 +11,13 @@ endforeach(TEST_OP)
 
 if(WITH_DISTRIBUTE)
     py_test_modules(test_fleet_with_asp MODULES test_fleet_with_asp ENVS ${dist_ENVS})
+    if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
         py_test_modules(test_fleet_with_asp_amp MODULES test_fleet_with_asp_amp ENVS ${dist_ENVS})
+    endif()
 endif()
 
 if((WITH_DISTRIBUTE) AND (NOT WIN32) AND (NOT APPLE))
+    if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
         py_test_modules(test_fleet_with_asp_sharding MODULES test_fleet_with_asp_sharding ENVS ${dist_ENVS})
+    endif()
 endif()

python/paddle/fluid/tests/unittests/distributed_passes/CMakeLists.txt

 file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
 string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
 
+if ((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_adam_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_all_reduce_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_bn_act_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_bn_add_act_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_momentum_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_relu_depthwise_conv_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_fuse_sgd_pass")
+    list(REMOVE_ITEM TEST_OPS "test_dist_gradient_merge_pass")
+endif()
+
 foreach(TEST_OP ${TEST_OPS})
     py_test_modules(${TEST_OP} MODULES ${TEST_OP})
     list(APPEND DIST_TEST_OPS ${TEST_OP})