CMakeLists.txt 64.0 KB
Newer Older
1 2
file(GLOB TEST_OPS RELATIVE
"${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
3
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
4
set(GC_ENVS FLAGS_eager_delete_tensor_gb=0.0 FLAGS_fast_eager_deletion_mode=1 FLAGS_memory_fraction_of_eager_deletion=1.0)
5
set(dist_ENVS http_proxy="" https_proxy="")
6

7
file(GLOB DIST_TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_dist_*.py")
8
list(REMOVE_ITEM DIST_TEST_OPS "test_dist_op")
9
if ((NOT WITH_NCCL) AND (NOT WITH_RCCL))
10 11
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")
endif()
12

13
string(REPLACE ".py" "" DIST_TEST_OPS "${DIST_TEST_OPS}")
14 15 16 17 18 19 20 21

if ((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_grad_clip")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_heter_ctr")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_ps_gpu_ctr")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_batch_merge")
endif()

22
list(APPEND DIST_TEST_OPS test_parallel_dygraph_mnist)
23
list(APPEND DIST_TEST_OPS test_pipeline)
24
list(APPEND DIST_TEST_OPS test_ir_pass_pipeline)
L
lilong12 已提交
25
list(APPEND DIST_TEST_OPS test_static_model_parallel)
26
list(APPEND DIST_TEST_OPS test_parallel_dygraph_se_resnext)
27
list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding)
28
list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding_over_height)
29
list(APPEND DIST_TEST_OPS test_parallel_dygraph_transformer)
30 31 32 33 34 35 36 37
if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
    list(APPEND DIST_TEST_OPS test_fleet_graph_execution_meta_optimizer)
    list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer)
    list(APPEND DIST_TEST_OPS test_fleet_pipeline_meta_optimizer_with_recompute)
    list(APPEND DIST_TEST_OPS test_fleet_raw_program_meta_optimizer)
    list(APPEND DIST_TEST_OPS test_gen_nccl_id_op)
    list(APPEND DIST_TEST_OPS test_rnn_dp)
endif()
38
list(APPEND DIST_TEST_OPS test_parallel_dygraph_unused_variables)
39
list(APPEND DIST_TEST_OPS test_parallel_dygraph_control_flow)
40 41
list(APPEND DIST_TEST_OPS test_parallel_dygraph_no_sync)
list(APPEND DIST_TEST_OPS test_parallel_dygraph_no_sync_gradient_check)
42
list(APPEND DIST_TEST_OPS test_parallel_dygraph_dataparallel)
43 44
list(APPEND DIST_TEST_OPS test_parallel_dygraph_pipeline_parallel)
list(APPEND DIST_TEST_OPS test_parallel_dygraph_tensor_parallel)
J
JZ-LIANG 已提交
45
list(APPEND DIST_TEST_OPS test_parallel_dygraph_sharding_parallel)
46
list(APPEND DIST_TEST_OPS test_dygraph_sharding_optimizer_stage2)
B
Baibaifan 已提交
47
list(APPEND DIST_TEST_OPS test_dygraph_sharding_stage2)
B
Baibaifan 已提交
48
list(APPEND DIST_TEST_OPS test_dygraph_sharding_stage3)
49
list(APPEND DIST_TEST_OPS test_auto_parallel_parallelizer)
50
list(APPEND DIST_TEST_OPS test_parallel_dygraph_mp_layers)
51
list(APPEND DIST_TEST_OPS test_hybrid_parallel_inference_helper)
52
list(APPEND DIST_TEST_OPS test_parallel_class_center_sample)
53
list(APPEND DIST_TEST_OPS test_parallel_margin_cross_entropy)
54
list(APPEND DIST_TEST_OPS test_auto_parallel_data_unshard)
55
list(APPEND DIST_TEST_OPS test_auto_parallel_save_load)
56
list(APPEND DIST_TEST_OPS test_auto_parallel_autoconvert)
57
list(APPEND DIST_TEST_OPS test_collective_process_group)
58 59 60
set(MIXED_DIST_TEST_OPS ${DIST_TEST_OPS})
#remove distribute unittests.
list(APPEND MIXED_DIST_TEST_OPS test_dgc_op)
61 62
list(APPEND MIXED_DIST_TEST_OPS test_dgc_momentum_op)
list(APPEND MIXED_DIST_TEST_OPS test_dgc_optimizer)
63
list(APPEND MIXED_DIST_TEST_OPS test_simple_dist_transpiler)
T
tangwei12 已提交
64
list(APPEND MIXED_DIST_TEST_OPS test_recv_save_op)
65
list(APPEND MIXED_DIST_TEST_OPS test_c_comm_init_op)
66
list(APPEND MIXED_DIST_TEST_OPS test_communicator_async)
T
Thunderbrook 已提交
67
list(APPEND MIXED_DIST_TEST_OPS test_communicator_ps_gpu)
68 69
list(APPEND MIXED_DIST_TEST_OPS test_communicator_geo)
list(APPEND MIXED_DIST_TEST_OPS test_communicator_half_async)
T
tangwei12 已提交
70
list(APPEND MIXED_DIST_TEST_OPS test_communicator_sync)
71 72 73 74 75 76
list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_ps)
list(APPEND MIXED_DIST_TEST_OPS test_launch_coverage)
list(APPEND MIXED_DIST_TEST_OPS test_fleetrun)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_run_random_port)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_async)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_cloud)
77 78
list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_ascend)
list(APPEND MIXED_DIST_TEST_OPS test_ascend_group)
79
list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_nproc)
80
list(APPEND MIXED_DIST_TEST_OPS test_fleet_api_input)
M
mapingshuo 已提交
81
list(APPEND MIXED_DIST_TEST_OPS test_collective_optimizer)
82
list(APPEND MIXED_DIST_TEST_OPS test_fleet_base)
83 84
list(APPEND MIXED_DIST_TEST_OPS test_fleet_base_2)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_base_3)
85
list(APPEND MIXED_DIST_TEST_OPS test_fleet_recompute_meta_optimizer)
86
list(APPEND MIXED_DIST_TEST_OPS test_fleet_pipeline_meta_optimizer)
87
list(APPEND MIXED_DIST_TEST_OPS test_fleet_pipeline_meta_optimizer_with_recompute)
88
list(APPEND MIXED_DIST_TEST_OPS test_fleet_raw_program_meta_optimizer)
李季 已提交
89
list(APPEND MIXED_DIST_TEST_OPS test_rnn_dp)
90
list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_meta_optimizer)
91
list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_init)
92
list(APPEND MIXED_DIST_TEST_OPS test_fleet_gradient_merge_meta_optimizer)
93
list(APPEND MIXED_DIST_TEST_OPS test_fleet_sharding_meta_optimizer)
94
list(APPEND MIXED_DIST_TEST_OPS test_fleet_hybrid_meta_optimizer)
Y
Yi Liu 已提交
95
list(APPEND MIXED_DIST_TEST_OPS test_fleet_localsgd_meta_optimizer)
96
list(APPEND MIXED_DIST_TEST_OPS test_fleet_lars_meta_optimizer)
97
list(APPEND MIXED_DIST_TEST_OPS test_fleet_lamb_meta_optimizer)
98
list(APPEND MIXED_DIST_TEST_OPS test_fleet_dgc_meta_optimizer)
99
list(APPEND MIXED_DIST_TEST_OPS test_fleet_fp16_allreduce_meta_optimizer)
100
list(APPEND MIXED_DIST_TEST_OPS test_fleet_private_function)
101
list(APPEND MIXED_DIST_TEST_OPS test_fleet_graph_executor)
102
list(APPEND MIXED_DIST_TEST_OPS test_fleet_meta_optimizer_base)
103
list(APPEND MIXED_DIST_TEST_OPS test_fleet_distributed_strategy)
104
list(APPEND MIXED_DIST_TEST_OPS test_fleet_auto)
W
WangXi 已提交
105
list(APPEND MIXED_DIST_TEST_OPS test_fleet_static_mp_layers)
106 107
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_partitioner)
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_partitioner_gpt)
C
caozhou 已提交
108
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_searcher)
C
caozhou 已提交
109
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_reshard)
110
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_dist_tensor)
C
caozhou 已提交
111 112 113
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_reshard_serial)
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_reshard_mppp)
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_reshard_dpmppp)
114
list(APPEND MIXED_DIST_TEST_OPS test_auto_parallel_cost_model)
115 116 117
foreach(TEST_OP ${MIXED_DIST_TEST_OPS})
  list(REMOVE_ITEM TEST_OPS ${TEST_OP})
endforeach()
118

119 120 121 122
if(ON_INFER)
    LIST(REMOVE_ITEM TEST_OPS test_eager_trace_op)
endif()

L
Li Min 已提交
123
if(NOT WITH_GPU)
124
    LIST(REMOVE_ITEM TEST_OPS test_fused_feedforward_op)
L
Li Min 已提交
125
    LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op)
126
    LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op_api)
127
    LIST(REMOVE_ITEM TEST_OPS test_fused_transformer_encoder_layer)
L
Li Min 已提交
128 129
endif()

130
if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32)
131
    LIST(REMOVE_ITEM TEST_OPS test_c_comm_init_all_op)
L
lilong12 已提交
132 133
    LIST(REMOVE_ITEM TEST_OPS test_c_concat)
    LIST(REMOVE_ITEM TEST_OPS test_c_split)
134
    LIST(REMOVE_ITEM TEST_OPS test_allgather)
L
lilong12 已提交
135
    LIST(REMOVE_ITEM TEST_OPS test_c_identity)
136
    LIST(REMOVE_ITEM TEST_OPS test_c_embedding_op)
137 138
    LIST(REMOVE_ITEM TEST_OPS test_allreduce)
    LIST(REMOVE_ITEM TEST_OPS test_broadcast)
L
lilong12 已提交
139
    LIST(REMOVE_ITEM TEST_OPS test_collective_reduce)
140
    LIST(REMOVE_ITEM TEST_OPS test_pipeline_parallel)
L
lilong12 已提交
141
    LIST(REMOVE_ITEM TEST_OPS test_collective_scatter)
L
lilong12 已提交
142
    LIST(REMOVE_ITEM TEST_OPS test_collective_sendrecv)
143
    LIST(REMOVE_ITEM TEST_OPS test_reducescatter)
144
    LIST(REMOVE_ITEM TEST_OPS test_reducescatter_api)
145 146 147 148
    LIST(REMOVE_ITEM TEST_OPS test_collective_split_embedding)
    LIST(REMOVE_ITEM TEST_OPS test_collective_split_embedding_none_divisible)
    LIST(REMOVE_ITEM TEST_OPS test_collective_split_row_linear)
    LIST(REMOVE_ITEM TEST_OPS test_collective_split_col_linear)
L
lilong12 已提交
149 150 151 152
    LIST(REMOVE_ITEM TEST_OPS test_collective_reduce_api)
    LIST(REMOVE_ITEM TEST_OPS test_collective_scatter_api)
    LIST(REMOVE_ITEM TEST_OPS test_collective_barrier_api)
    LIST(REMOVE_ITEM TEST_OPS test_collective_allreduce_api)
K
kuizhiqing 已提交
153
    LIST(REMOVE_ITEM TEST_OPS test_new_group_api)
L
lilong12 已提交
154 155
    LIST(REMOVE_ITEM TEST_OPS test_collective_broadcast_api)
    LIST(REMOVE_ITEM TEST_OPS test_collective_allgather_api)
L
lilong12 已提交
156
    LIST(REMOVE_ITEM TEST_OPS test_collective_alltoall_api)
157 158
    LIST(REMOVE_ITEM TEST_OPS test_collective_global_gather)
    LIST(REMOVE_ITEM TEST_OPS test_collective_global_scatter)
L
lilong12 已提交
159
    LIST(REMOVE_ITEM TEST_OPS test_collective_sendrecv_api)
W
WangXi 已提交
160
    LIST(REMOVE_ITEM TEST_OPS test_collective_wait)
J
JZ-LIANG 已提交
161
    LIST(REMOVE_ITEM TEST_OPS test_memcpy_op)
李季 已提交
162
    LIST(REMOVE_ITEM TEST_OPS test_raw_program_optimizer)
Y
Yuang Liu 已提交
163
    LIST(REMOVE_ITEM TEST_OPS test_fleet_gradient_scale)
164
    LIST(REMOVE_ITEM TEST_OPS test_disable_signal_handler)
L
LiYuRio 已提交
165
    LIST(REMOVE_ITEM TEST_OPS test_fleet_executor)
166
    LIST(REMOVE_ITEM TEST_OPS test_fleet_executor_with_task_nodes)
167
    LIST(REMOVE_ITEM TEST_OPS test_fleet_executor_multi_devices)
168
    LIST(REMOVE_ITEM TEST_OPS test_fleet_executor_origin_scheduler)
169
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_mapper)
L
LiYuRio 已提交
170
    LIST(REMOVE_ITEM TEST_OPS test_fleet_executor_task_node)
171 172
    LIST(REMOVE_ITEM TEST_OPS test_fleet_exe_dist_model_run)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_exe_dist_model_tensor)
173 174
endif()

175 176 177
# Temporally disable test_deprecated_decorator
LIST(REMOVE_ITEM TEST_OPS test_deprecated_decorator)

178 179 180 181
if(WIN32)
    LIST(REMOVE_ITEM TEST_OPS test_multiprocess_reader_exception)
    LIST(REMOVE_ITEM TEST_OPS test_trainer_desc)
    LIST(REMOVE_ITEM TEST_OPS test_checkpoint_notify_op)
182 183 184 185 186 187 188
    LIST(REMOVE_ITEM TEST_OPS test_downpoursgd)
    LIST(REMOVE_ITEM TEST_OPS test_fleet)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_nocvm_1)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_rolemaker)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_rolemaker_3)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_unitaccessor)
    LIST(REMOVE_ITEM TEST_OPS test_ps_dispatcher)
189 190
    LIST(REMOVE_ITEM TEST_OPS test_ir_memory_optimize_nlp)
    LIST(REMOVE_ITEM TEST_OPS test_nvprof)
191

192 193
    # TODO: Fix these unittests failed on Windows
    LIST(REMOVE_ITEM TEST_OPS test_debugger)
H
huangxu96 已提交
194 195 196
    if (WITH_GPU)
        LIST(REMOVE_ITEM TEST_OPS test_update_loss_scaling_op)
    endif()
197 198 199 200 201 202 203 204 205 206
endif()

if(NOT WITH_DISTRIBUTE OR WIN32)
    # DISTRIBUTE related
    LIST(REMOVE_ITEM TEST_OPS test_avoid_twice_initialization)
    LIST(REMOVE_ITEM TEST_OPS test_distributed_strategy)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_metric)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_ps)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_rolemaker_2)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_utils)
207
    LIST(REMOVE_ITEM TEST_OPS test_collective_cpu_barrier_with_gloo)
208 209

    # TODO: Fix these unittests failed on Windows
210
    list(REMOVE_ITEM TEST_OPS test_fake_init_op)
H
hutuxian 已提交
211 212
endif()

213 214 215 216 217
if(NOT WITH_DISTRIBUTE)
    LIST(REMOVE_ITEM TEST_OPS test_fleet_rolemaker_new)
    LIST(REMOVE_ITEM TEST_OPS test_desc_clone_dist)
endif()

218 219 220 221
if(WIN32)
    LIST(REMOVE_ITEM TEST_OPS test_complex_matmul)
endif()

G
gongweibao 已提交
222
LIST(REMOVE_ITEM TEST_OPS test_fleet_checkpoint)
223
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint)
224
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint1)
225
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint2)
226 227 228 229 230 231
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint3)
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint_multiple)
LIST(REMOVE_ITEM TEST_OPS test_auto_checkpoint_dist_basic)
LIST(REMOVE_ITEM TEST_OPS test_hdfs1)
LIST(REMOVE_ITEM TEST_OPS test_hdfs2)
LIST(REMOVE_ITEM TEST_OPS test_hdfs3)
232
LIST(REMOVE_ITEM TEST_OPS test_checkpoint_saver)
T
tangwei12 已提交
233

G
gongweibao 已提交
234 235
if(APPLE OR WIN32)
    LIST(REMOVE_ITEM TEST_OPS test_fs_interface)
236
    LIST(REMOVE_ITEM TEST_OPS test_fleet_metric)
G
gongweibao 已提交
237 238
endif()

S
ShenLiang 已提交
239 240
list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_hybrid_parallel)

X
xiongkun 已提交
241 242
LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_transformer_gloo) # NOTE: @xiongkun03, cpu is too slow, fix it in next PR

243 244
if (NOT WITH_GLOO)
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_dataparallel_cpuonly)
X
xiongkun 已提交
245 246 247 248

    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_unused_variables_gloo)
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sparse_embedding_over_height_gloo)
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sparse_embedding_gloo)
249
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sparse_embedding_diff_length_gloo)
250 251
endif()

252
if ((NOT WITH_GPU) AND (NOT WITH_ROCM))
Y
Yu Yang 已提交
253
    LIST(REMOVE_ITEM TEST_OPS test_conv2d_fusion_op)
S
ShenLiang 已提交
254
    LIST(REMOVE_ITEM TEST_OPS test_rank_attention_op) # TODO(shenliang03): rank_attention_op support CPU device in future
S
ShenLiang 已提交
255
    LIST(REMOVE_ITEM TEST_OPS test_batch_fc_op) # TODO(shenliang03): batch_fc_op support CPU device in future
256
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_mnist) # TODO(Yancey1989): parallel dygraph support CPU device in future
257
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_unused_variables)
258 259
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_se_resnext)
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sparse_embedding)
260
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sparse_embedding_over_height)
261
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_transformer)
C
ceci3 已提交
262
    LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sync_batch_norm)
263
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_control_flow)
264 265
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_no_sync)
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_no_sync_gradient_check)
266
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_dataparallel)
267 268
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_pipeline_parallel)
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_tensor_parallel)
J
JZ-LIANG 已提交
269
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sharding_parallel)
270
    list(REMOVE_ITEM TEST_OPS test_dygraph_sharding_optimizer_stage2)
B
Baibaifan 已提交
271
    list(REMOVE_ITEM TEST_OPS test_dygraph_sharding_stage2)
B
Baibaifan 已提交
272
    list(REMOVE_ITEM TEST_OPS test_dygraph_sharding_stage3)
273
    list(REMOVE_ITEM TEST_OPS test_auto_parallel_parallelizer)
274
    list(REMOVE_ITEM TEST_OPS test_parallel_dygraph_mp_layers)
275
    LIST(REMOVE_ITEM TEST_OPS test_imperative_auto_mixed_precision)
276
    LIST(REMOVE_ITEM TEST_OPS test_mixed_precision)
277
    LIST(REMOVE_ITEM TEST_OPS test_fleet_base_single)
J
JZ-LIANG 已提交
278
    LIST(REMOVE_ITEM TEST_OPS test_dygraph_recompute)
279
    list(REMOVE_ITEM TEST_OPS test_hybrid_parallel_inference_helper)
280
    list(REMOVE_ITEM TEST_OPS test_parallel_class_center_sample)
281
    LIST(REMOVE_ITEM TEST_OPS test_parallel_margin_cross_entropy)
282 283
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_partitioner)
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_partitioner_gpt)
C
caozhou 已提交
284
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_searcher)
C
caozhou 已提交
285
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_reshard)
286
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_dist_tensor)
C
caozhou 已提交
287 288 289
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_reshard_serial)
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_reshard_mppp)
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_reshard_dpmppp)
290
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_cost_model)
291
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_data_unshard)
292
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_save_load)
293
    LIST(REMOVE_ITEM TEST_OPS test_auto_parallel_autoconvert)
294
    LIST(REMOVE_ITEM TEST_OPS test_collective_process_group)
295 296 297 298
elseif(WITH_GPU)
    if (${CUDNN_VERSION} VERSION_LESS 7100)
        LIST(REMOVE_ITEM TEST_OPS test_conv2d_fusion_op)
    endif()
D
Dang Qingqing 已提交
299 300
endif()

301 302 303
if (WITH_NCCL)
    if (${NCCL_VERSION} VERSION_LESS 2212)
        LIST(REMOVE_ITEM DIST_TEST_OPS test_parallel_dygraph_sparse_embedding)
304
        LIST(REMOVE_ITEM DIST_TEST_OPS test_parallel_dygraph_sparse_embedding_over_height)
305 306 307 308
        LIST(REMOVE_ITEM DIST_TEST_OPS test_parallel_dygraph_transformer)
    endif()
endif()

309
if ((NOT WITH_NCCL) AND (NOT WITH_RCCL))
310
    list(REMOVE_ITEM TEST_OPS test_imperative_group)
K
kuizhiqing 已提交
311
    LIST(REMOVE_ITEM TEST_OPS test_new_group_api)
312 313
endif()

314
if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32)
H
hutuxian 已提交
315
    LIST(REMOVE_ITEM TEST_OPS test_boxps)
316
endif()
317 318 319
list(REMOVE_ITEM TEST_OPS test_seq_concat_op) # FIXME(helin): https://github.com/PaddlePaddle/Paddle/issues/8290
list(REMOVE_ITEM TEST_OPS test_lstm_unit_op) # # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5185
list(REMOVE_ITEM TEST_OPS test_cond_op) # FIXME(qijun): https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957
H
Hongyu Liu 已提交
320

321
list(REMOVE_ITEM TEST_OPS op_test) # op_test is a helper python file, not a test
322
list(REMOVE_ITEM TEST_OPS decorator_helper) # decorator_helper is a helper python file, not a test
323

324
if(APPLE)
325 326 327 328
    if(NOT WITH_DISTRIBUTE)
        list(REMOVE_ITEM TEST_OPS test_desc_clone)
        list(REMOVE_ITEM TEST_OPS test_program_code)
    endif(NOT WITH_DISTRIBUTE)
329
    message(WARNING "These tests has been disabled in OSX before being fixed:\n test_fuse_elewise_add_act_pass \n test_detection_map_op \n test_dist_se_resnext_*")
330 331
    # this op is not support on mac
    list(REMOVE_ITEM TEST_OPS test_fusion_seqexpand_concat_fc_op)
332
    list(REMOVE_ITEM TEST_OPS test_detection_map_op)
333
    list(REMOVE_ITEM TEST_OPS test_fuse_elewise_add_act_pass)
334
endif()
T
Tao Luo 已提交
335 336 337 338
if(NOT WITH_MKLML)
    # this op is not support on openblas
    list(REMOVE_ITEM TEST_OPS test_fusion_seqexpand_concat_fc_op)
endif()
339

340
if(NOT WITH_MKL OR NOT WITH_AVX)
A
Aurelius84 已提交
341
  list(REMOVE_ITEM TEST_OPS test_match_matrix_tensor_op)
K
Kevin 已提交
342
  list(REMOVE_ITEM TEST_OPS test_var_conv_2d)
343
endif()
344

345
if(WITH_COVERAGE OR WIN32 OR WITH_NV_JETSON)
A
Aurelius84 已提交
346
  list(REMOVE_ITEM TEST_OPS test_pyramid_hash_op)
347 348
endif()

T
tangwei12 已提交
349
list(REMOVE_ITEM TEST_OPS test_fleet_pyramid_hash)
K
Kevin 已提交
350

351
if((WITH_ROCM OR WITH_GPU) OR NOT WITH_MKLML)
352 353 354 355
    # matmul with multiple heads need MKL support
    LIST(REMOVE_ITEM TEST_OPS test_matmul_op_with_head)
endif()

Y
Yanghello 已提交
356 357 358 359
if(NOT WITH_CRYPTO)
    LIST(REMOVE_ITEM TEST_OPS test_crypto)
endif()

360 361
function(py_test_modules TARGET_NAME)
  if(WITH_TESTING)
362
    set(options SERIAL)
363
    set(oneValueArgs "")
364
    set(multiValueArgs MODULES DEPS ENVS)
365
    cmake_parse_arguments(py_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
366

367
    if(WITH_COVERAGE AND NOT (WITH_INCREMENTAL_COVERAGE AND "$ENV{PADDLE_GIT_DIFF_PY_FILE}" STREQUAL ""))
368 369 370 371 372 373 374 375 376 377 378 379 380
        if(WITH_ASCEND_CL)
            add_test(NAME ${TARGET_NAME}
                    COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python:$ENV{PYTHONPATH} ${py_test_modules_ENVS}
                    COVERAGE_FILE=${PADDLE_BINARY_DIR}/python-coverage.data
                    ${PYTHON_EXECUTABLE} -m coverage run --branch -p ${PADDLE_SOURCE_DIR}/tools/test_runner.py ${py_test_modules_MODULES}
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
        else()
            add_test(NAME ${TARGET_NAME}
                    COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_modules_ENVS}
                    COVERAGE_FILE=${PADDLE_BINARY_DIR}/python-coverage.data
                    ${PYTHON_EXECUTABLE} -m coverage run --branch -p ${PADDLE_SOURCE_DIR}/tools/test_runner.py ${py_test_modules_MODULES}
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
        endif()
381
    else()
382 383 384 385 386 387 388 389 390 391 392
        if(WITH_ASCEND_CL)
            add_test(NAME ${TARGET_NAME}
                    COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python:$ENV{PYTHONPATH} ${py_test_modules_ENVS}
                    ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/tools/test_runner.py ${py_test_modules_MODULES}
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
        else()
            add_test(NAME ${TARGET_NAME}
                    COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_modules_ENVS}
                    ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/tools/test_runner.py ${py_test_modules_MODULES}
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
        endif()
393 394 395
    endif()

    if (py_test_modules_SERIAL)
396
        set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
397
    endif()
398 399 400
    if(WIN32)
        set_tests_properties(${TARGET_NAME} PROPERTIES TIMEOUT 150)
    endif()
401 402
  endif()
endfunction()
G
gongweibao 已提交
403

404

G
gongweibao 已提交
405 406 407 408 409 410
function(bash_test_modules TARGET_NAME)
    if(NOT WITH_TESTING)
        return()
    endif()

    set(options SERIAL)
411 412
    set(oneValueArgs TIMEOUT START_BASH)
    set(multiValueArgs DEPS ENVS LABELS)
G
gongweibao 已提交
413 414
    cmake_parse_arguments(bash_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})

415

416 417 418 419
    set(timeout 350)
    if(${bash_test_modules_TIMEOUT})
        set(timeout ${bash_test_modules_TIMEOUT})
    endif()
G
gongweibao 已提交
420

421 422
    if(WITH_COVERAGE)
        add_test(NAME ${TARGET_NAME}
423
            COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
424 425
            TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout} ${bash_test_modules_ENVS}
            WITH_COVERAGE=ON COVERAGE_FILE=${PADDLE_BINARY_DIR}/python-coverage.data
426
            bash ${CMAKE_CURRENT_BINARY_DIR}/${bash_test_modules_START_BASH}
427 428 429
            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
    else()
        add_test(NAME ${TARGET_NAME}
430
            COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
431
            TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout} ${bash_test_modules_ENVS}
432
            bash ${CMAKE_CURRENT_BINARY_DIR}/${bash_test_modules_START_BASH}
433 434
            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
    endif()
435

G
gongweibao 已提交
436 437 438
    if (bash_test_modules_SERIAL)
        set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
    endif()
439 440

    if(bash_test_modules_LABELS)
441
        set_tests_properties(${TARGET_NAME} PROPERTIES LABELS ${bash_test_modules_LABELS})
442
    endif()
G
gongweibao 已提交
443 444
endfunction()

445 446 447 448 449 450 451 452 453 454 455
function(parallel_bash_test_modules TARGET_NAME)
    if(NOT WITH_TESTING)
        return()
    endif()

    set(options SERIAL)
    set(oneValueArgs TIMEOUT START_BASH)
    set(multiValueArgs DEPS ENVS LABELS UnitTests)
    cmake_parse_arguments(parallel_bash_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})


Y
YUNSHEN XIE 已提交
456
    set(timeout 120)
457 458 459 460 461 462 463 464
    if(${parallel_bash_test_modules_TIMEOUT})
        set(timeout ${parallel_bash_test_modules_TIMEOUT})
    endif()

    list(JOIN  parallel_bash_test_modules_UnitTests " " uts_string)

    if(WITH_COVERAGE)
        add_test(NAME ${TARGET_NAME}
465
            COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
466 467 468 469 470 471
            TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout} ${parallel_bash_test_modules_ENVS} UnitTests=${uts_string}
            WITH_COVERAGE=ON COVERAGE_FILE=${PADDLE_BINARY_DIR}/python-coverage.data
            bash ${CMAKE_CURRENT_BINARY_DIR}/${parallel_bash_test_modules_START_BASH}
            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
    else()
        add_test(NAME ${TARGET_NAME}
472
            COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python
473 474 475 476 477 478 479 480 481 482
            TEST_TARGET_NAME=${TARGET_NAME} TEST_TIMEOUT=${timeout} ${parallel_bash_test_modules_ENVS} UnitTests=${uts_string}
            bash ${CMAKE_CURRENT_BINARY_DIR}/${parallel_bash_test_modules_START_BASH}
            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
    endif()

    if (parallel_bash_test_modules_SERIAL)
        set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
    endif()

    if(parallel_bash_test_modules_LABELS)
483
        set_tests_properties(${TARGET_NAME} PROPERTIES LABELS ${parallel_bash_test_modules_LABELS})
484 485 486
    endif()
endfunction()

H
Huihuang Zheng 已提交
487
list(REMOVE_ITEM TEST_OPS test_feed_data_check_shape_type)
488
list(REMOVE_ITEM TEST_OPS test_fetch_lod_tensor_array)
489
list(REMOVE_ITEM TEST_OPS test_warpctc_op)
Y
yuyang18 已提交
490
list(REMOVE_ITEM TEST_OPS test_parallel_executor_crf)
491
list(REMOVE_ITEM TEST_OPS test_parallel_executor_profiler)
H
hutuxian 已提交
492
list(REMOVE_ITEM TEST_OPS test_data_norm_op)
493
list(REMOVE_ITEM TEST_OPS test_parallel_executor_fetch_feed)
494
list(REMOVE_ITEM TEST_OPS test_parallel_executor_transformer)
495
list(REMOVE_ITEM TEST_OPS test_parallel_executor_transformer_auto_growth)
D
dengkaipeng 已提交
496 497
list(REMOVE_ITEM TEST_OPS test_bilinear_interp_op)
list(REMOVE_ITEM TEST_OPS test_nearest_interp_op)
498
list(REMOVE_ITEM TEST_OPS test_imperative_resnet)
499 500
list(REMOVE_ITEM TEST_OPS test_imperative_resnet_sorted_gradient)
list(REMOVE_ITEM TEST_OPS test_imperative_mnist_sorted_gradient)
Y
Yan Xu 已提交
501
list(REMOVE_ITEM TEST_OPS test_imperative_se_resnext)
M
minqiyang 已提交
502
list(REMOVE_ITEM TEST_OPS test_imperative_mnist)
D
dzhwinter 已提交
503
list(REMOVE_ITEM TEST_OPS test_ir_memory_optimize_transformer)
L
lujun 已提交
504
list(REMOVE_ITEM TEST_OPS test_layers)
505 506 507
list(REMOVE_ITEM TEST_OPS test_parallel_executor_seresnext_base_cpu)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_seresnext_with_reduce_cpu)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_seresnext_with_fuse_all_reduce_cpu)
508
list(REMOVE_ITEM TEST_OPS test_imperative_ocr_attention_model)
509
list(REMOVE_ITEM TEST_OPS test_async_ssa_graph_executor_mnist)
510
list(REMOVE_ITEM TEST_OPS test_install_check)
511 512 513 514
list(REMOVE_ITEM TEST_OPS test_basic_gru_api)
list(REMOVE_ITEM TEST_OPS test_basic_gru_unit_op)
list(REMOVE_ITEM TEST_OPS test_basic_lstm_api)
list(REMOVE_ITEM TEST_OPS test_basic_lstm_unit_op)
H
Huihuang Zheng 已提交
515
list(REMOVE_ITEM TEST_OPS test_fuse_all_reduce_pass)
Z
Zhen Wang 已提交
516
list(REMOVE_ITEM TEST_OPS test_fuse_bn_act_pass)
Z
Zhang Ting 已提交
517
list(REMOVE_ITEM TEST_OPS test_fuse_bn_add_act_pass)
518 519
list(REMOVE_ITEM TEST_OPS test_imperative_static_runner_mnist)
list(REMOVE_ITEM TEST_OPS test_imperative_static_runner_while)
Y
YUNSHEN XIE 已提交
520
# disable test_cumsum_op temporaily
521
# list(REMOVE_ITEM TEST_OPS test_cumsum_op)
Z
Zeng Jinle 已提交
522

523 524
# disable this unittest temporarily
list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_exception)
Y
YUNSHEN XIE 已提交
525

526 527 528 529 530
# disable sparse_attention which not in suitable env
if ( (NOT WITH_GPU) OR (WIN32) OR (PADDLE_WITH_ARM) OR (WITH_ROCM) )
    list(REMOVE_ITEM TEST_OPS test_sparse_attention_op)
endif()

Z
Zeng Jinle 已提交
531 532 533
if (APPLE OR WIN32)
  list(REMOVE_ITEM TEST_OPS test_dataset)
  list(REMOVE_ITEM TEST_OPS test_dataset_dataloader)
534
  list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_base)
535
  # list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_exception)
536
  list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_process)
537 538
  list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_fds_clear)
  list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_exit_func)
539
  list(REMOVE_ITEM TEST_OPS test_imperative_signal_handler)
540 541
  list(REMOVE_ITEM TEST_OPS test_multiprocess_dataloader_static)
  list(REMOVE_ITEM TEST_OPS test_multiprocess_dataloader_dynamic)
542
  list(REMOVE_ITEM TEST_OPS test_multiprocess_dataloader_exception)
543
  list(REMOVE_ITEM TEST_OPS test_multiprocess_dataloader_iterable_dataset)
544
  list(REMOVE_ITEM TEST_OPS test_multiprocess_dataloader_dataset)
Z
Zeng Jinle 已提交
545 546
endif()

X
xiongkun 已提交
547 548 549 550
if (NOT WITH_GLOO)
    LIST(REMOVE_ITEM TEST_OPS test_cpuonly_spawn)
endif()

551 552 553 554
if(NOT WITH_GPU OR WIN32 OR APPLE)
  list(REMOVE_ITEM TEST_OPS test_build_strategy_fusion_group_pass)
endif()

555
# Some ops need to check results when gc is enabled
556
# Currently, only ops that register NoNeedBufferVarsInference need to do this test
557 558
set(TEST_OPS_WITH_GC
  test_affine_channel_op
559 560 561 562 563
  test_concat_op
  test_elementwise_add_op
  test_elementwise_sub_op
  test_fill_zeros_like2_op
  test_gather_op
564
  test_gather_nd_op
565 566 567 568 569 570 571 572
  test_linear_chain_crf_op
  test_lod_reset_op
  test_lookup_table_op
  test_mean_op
  test_pad2d_op
  test_scatter_op
  test_slice_op
  test_space_to_depth_op
573
  test_squared_l2_distance_op)
574 575 576 577 578 579

foreach(TEST_OP ${TEST_OPS_WITH_GC})
  list(REMOVE_ITEM TEST_OPS ${TEST_OP})
  py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS ${GC_ENVS})
endforeach()

580 581 582 583 584 585 586 587 588
if ((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
    list(REMOVE_ITEM TEST_OPS "test_fleet_graph_execution_meta_optimizer")
    list(REMOVE_ITEM TEST_OPS "test_gen_nccl_id_op")
    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_grad_clip")
    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_heter_ctr")
    list(REMOVE_ITEM TEST_OPS "test_dist_fleet_ps_gpu_ctr")
    list(REMOVE_ITEM TEST_OPS "test_dist_mnist_batch_merge")
endif()

Y
yuyang18 已提交
589 590 591
foreach(TEST_OP ${TEST_OPS})
    py_test_modules(${TEST_OP} MODULES ${TEST_OP})
endforeach(TEST_OP)
592
py_test_modules(test_adam_op_multi_thread MODULES test_adam_op ENVS FLAGS_inner_op_parallelism=4)
593
if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL OR APPLE)
Z
zhangchunle 已提交
594 595 596
    py_test_modules(test_warpctc_op MODULES test_warpctc_op)
    set_tests_properties(test_warpctc_op PROPERTIES TIMEOUT 120)
endif()
597 598
py_test_modules(test_bilinear_interp_op MODULES test_bilinear_interp_op ENVS ${GC_ENVS})
py_test_modules(test_nearest_interp_op MODULES test_nearest_interp_op ENVS ${GC_ENVS})
599
py_test_modules(test_imperative_resnet MODULES test_imperative_resnet ENVS
600
    FLAGS_cudnn_deterministic=1 SERIAL)
601
set_tests_properties(test_imperative_resnet PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
602
py_test_modules(test_imperative_resnet_sorted_gradient MODULES test_imperative_resnet_sorted_gradient ENVS
603
        FLAGS_cudnn_deterministic=1 SERIAL)
604
set_tests_properties(test_imperative_resnet_sorted_gradient PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
M
minqiyang 已提交
605
py_test_modules(test_imperative_mnist MODULES test_imperative_mnist ENVS
606
    FLAGS_cudnn_deterministic=1)
607
py_test_modules(test_imperative_mnist_sorted_gradient MODULES test_imperative_mnist_sorted_gradient ENVS
608
        FLAGS_cudnn_deterministic=1)
Y
Yan Xu 已提交
609
py_test_modules(test_imperative_se_resnext MODULES test_imperative_se_resnext ENVS
610
    FLAGS_cudnn_deterministic=1 SERIAL)
611
set_tests_properties(test_imperative_se_resnext PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
612 613
py_test_modules(test_imperative_ocr_attention_model MODULES test_imperative_ocr_attention_model ENVS
        FLAGS_cudnn_deterministic=1 SERIAL)
614 615 616
py_test_modules(test_install_check MODULES test_install_check ENVS
        FLAGS_cudnn_deterministic=1 SERIAL)
set_tests_properties(test_install_check PROPERTIES LABELS "RUN_TYPE=DIST")
617 618 619 620
py_test_modules(test_imperative_static_runner_mnist MODULES test_imperative_static_runner_mnist ENVS
    FLAGS_cudnn_deterministic=1)
py_test_modules(test_imperative_static_runner_while MODULES test_imperative_static_runner_while ENVS
    FLAGS_cudnn_deterministic=1)
621
set_tests_properties(test_conv2d_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
622
set_tests_properties(test_faster_tokenizer_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
623 624
set_tests_properties(test_conv2d_op_depthwise_conv PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
set_tests_properties(test_conv2d_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
C
ceci3 已提交
625 626 627
set_tests_properties(test_conv_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
set_tests_properties(test_norm_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
set_tests_properties(test_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
628
if(WITH_DISTRIBUTE)
629 630
    add_subdirectory(distributed_passes)
  
631 632
    add_subdirectory(auto_parallel)

633 634 635 636
    # FIXME(typhoonzero): add these tests back
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_transformer")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_transpiler")

637 638 639
    # TODO(sandyhouse): fix and add the ut back
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_hallreduce")

640 641 642 643
    #not need
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_base")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_base")

T
tangwei12 已提交
644

645 646 647 648 649 650 651 652
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_ctr")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_lars")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_train")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_save_load")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_text_classification")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_train")
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_word2vec")

T
tangwei12 已提交
653 654
    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_gloo")

T
tangwei12 已提交
655
    py_test_modules(test_recv_save_op MODULES test_recv_save_op ENVS ${dist_ENVS})
656
    py_test_modules(test_communicator_async MODULES test_communicator_async ENVS ${dist_ENVS})
T
Thunderbrook 已提交
657
    py_test_modules(test_communicator_ps_gpu MODULES test_communicator_ps_gpu ENVS ${dist_ENVS})
658 659
    py_test_modules(test_communicator_geo MODULES test_communicator_geo ENVS ${dist_ENVS})
    py_test_modules(test_communicator_half_async MODULES test_communicator_half_async ENVS ${dist_ENVS} FLAGS_communicator_send_queue_size=1 FLAGS_communicator_max_merge_var_num=1)
T
tangwei12 已提交
660
    py_test_modules(test_communicator_sync MODULES test_communicator_sync ENVS ${dist_ENVS} FLAGS_communicator_send_queue_size=1 FLAGS_communicator_max_merge_var_num=1)
M
mapingshuo 已提交
661
    py_test_modules(test_collective_optimizer MODULES test_collective_optimizer)
662
    if(NOT APPLE)
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
        py_test_modules(test_fleet_base MODULES test_fleet_base ENVS ${dist_ENVS})
        py_test_modules(test_fleet_base_2 MODULES test_fleet_base_2 ENVS ${dist_ENVS})
        py_test_modules(test_fleet_base_3 MODULES test_fleet_base_3 ENVS ${dist_ENVS})
        py_test_modules(test_fleet_amp_init MODULES test_fleet_amp_init ENVS ${dist_ENVS})
        py_test_modules(test_fleet_fp16_allreduce_meta_optimizer MODULES test_fleet_fp16_allreduce_meta_optimizer ENVS ${dist_ENVS})
        py_test_modules(test_fleet_private_function MODULES test_fleet_private_function ENVS ${dist_ENVS})
        py_test_modules(test_fleet_meta_optimizer_base MODULES test_fleet_meta_optimizer_base ENVS ${dist_ENVS})
        py_test_modules(test_fleet_distributed_strategy MODULES test_fleet_distributed_strategy)
        py_test_modules(test_fleet_static_mp_layers MODULES test_fleet_static_mp_layers)
        #py_test_modules(test_fleet_auto MODULES test_fleet_auto ENVS ${dist_ENVS})
        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
            py_test_modules(test_fleet_amp_meta_optimizer MODULES test_fleet_amp_meta_optimizer ENVS ${dist_ENVS})
            py_test_modules(test_fleet_gradient_merge_meta_optimizer MODULES test_fleet_gradient_merge_meta_optimizer ENVS ${dist_ENVS})
            py_test_modules(test_fleet_graph_executor MODULES test_fleet_graph_executor ENVS ${dist_ENVS})
            py_test_modules(test_fleet_hybrid_meta_optimizer MODULES test_fleet_hybrid_meta_optimizer ENVS ${dist_ENVS})
            py_test_modules(test_fleet_recompute_meta_optimizer MODULES test_fleet_recompute_meta_optimizer ENVS ${dist_ENVS})
            py_test_modules(test_fleet_sharding_meta_optimizer MODULES test_fleet_sharding_meta_optimizer ENVS ${dist_ENVS})
        endif()
Y
Yi Liu 已提交
681
        if(NOT WIN32)
682 683
            py_test_modules(test_auto_parallel_partitioner MODULES test_auto_parallel_partitioner ENVS ${dist_ENVS})
            py_test_modules(test_auto_parallel_partitioner_gpt MODULES test_auto_parallel_partitioner_gpt ENVS ${dist_ENVS})
C
caozhou 已提交
684
            py_test_modules(test_auto_parallel_searcher MODULES test_auto_parallel_searcher ENVS ${dist_ENVS})
C
caozhou 已提交
685
            py_test_modules(test_auto_parallel_reshard MODULES test_auto_parallel_reshard ENVS ${dist_ENVS})
686
            py_test_modules(test_auto_parallel_dist_tensor MODULES test_auto_parallel_dist_tensor ENVS ${dist_ENVS})
C
caozhou 已提交
687 688 689
            py_test_modules(test_auto_parallel_reshard_serial MODULES test_auto_parallel_reshard_serial ENVS ${dist_ENVS})
            py_test_modules(test_auto_parallel_reshard_mppp MODULES test_auto_parallel_reshard_mppp ENVS ${dist_ENVS})
            py_test_modules(test_auto_parallel_reshard_dpmppp MODULES test_auto_parallel_reshard_dpmppp ENVS ${dist_ENVS})
690
            py_test_modules(test_auto_parallel_cost_model MODULES test_auto_parallel_cost_model ENVS ${dist_ENVS})
691 692 693 694 695 696 697 698
            if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
                py_test_modules(test_fleet_lamb_meta_optimizer MODULES test_fleet_lamb_meta_optimizer ENVS ${dist_ENVS})
                py_test_modules(test_fleet_lars_meta_optimizer MODULES test_fleet_lars_meta_optimizer ENVS ${dist_ENVS})
                py_test_modules(test_fleet_localsgd_meta_optimizer MODULES test_fleet_localsgd_meta_optimizer ENVS ${dist_ENVS})



            endif()
Y
Yi Liu 已提交
699
        endif(NOT WIN32)
700
    endif(NOT APPLE)
G
gongweibao 已提交
701
    if(WITH_DGC)
W
WangXi 已提交
702 703
        # if with dgc, test all dgc tests.
        # NOTE. dist dgc tests is already in DIST_TEST_OPS
G
gongweibao 已提交
704
        py_test_modules(test_dgc_op MODULES test_dgc_op)
705 706
        py_test_modules(test_dgc_momentum_op MODULES test_dgc_momentum_op)
        py_test_modules(test_dgc_optimizer MODULES test_dgc_optimizer)
707
        py_test_modules(test_fleet_dgc_meta_optimizer MODULES test_fleet_dgc_meta_optimizer)
W
WangXi 已提交
708 709 710 711
    else()
        # if not with dgc, must close all dgc tests
        list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")
        list(REMOVE_ITEM DIST_TEST_OPS "test_dist_se_resnext_dgc")
G
gongweibao 已提交
712
    endif()
J
JiabinYang 已提交
713
    if(NOT APPLE)
714
        if(WITH_GPU OR WITH_ROCM)
715
            bash_test_modules(test_c_comm_init_op START_BASH test_c_comm_init_op.sh  ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
716
            py_test_modules(test_launch_coverage MODULES test_launch_coverage)
W
WangXi 已提交
717
        endif()
718

719
        bash_test_modules(test_fleetrun START_BASH test_fleetrun.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
720 721
        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
            bash_test_modules(test_fleet_launch_nproc START_BASH test_fleet_launch_nproc.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
722 723 724
            bash_test_modules(test_fleet_run_random_port START_BASH test_fleet_run_random_port.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
            bash_test_modules(test_fleet_launch_async START_BASH test_fleet_launch_async.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
            bash_test_modules(test_fleet_launch_cloud START_BASH test_fleet_launch_cloud.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
725
        endif()
726
        if(WITH_ASCEND OR WITH_ASCEND_CL)
727 728 729
            bash_test_modules(test_fleet_launch_ascend START_BASH test_fleet_launch_ascend.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
            bash_test_modules(test_ascend_group START_BASH test_ascend_group.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR})
        endif()
730

731
        # port range (20000, 23000) is reserved for dist-ops
G
gongweibao 已提交
732
        set(dist_ut_port 20001)
733
        foreach(TEST_OP ${DIST_TEST_OPS})
734
            bash_test_modules(${TEST_OP} START_BASH dist_test.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}")
735
            MATH(EXPR dist_ut_port "${dist_ut_port}+30")
736 737 738
            if(dist_ut_port GREATER_EQUAL 22998)
                message(FATAL_ERROR "available ports have been exhausted:${dist_ut_port}")
            endif()
739
        endforeach(TEST_OP)
740
        # solve it later.
741
        bash_test_modules(test_fleet_launch_ps START_BASH test_fleet_launch_ps.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}" PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} )
X
xiongkun 已提交
742 743 744
        if (WITH_GLOO)
            bash_test_modules(test_cpuonly_launch START_BASH test_cpuonly_launch.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}" PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} )
        endif()
745 746 747
        if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
            bash_test_modules(test_new_group START_BASH test_new_group.sh SERIAL LABELS "RUN_TYPE=EXCLUSIVE" ENVS "PADDLE_DIST_UT_PORT=${dist_ut_port}+20" PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} )
        endif()
748
    endif(NOT APPLE)
749
endif()
750

751
py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf)
752 753
# Coverage pipeline use cuda 10.1 now, profiler will random hang in cuda 10.1,
# see https://github.com/PaddlePaddle/Paddle/issues/29082 for details.
754
# We guess there are some bugs in cuda 10.1 or 10.2,
755 756 757 758 759 760
# since this unittest is stable in cuda 11 (py3 pipeline) now.
if(NOT WITH_COVERAGE)
  py_test_modules(test_parallel_executor_profiler MODULES test_parallel_executor_profiler)
  set_tests_properties(test_parallel_executor_profiler PROPERTIES LABELS "RUN_TYPE=DIST")
  set_tests_properties(test_parallel_executor_profiler PROPERTIES TIMEOUT 120)
endif()
761
py_test_modules(test_parallel_executor_transformer MODULES test_parallel_executor_transformer)
762 763
if(WIN32)
    py_test_modules(test_parallel_executor_transformer_auto_growth MODULES test_parallel_executor_transformer_auto_growth ENVS FLAGS_allocator_strategy=auto_growth CUDA_VISIBLE_DEVICES=0)
H
Huihuang Zheng 已提交
764 765
    py_test_modules(test_fuse_all_reduce_pass MODULES test_fuse_all_reduce_pass ENVS CUDA_VISIBLE_DEVICES=0)
    py_test_modules(test_feed_data_check_shape_type MODULES test_feed_data_check_shape_type ENVS CUDA_VISIBLE_DEVICES=0)
766
    py_test_modules(test_fetch_lod_tensor_array MODULES test_fetch_lod_tensor_array ENVS CUDA_VISIBLE_DEVICES=0)
767 768
else()
    py_test_modules(test_parallel_executor_transformer_auto_growth MODULES test_parallel_executor_transformer_auto_growth ENVS FLAGS_allocator_strategy=auto_growth)
H
Huihuang Zheng 已提交
769 770
    py_test_modules(test_fuse_all_reduce_pass MODULES test_fuse_all_reduce_pass)
    py_test_modules(test_feed_data_check_shape_type MODULES test_feed_data_check_shape_type)
771
    py_test_modules(test_fetch_lod_tensor_array MODULES test_fetch_lod_tensor_array)
772
endif()
773

H
hutuxian 已提交
774
py_test_modules(test_data_norm_op MODULES test_data_norm_op)
Z
Zhen Wang 已提交
775
py_test_modules(test_fuse_bn_act_pass MODULES test_fuse_bn_act_pass ENVS FLAGS_cudnn_deterministic=1 FLAGS_cudnn_batchnorm_spatial_persistent=1 FLAGS_conv_workspace_size_limit=1000)
Z
Zhang Ting 已提交
776
py_test_modules(test_fuse_bn_add_act_pass MODULES test_fuse_bn_add_act_pass ENVS FLAGS_cudnn_deterministic=1 FLAGS_cudnn_batchnorm_spatial_persistent=1 FLAGS_conv_workspace_size_limit=1000)
777

778
# NOTE: These unittests will appear NaN steadily in windows CI. After analysis,
779 780
# it is found that windows CI will run all the training unittests with the ON_INFER option turned on,
# which will not appear in other CIs. The calculation behavior of some ops in inference mode is
781 782
# inconsistent with that in non-inference mode.
if(NOT ON_INFER)
783 784 785 786
    py_test_modules(test_parallel_executor_seresnext_base_cpu MODULES test_parallel_executor_seresnext_base_cpu)
    py_test_modules(test_parallel_executor_seresnext_with_reduce_cpu MODULES test_parallel_executor_seresnext_with_reduce_cpu)
    py_test_modules(test_parallel_executor_seresnext_with_fuse_all_reduce_cpu MODULES test_parallel_executor_seresnext_with_fuse_all_reduce_cpu)
    set_tests_properties(test_parallel_executor_seresnext_base_cpu PROPERTIES TIMEOUT 900)
787
    set_tests_properties(test_parallel_executor_seresnext_base_cpu PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
788
    set_tests_properties(test_parallel_executor_seresnext_with_reduce_cpu PROPERTIES TIMEOUT 750)
789
    set_tests_properties(test_parallel_executor_seresnext_with_reduce_cpu PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
790
    set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_cpu PROPERTIES TIMEOUT 750)
791
    set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_cpu PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
792
endif()
793

794 795 796
if(NOT WIN32)
    # TODO: fix these unittests failure on Windows
    py_test_modules(test_layers MODULES test_layers ENVS FLAGS_cudnn_deterministic=1)
797
    py_test_modules(test_ir_memory_optimize_transformer MODULES test_ir_memory_optimize_transformer)
798 799 800
    # FIXME(zcd): temporally disable test_parallel_executor_fetch_feed in Windows CI because of the random failure.
    py_test_modules(test_parallel_executor_fetch_feed MODULES test_parallel_executor_fetch_feed)
    set_tests_properties(test_parallel_executor_fetch_feed PROPERTIES TIMEOUT 450)
D
dzhwinter 已提交
801
endif()
802

803
if(WITH_DISTRIBUTE AND NOT APPLE AND NOT WIN32)
G
gongweibao 已提交
804 805 806 807 808 809 810 811 812 813 814 815
    py_test_modules(test_fleet_checkpoint MODULES test_fleet_checkpoint)
    set_tests_properties(test_fleet_checkpoint PROPERTIES TIMEOUT 200)
    set_tests_properties(test_fleet_checkpoint PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint START_BASH dist_test.sh TIMEOUT 200 LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint1 START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint2 START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint3 START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint_multiple START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_auto_checkpoint_dist_basic START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_hdfs1 START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_hdfs2 START_BASH dist_test.sh TIMEOUT 200   LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
    bash_test_modules(test_hdfs3 START_BASH dist_test.sh TIMEOUT 200  LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
816 817
endif()

818
add_subdirectory(sequence)
819
add_subdirectory(dygraph_to_static)
F
Feiyu Chan 已提交
820
add_subdirectory(rnn)
821
add_subdirectory(autograd)
822
add_subdirectory(distribution)
823 824 825 826

if (NOT WIN32 OR NOT WITH_GPU)
    add_subdirectory(fft)
endif()
827

T
tianshuo78520a 已提交
828 829 830 831
if (WITH_XPU)
    add_subdirectory(xpu)
endif()

832
# dist xpu tests:
833 834 835 836 837
if (WITH_XPU_BKCL)
    py_test(test_collective_reduce_api_xpu SRCS "test_collective_reduce_api.py")
    py_test(test_collective_allreduce_api_xpu SRCS "test_collective_allreduce_api.py")
endif()

838 839 840 841
if(WIN32)
    cc_test(cc_imp_py_test SRCS cc_imp_py_test.cc DEPS python)
endif()

842 843 844 845
if (WITH_ASCEND_CL)
    add_subdirectory(npu)
endif()

846 847 848
if (WITH_MKLDNN)
    add_subdirectory(mkldnn)
endif()
849

J
jianghaicheng 已提交
850 851 852 853
if (WITH_IPU)
    add_subdirectory(ipu)
endif()

854 855 856 857
if (WITH_MLU)
    add_subdirectory(mlu)
endif()

858 859
add_subdirectory(asp)

860 861
add_subdirectory(ir)

H
hong 已提交
862 863
add_subdirectory(interpreter)

864
if (WITH_TESTING)
Z
Zeng Jinle 已提交
865
    set_property(TEST test_parallel_executor_mnist PROPERTY ENVIRONMENT GLOG_vmodule=all_reduce_deps_pass=10)
Z
Zeng Jinle 已提交
866
    set_property(TEST test_parallel_executor_fix_op_run_order PROPERTY ENVIRONMENT GLOG_vmodule=fix_op_run_order_pass=10)
867 868
endif()

869
set_tests_properties(test_parallel_executor_test_while_train test_parallel_executor_mnist
870
        test_parallel_executor_feed_persistable_var
871
        test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass
872
        test_data_norm_op
873
        test_dataloader_keep_order
874 875
        test_dataloader_unkeep_order
        test_parallel_executor_inference_feed_partial_data
876
        test_parallel_ssa_graph_inference_feed_partial_data
877
        test_fetch_unmerged
878
        test_buffer_shared_memory_reuse_pass PROPERTIES LABELS "RUN_TYPE=DIST")
879 880
# disable test_parallel_executor_fetch_isolated_var
# set_tests_properties(test_parallel_executor_fetch_isolated_var PROPERTIES LABELS "RUN_TYPE=DIST")
K
Kaipeng Deng 已提交
881
set_tests_properties(test_parallel_executor_crf test_sync_batch_norm_op test_inplace_abn_op
882 883 884
        test_parallel_executor_seresnext_base_gpu
        test_parallel_executor_seresnext_with_reduce_gpu
        test_parallel_executor_seresnext_with_fuse_all_reduce_gpu
885 886
        test_distributed_fused_lamb_op_with_clip
        test_distributed_fused_lamb_op_without_clip
887
        test_parallel_executor_fetch_isolated_var
C
chalsliu 已提交
888
        PROPERTIES LABELS "RUN_TYPE=DIST")
889

890
if(NOT WIN32 AND NOT APPLE)
891
    set_tests_properties(test_imperative_signal_handler PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
C
chalsliu 已提交
892 893 894 895 896 897
    set_tests_properties(test_imperative_data_loader_base PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    set_tests_properties(test_imperative_data_loader_fds_clear PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    # set_tests_properties(test_imperative_data_loader_exception PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    set_tests_properties(test_multiprocess_dataloader_static PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    set_tests_properties(test_multiprocess_dataloader_dynamic PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    set_tests_properties(test_multiprocess_dataloader_exception PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
898 899
    set_tests_properties(test_multiprocess_dataloader_iterable_dataset_static PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
    set_tests_properties(test_multiprocess_dataloader_iterable_dataset_dynamic PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
900
    set_tests_properties(test_multiprocess_dataloader_dataset PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
901
    set_tests_properties(test_multiprocess_dataloader_static PROPERTIES TIMEOUT 120)
902
endif()
Y
YUNSHEN XIE 已提交
903

904
if (NOT WIN32)
905
    set_tests_properties(test_multiprocess_reader_exception PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
906
    set_tests_properties(test_layers PROPERTIES TIMEOUT 120)
F
feng_shuai 已提交
907 908 909
    if (WITH_NV_JETSON)
    set_tests_properties(test_ir_memory_optimize_transformer PROPERTIES TIMEOUT 1200)
    else ()
910
    set_tests_properties(test_ir_memory_optimize_transformer PROPERTIES TIMEOUT 120)
F
feng_shuai 已提交
911
    endif ()
912 913 914
endif()

if (WITH_DISTRIBUTE AND NOT WIN32)
915
    set_tests_properties(test_fleet_utils PROPERTIES TIMEOUT 120)
916
    set_tests_properties(test_collective_cpu_barrier_with_gloo PROPERTIES TIMEOUT 40)
917 918 919 920
endif()

if (WITH_DISTRIBUTE)
    set_tests_properties(test_communicator_half_async PROPERTIES TIMEOUT 120)
921 922
    set_tests_properties(test_dist_fleet_ctr2 PROPERTIES TIMEOUT 200)
    set_tests_properties(test_dist_fleet_sparse_embedding_ctr PROPERTIES TIMEOUT 200)
923
    set_tests_properties(test_dist_fleet_infer PROPERTIES TIMEOUT 200)
924
    set_tests_properties(test_dist_fleet_raw_program_optimizer PROPERTIES TIMEOUT 120)
925
    set_tests_properties(test_dist_fleet_raw_program_optimizer_fuse_allreduce PROPERTIES TIMEOUT 60)
926
endif()
927

928
if (WITH_DISTRIBUTE AND NOT APPLE)
929
    if(WITH_GPU OR WITH_ROCM)
930
        set_tests_properties(test_c_comm_init_op PROPERTIES TIMEOUT 120)
931
        set_tests_properties(test_dist_mnist_gradient_merge PROPERTIES TIMEOUT 160)
932
    endif()
Y
YUNSHEN XIE 已提交
933 934 935 936 937
endif()

# setting timeout value as 15S
set_tests_properties(test_sync_batch_norm_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_cross_op PROPERTIES TIMEOUT 120)
938
set_tests_properties(test_imperative_lod_tensor_to_selected_rows PROPERTIES TIMEOUT 200)
Y
YUNSHEN XIE 已提交
939 940
set_tests_properties(test_lstm_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_star_gan_with_gradient_penalty PROPERTIES TIMEOUT 120)
Z
zhangchunle 已提交
941

Y
YUNSHEN XIE 已提交
942 943 944 945 946 947 948 949
set_tests_properties(test_bicubic_interp_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_deformable_conv_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nearest_interp_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_profiler PROPERTIES TIMEOUT 120)
set_tests_properties(test_inplace_softmax_with_cross_entropy PROPERTIES TIMEOUT 120)
set_tests_properties(test_cross_entropy2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_fetch_unmerged PROPERTIES TIMEOUT 120)
set_tests_properties(test_gru_unit_op PROPERTIES TIMEOUT 120)
D
Double_V 已提交
950
set_tests_properties(test_activation_nn_grad PROPERTIES TIMEOUT 200)
Y
YUNSHEN XIE 已提交
951 952 953
set_tests_properties(test_empty_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_transformer PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_div_op PROPERTIES TIMEOUT 120)
954
set_tests_properties(test_regularizer_api PROPERTIES TIMEOUT 150)
Y
YUNSHEN XIE 已提交
955
set_tests_properties(test_multiclass_nms_op PROPERTIES TIMEOUT 120)
956
if(NOT WIN32)
F
feng_shuai 已提交
957 958 959
    if (WITH_NV_JETSON)
    set_tests_properties(test_ir_memory_optimize_nlp PROPERTIES TIMEOUT 1200)
    else ()
960
    set_tests_properties(test_ir_memory_optimize_nlp PROPERTIES TIMEOUT 120)
F
feng_shuai 已提交
961
    endif ()
962
endif()
Y
YUNSHEN XIE 已提交
963 964 965 966 967 968 969 970 971 972 973
set_tests_properties(test_add_reader_dependency PROPERTIES TIMEOUT 120)
set_tests_properties(test_bilateral_slice_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_buffer_shared_memory_reuse_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_relu_depthwise_conv_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_fleet_util PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_transformer_sorted_gradient PROPERTIES TIMEOUT 120)
set_tests_properties(test_matmul_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nearest_interp_v2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_trilinear_interp_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_bicubic_interp_v2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_gather_op PROPERTIES TIMEOUT 120)
974
set_tests_properties(test_static_save_load PROPERTIES TIMEOUT 250)
975
set_tests_properties(test_pylayer_op PROPERTIES TIMEOUT 120)
976
set_tests_properties(test_paddle_save_load_binary PROPERTIES TIMEOUT 120)
W
WeiXin 已提交
977
if (WIN32)
W
WeiXin 已提交
978
    set_tests_properties(test_static_save_load_large PROPERTIES TIMEOUT 900)
W
WeiXin 已提交
979 980
    set_tests_properties(test_paddle_save_load PROPERTIES TIMEOUT 250)
else()
W
WeiXin 已提交
981
    set_tests_properties(test_static_save_load_large PROPERTIES TIMEOUT 600)
982
    set_tests_properties(test_paddle_save_load PROPERTIES TIMEOUT 250)
W
WeiXin 已提交
983
endif()
F
feng_shuai 已提交
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
if (WITH_NV_JETSON)
    set_tests_properties(test_concat_op PROPERTIES TIMEOUT 1200)
    set_tests_properties(test_conv3d_transpose_part2_op PROPERTIES TIMEOUT 1200)
    set_tests_properties(test_conv3d_transpose_op PROPERTIES TIMEOUT 1200)
    set_tests_properties(test_conv3d_op PROPERTIES TIMEOUT 1200)
    set_tests_properties(test_norm_op PROPERTIES TIMEOUT 1200)
    set_tests_properties(test_layer_norm_op PROPERTIES TIMEOUT 1500)
    set_tests_properties(test_pool3d_op PROPERTIES TIMEOUT 1500)
else()
    set_tests_properties(test_concat_op PROPERTIES TIMEOUT 120)
    set_tests_properties(test_conv3d_transpose_part2_op PROPERTIES TIMEOUT 120)
    set_tests_properties(test_conv3d_transpose_op PROPERTIES TIMEOUT 120)
    set_tests_properties(test_conv3d_op PROPERTIES TIMEOUT 120)
    set_tests_properties(test_norm_op PROPERTIES TIMEOUT 120)
    set_tests_properties(test_layer_norm_op PROPERTIES TIMEOUT 150)
    set_tests_properties(test_pool3d_op PROPERTIES TIMEOUT 150)
endif()
1001
set_tests_properties(test_imperative_selected_rows_to_lod_tensor PROPERTIES TIMEOUT 200)
Y
YUNSHEN XIE 已提交
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
set_tests_properties(test_index_select_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_ssa_graph_inference_feed_partial_data PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_crf PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_save_load PROPERTIES TIMEOUT 120)
set_tests_properties(test_partial_eager_deletion_transformer PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_seresnext_with_reduce_gpu PROPERTIES TIMEOUT 120)
set_tests_properties(test_dropout_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_argsort_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_gather_nd_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nn_grad PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_sub_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_row_conv_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_gpu PROPERTIES TIMEOUT 120)
1015 1016
set_tests_properties(test_distributed_fused_lamb_op_with_clip PROPERTIES TIMEOUT 120)
set_tests_properties(test_distributed_fused_lamb_op_without_clip PROPERTIES TIMEOUT 120)
Y
YUNSHEN XIE 已提交
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
set_tests_properties(test_elementwise_min_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nan_inf PROPERTIES TIMEOUT 120)
set_tests_properties(test_deformable_conv_v1_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_transformer_auto_growth PROPERTIES TIMEOUT 120)
set_tests_properties(test_py_reader_using_executor PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_add_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_weight_decay PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_ptb_rnn_sorted_gradient PROPERTIES TIMEOUT 120)
set_tests_properties(test_crop_tensor_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_eager_deletion_lstm_net PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_mnist PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_ptb_rnn PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_save_load_v2 PROPERTIES TIMEOUT 120)
K
Kaipeng Deng 已提交
1030
set_tests_properties(test_conv2d_transpose_op PROPERTIES TIMEOUT 120)
Y
YUNSHEN XIE 已提交
1031 1032 1033 1034 1035
set_tests_properties(test_prroi_pool_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_multiprocess_dataloader_iterable_dataset_static PROPERTIES TIMEOUT 120)
set_tests_properties(test_lstm_cudnn_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_stack_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_bilinear_interp_v2_op PROPERTIES TIMEOUT 120)
Y
YUNSHEN XIE 已提交
1036
set_tests_properties(test_svd_op PROPERTIES TIMEOUT 80)
1037
set_tests_properties(test_qr_op PROPERTIES TIMEOUT 60)
Y
YUNSHEN XIE 已提交
1038 1039 1040 1041 1042
set_tests_properties(test_deformable_psroi_pooling PROPERTIES TIMEOUT 120)
set_tests_properties(test_trilinear_interp_v2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_static_runner_mnist PROPERTIES TIMEOUT 120)
set_tests_properties(test_masked_select_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_sigmoid_cross_entropy_with_logits_op PROPERTIES TIMEOUT 120)
1043
set_tests_properties(test_imperative_optimizer_v2 PROPERTIES TIMEOUT 150)
Y
YUNSHEN XIE 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
set_tests_properties(test_partial_sum_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_cond PROPERTIES TIMEOUT 120)
set_tests_properties(test_space_to_depth_op PROPERTIES TIMEOUT 200)
set_tests_properties(test_dyn_rnn PROPERTIES TIMEOUT 120)
set_tests_properties(test_sgd_op PROPERTIES TIMEOUT 250)
set_tests_properties(test_parallel_executor_seresnext_base_gpu PROPERTIES TIMEOUT 120)
set_tests_properties(test_norm_nn_grad PROPERTIES TIMEOUT 120)
set_tests_properties(test_matrix_nms_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_generator_dataloader PROPERTIES TIMEOUT 120)
set_tests_properties(test_partial_concat_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_optimizer_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_softmax_with_cross_entropy_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_reduce_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_adam_optimizer_fp32_fp64 PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_nn_grad PROPERTIES TIMEOUT 120)
set_tests_properties(test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_conv_nn_grad PROPERTIES TIMEOUT 120)
set_tests_properties(test_program_prune_backward PROPERTIES TIMEOUT 120)
set_tests_properties(test_group_norm_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_optimizer PROPERTIES TIMEOUT 120)
set_tests_properties(test_pool2d_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_transpose_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_eager_deletion_gru_net PROPERTIES TIMEOUT 120)
S
Shang Zhizhou 已提交
1067
set_tests_properties(test_activation_op PROPERTIES TIMEOUT 270)
Y
YUNSHEN XIE 已提交
1068 1069 1070 1071 1072 1073
set_tests_properties(test_normal PROPERTIES TIMEOUT 120)
set_tests_properties(test_lstmp_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_bilinear_interp_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_decoupled_py_reader PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_bn_act_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_conv2d_op PROPERTIES TIMEOUT 120)
1074 1075
set_tests_properties(test_conv2d_op_depthwise_conv PROPERTIES TIMEOUT 120)
set_tests_properties(test_conv2d_api PROPERTIES TIMEOUT 120)
Y
YUNSHEN XIE 已提交
1076 1077 1078 1079 1080 1081
set_tests_properties(test_elementwise_mul_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_cyclic_cifar_dataset PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_all_reduce_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_dygraph_multi_forward PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_ocr_attention_model PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_mnist PROPERTIES TIMEOUT 120)
1082
set_tests_properties(test_fused_elemwise_activation_op PROPERTIES TIMEOUT 270)
1083
set_tests_properties(test_fused_elemwise_activation_op PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
1084 1085
set_tests_properties(test_gru_op PROPERTIES TIMEOUT 200)
set_tests_properties(test_regularizer PROPERTIES TIMEOUT 150)
Y
YUNSHEN XIE 已提交
1086 1087 1088 1089 1090
set_tests_properties(test_imperative_resnet PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_resnet_sorted_gradient PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_se_resnext PROPERTIES TIMEOUT 200)
set_tests_properties(test_matmul_v2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_slice_op PROPERTIES TIMEOUT 120)
1091 1092 1093 1094 1095 1096 1097
set_tests_properties(test_strided_slice_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_translated_layer PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_inference_feed_partial_data PROPERTIES TIMEOUT 120)
set_tests_properties(test_pad3d_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_dataloader_keep_order PROPERTIES TIMEOUT 120)
set_tests_properties(test_mean_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_dataloader_unkeep_order PROPERTIES TIMEOUT 120)
1098 1099
set_tests_properties(test_reader_reset PROPERTIES TIMEOUT 120)
set_tests_properties(test_pool3d_api PROPERTIES TIMEOUT 120)
H
hlygit66666 已提交
1100
set_tests_properties(test_cumprod_op PROPERTIES TIMEOUT 120)
1101
set_tests_properties(test_split_program PROPERTIES TIMEOUT 120)
1102
if(WITH_DISTRIBUTE AND WITH_GPU AND WITH_NCCL)
1103 1104 1105
    set_tests_properties(test_parallel_dygraph_dataparallel PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_unused_variables PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_control_flow PROPERTIES TIMEOUT 120)
1106 1107
    set_tests_properties(test_parallel_dygraph_no_sync PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_no_sync_gradient_check PROPERTIES TIMEOUT 30)
1108 1109
    set_tests_properties(test_parallel_dygraph_pipeline_parallel PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_tensor_parallel PROPERTIES TIMEOUT 200)
J
JZ-LIANG 已提交
1110
    set_tests_properties(test_parallel_dygraph_sharding_parallel PROPERTIES TIMEOUT 120)
1111
    set_tests_properties(test_dygraph_sharding_optimizer_stage2 PROPERTIES TIMEOUT 120)
B
Baibaifan 已提交
1112
    set_tests_properties(test_dygraph_sharding_stage2 PROPERTIES TIMEOUT 120)
B
Baibaifan 已提交
1113
    set_tests_properties(test_dygraph_sharding_stage3 PROPERTIES TIMEOUT 120)
1114
    set_tests_properties(test_auto_parallel_parallelizer PROPERTIES TIMEOUT 120)
1115
    set_tests_properties(test_parallel_dygraph_mp_layers PROPERTIES TIMEOUT 120)
1116
    set_tests_properties(test_hybrid_parallel_inference_helper PROPERTIES TIMEOUT 120)
1117
    set_tests_properties(test_parallel_class_center_sample PROPERTIES TIMEOUT 120)
1118
    set_tests_properties(test_parallel_margin_cross_entropy PROPERTIES TIMEOUT 120)
1119
    set_tests_properties(test_auto_parallel_data_unshard PROPERTIES TIMEOUT 120)
1120
    set_tests_properties(test_auto_parallel_save_load PROPERTIES TIMEOUT 120)
1121
    set_tests_properties(test_auto_parallel_autoconvert PROPERTIES TIMEOUT 120)
1122 1123
    set_tests_properties(test_collective_process_group PROPERTIES TIMEOUT 120)
    
1124 1125 1126 1127
    if(${NCCL_VERSION} VERSION_GREATER_EQUAL 2212)
        set_tests_properties(test_parallel_dygraph_sparse_embedding PROPERTIES TIMEOUT 120)
        set_tests_properties(test_parallel_dygraph_transformer PROPERTIES TIMEOUT 120)
    endif()
Y
YUNSHEN XIE 已提交
1128
endif()
1129
if((WITH_ROCM OR WITH_GPU) AND NOT WIN32)
Y
YUNSHEN XIE 已提交
1130
    set_tests_properties(test_collective_allgather_api PROPERTIES TIMEOUT 120)
L
lilong12 已提交
1131
    set_tests_properties(test_collective_alltoall_api PROPERTIES TIMEOUT 120)
1132 1133
    set_tests_properties(test_collective_global_gather PROPERTIES TIMEOUT 120)
    set_tests_properties(test_collective_global_scatter PROPERTIES TIMEOUT 120)
L
lilong12 已提交
1134
    set_tests_properties(test_collective_sendrecv_api PROPERTIES TIMEOUT 120)
1135 1136
    set_tests_properties(test_collective_broadcast_api PROPERTIES TIMEOUT 120)
    set_tests_properties(test_collective_allreduce_api PROPERTIES TIMEOUT 120)
1137
    if(WITH_DISTRIBUTE)
K
kuizhiqing 已提交
1138
        set_tests_properties(test_new_group_api PROPERTIES TIMEOUT 120)
1139
        set_tests_properties(test_pipeline PROPERTIES TIMEOUT 120)
1140
        set_tests_properties(test_ir_pass_pipeline PROPERTIES TIMEOUT 120)
L
lilong12 已提交
1141
        set_tests_properties(test_static_model_parallel PROPERTIES TIMEOUT 240)
K
kuizhiqing 已提交
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
        set_tests_properties(test_collective_split_embedding
            test_collective_split_embedding_none_divisible
            test_collective_split_row_linear
            test_collective_split_col_linear
            test_collective_scatter_api
            test_collective_barrier_api
            test_collective_reduce_api
            test_pipeline_parallel
            test_collective_allreduce_api
            test_new_group_api
            test_collective_broadcast_api
            test_collective_allgather_api
            test_collective_alltoall_api
1155 1156
            test_collective_global_gather
            test_collective_global_scatter
K
kuizhiqing 已提交
1157
            PROPERTIES LABELS "RUN_TYPE=DIST")
1158
    endif()
1159 1160 1161 1162
    set_tests_properties(test_reducescatter_api PROPERTIES TIMEOUT 120)
    set_tests_properties(test_broadcast PROPERTIES TIMEOUT 120)
    set_tests_properties(test_reducescatter PROPERTIES TIMEOUT 120)
    set_tests_properties(test_collective_reduce_api PROPERTIES TIMEOUT 120)
1163
    set_tests_properties(test_pipeline_parallel PROPERTIES TIMEOUT 120)
1164 1165
    set_tests_properties(test_collective_reduce PROPERTIES TIMEOUT 120)
    set_tests_properties(test_allreduce PROPERTIES TIMEOUT 120)
L
lilong12 已提交
1166 1167
    set_tests_properties(test_c_concat PROPERTIES TIMEOUT 120)
    set_tests_properties(test_c_split PROPERTIES TIMEOUT 120)
1168
    set_tests_properties(test_allgather PROPERTIES TIMEOUT 120)
L
lilong12 已提交
1169
    set_tests_properties(test_c_identity PROPERTIES TIMEOUT 120)
1170 1171
    set_tests_properties(test_collective_scatter_api PROPERTIES TIMEOUT 120)
    set_tests_properties(test_collective_barrier_api PROPERTIES TIMEOUT 120)
1172 1173
    set_tests_properties(test_collective_scatter PROPERTIES TIMEOUT 120)
    set_tests_properties(test_collective_sendrecv PROPERTIES TIMEOUT 120)
Y
YUNSHEN XIE 已提交
1174
endif()
1175
if(WITH_GPU OR WITH_ROCM)
1176
    set_tests_properties(test_imperative_auto_mixed_precision PROPERTIES TIMEOUT 300)
1177
    set_tests_properties(test_parallel_dygraph_sync_batch_norm PROPERTIES TIMEOUT 120)
1178
    set_tests_properties(test_rank_attention_op PROPERTIES TIMEOUT 120)
1179
endif()
Y
YUNSHEN XIE 已提交
1180
set_tests_properties(test_inplace_addto_strategy PROPERTIES TIMEOUT 120)
1181
set_tests_properties(test_eigvals_op PROPERTIES TIMEOUT 400)
F
From00 已提交
1182
set_tests_properties(test_tensordot PROPERTIES TIMEOUT 1000)
1183
set_tests_properties(test_tensordot PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
X
xiongkun 已提交
1184 1185 1186 1187 1188
if (WITH_GLOO)
    set_tests_properties(test_parallel_dygraph_unused_variables_gloo PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_sparse_embedding_gloo PROPERTIES TIMEOUT 120)
    set_tests_properties(test_parallel_dygraph_sparse_embedding_over_height_gloo PROPERTIES TIMEOUT 120)
endif()