From f7cbfc4c9ef92f29e81bdd48be4bc81ea7bd68b6 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Wed, 19 Jul 2023 11:28:25 +0800 Subject: [PATCH] Add gpups script (#55479) * Add gpups ci test script * test=gpups * test=gpups * test=gpups * test=gpups --- paddle/scripts/paddle_build.sh | 28 ++------- tools/gpups_test.sh | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 tools/gpups_test.sh diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index cd56e6a7176..ef2cff21c44 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -2193,30 +2193,10 @@ function parallel_test_base_gpups() { ======================================== EOF ut_startTime_s=`date +%s` - - protobuf_version=`pip list | grep "protobuf" | awk '{print $2}'` - if [[ "$protobuf_version" == 3.* ]]; then - echo "Your current protobuf version is $protobuf_version" - ctest -L "RUN_TYPE=GPUPS" --timeout 120 - else - echo "Your current protobuf version is $protobuf_version" - #get all unittests need to be run by protobuf 3 - python ${PADDLE_ROOT}/tools/test_run_by_protobuf_3.py > all_ut_run_by_protobuf3 - # get all unittets need to be run in gpups ci - ctest -N -V -L "RUN_TYPE=GPUPS"| grep -Ei "Test[ \t]+#" | grep -oEi "\w+$" > ut_gpups - #get the intersection of ut_run_by_protobuf3 and ut_gpups - grep -F -f all_ut_run_by_protobuf3 ut_gpups > ut_run_by_protobuf3_in_gpups - #get the difference set of ut_gpups and ut_run_by_protobuf3_in_gpups - grep -F -x -v -f ut_run_by_protobuf3_in_gpups ut_gpups > ut_run_in_gpups - - ctest -R ${ut_run_in_gpups} --timeout 120 - pip install protobuf==3.20.2 - ctest -R ${ut_run_by_protobuf3_in_gpups} --timeout 120 - ut_endTime_s=`date +%s` - echo "GPUPS testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s" - pip install protobuf==$protobuf_version - fi - + set +e + bash ${PADDLE_ROOT}/tools/gpups_test.sh + EXIT_CODE=$? + set -e ut_endTime_s=`date +%s` echo "GPUPS testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s" diff --git a/tools/gpups_test.sh b/tools/gpups_test.sh new file mode 100644 index 00000000000..d6437e01335 --- /dev/null +++ b/tools/gpups_test.sh @@ -0,0 +1,104 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +serial_list="^test_conv2d_op$|\ +^test_conv2d_transpose_op$|\ +^test_conv3d_op$" + +parallel_list="^init_phi_test$|\ +^operator_test$|\ +^test_collective_cpu_barrier_with_gloo$|\ +^test_conv1d_layer$|\ +^test_conv1d_transpose_layer$|\ +^test_conv2d_api$|\ +^test_conv2d_fusion_op$|\ +^test_conv2d_layer$|\ +^test_conv2d_op_depthwise_conv$|\ +^test_conv2d_transpose_layer$|\ +^test_conv2d_transpose_op_depthwise_conv$|\ +^test_conv3d_layer$|\ +^test_conv3d_transpose_layer$|\ +^test_conv3d_transpose_op$|\ +^test_conv_bn_fuse_pass_cc$|\ +^test_conv_nn_grad$|\ +^test_conv_shift_op$|\ +^test_conv_transpose_nn_grad$|\ +^test_convert_call$|\ +^test_convert_call_generator$|\ +^test_convert_operators$|\ +^test_convert_to_mixed_precision$|\ +^test_convert_to_process_meshes$|\ +^test_cpu_cuda_to_tensor$|\ +^test_cudnn_placement_pass$|\ +^test_custom_kernel$|\ +^test_dist_fleet_ps11$|\ +^test_dist_fleet_ps12$|\ +^test_executor_feed_non_tensor$|\ +^test_fused_adam_op$|\ +^test_fused_attention_no_dropout$|\ +^test_fused_attention_op$|\ +^test_fused_attention_op_api$|\ +^test_fused_attention_op_api_static_build$|\ +^test_fused_attention_op_static_build$|\ +^test_fused_bias_dropout_residual_layer_norm_op$|\ +^test_fused_bias_dropout_residual_layer_norm_op_api$|\ +^test_fused_comm_buffer$|\ +^test_fused_dropout_act_bias$|\ +^test_fused_dropout_add_op$|\ +^test_fused_emb_seq_pool_op$|\ +^test_fused_embedding_fc_lstm_op$|\ +^test_fused_fc_elementwise_layernorm_op$|\ +^test_fused_feedforward_op$|\ +^test_fused_feedforward_op_static_build$|\ +^test_fused_gate_attention_op$|\ +^test_fused_gemm_epilogue_grad_op$|\ +^test_fused_gemm_epilogue_grad_op_with_es$|\ +^test_fused_gemm_epilogue_op$|\ +^test_fused_gemm_epilogue_op_with_es$|\ +^test_fused_layernorm_residual_dropout_bias$|\ +^test_fused_linear_param_grad_add$|\ +^test_fused_linear_pass$|\ +^test_fused_matmul_bias$|\ +^test_fused_multi_transformer_decoder_pass$|\ +^test_fused_multi_transformer_encoder_pass$|\ +^test_fused_multi_transformer_int8_op$|\ +^test_fused_residual_dropout_bias$|\ +^test_fused_rotary_position_embedding$|\ +^test_fused_scale_bias_relu_conv_bnstats_op$|\ +^test_fused_token_prune_op$|\ +^test_fused_transformer_encoder_layer$|\ +^test_fused_transformer_with_amp_decorator$|\ +^test_gather_nd_op$|\ +^test_index_select_op$|\ +^test_pass_base_list$|\ +^test_roll_op$|\ +^test_switch_autotune$|\ +^test_tcp_store$|\ +^test_to_tensor$|\ +^test_top_k_v2_op$" + +cd ${work_dir}/build + +set +e +ctest --output-on-failure -R "($parallel_list)" --timeout 120 -j4 +EXIT_CODE_1=$? + +ctest --output-on-failure -R "($serial_list)" --timeout 120 -j1 +EXIT_CODE_2=$? +set -e + +if [ "${EXIT_CODE_1}" != "0" ] || [ "${EXIT_CODE_2}" != "0" ];then + echo "Sorry, some tests failed." + exit 8 +fi -- GitLab