diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index f1142dbbbba01c2f1c96ffefad8d2b16a6015ef6..3b20a403b711db363d3a301b396b0d751652e18c 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1238,6 +1238,9 @@ set +x exec_retry_threshold=10 is_retry_execuate=0 if [ -n "$failed_test_lists" ];then + if [ ${TIMEOUT_DEBUG_HELP:-OFF} == "ON" ];then + bash $PADDLE_ROOT/tools/timeout_debug_help.sh "$failed_test_lists" # cat logs for tiemout uts which killed by ctest + fi read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) need_retry_ut_count=${#need_retry_ut_arr[@]} diff --git a/tools/timeout_debug_help.sh b/tools/timeout_debug_help.sh new file mode 100644 index 0000000000000000000000000000000000000000..45de2db87e85302c5f281bf91538d480edcd3409 --- /dev/null +++ b/tools/timeout_debug_help.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set +e +failed_uts=$1 +need_debug_ut_re='test_dist_fleet' +cat_log_judge=$(echo "${failed_uts}" | grep 'Timeout' | grep -oEi "$need_debug_ut_re" ) +if [[ "$cat_log_judge" != "" ]];then + echo "==============================================" + echo "show timeout ut logs" + echo "==============================================" + cat /tmp/tr0_err.log /tmp/tr1_err.log /tmp/ps0_err.log /tmp/ps1_err.log + cat /tmp/heter0_err.log /tmp/heter1_err.log +fi +set -e