From f302bb4f8bfe9bd5c2b5fbb944e79601ac88bf72 Mon Sep 17 00:00:00 2001 From: YUNSHEN XIE <1084314248@qq.com> Date: Fri, 12 Mar 2021 10:59:41 +0800 Subject: [PATCH] help timeout ut debug (#31500) * To help timeout_ut debug * To help timeout_ut debug * added show information --- paddle/scripts/paddle_build.sh | 3 +++ tools/timeout_debug_help.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tools/timeout_debug_help.sh diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index f1142dbbbba..3b20a403b71 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1238,6 +1238,9 @@ set +x exec_retry_threshold=10 is_retry_execuate=0 if [ -n "$failed_test_lists" ];then + if [ ${TIMEOUT_DEBUG_HELP:-OFF} == "ON" ];then + bash $PADDLE_ROOT/tools/timeout_debug_help.sh "$failed_test_lists" # cat logs for tiemout uts which killed by ctest + fi read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) need_retry_ut_count=${#need_retry_ut_arr[@]} diff --git a/tools/timeout_debug_help.sh b/tools/timeout_debug_help.sh new file mode 100644 index 00000000000..45de2db87e8 --- /dev/null +++ b/tools/timeout_debug_help.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set +e +failed_uts=$1 +need_debug_ut_re='test_dist_fleet' +cat_log_judge=$(echo "${failed_uts}" | grep 'Timeout' | grep -oEi "$need_debug_ut_re" ) +if [[ "$cat_log_judge" != "" ]];then + echo "==============================================" + echo "show timeout ut logs" + echo "==============================================" + cat /tmp/tr0_err.log /tmp/tr1_err.log /tmp/ps0_err.log /tmp/ps1_err.log + cat /tmp/heter0_err.log /tmp/heter1_err.log +fi +set -e -- GitLab