From fea09fe5344ff4928cf5e0309b30809c3de8a0b5 Mon Sep 17 00:00:00 2001
From: YUNSHEN XIE <1084314248@qq.com>
Date: Wed, 14 Oct 2020 21:21:51 +0800
Subject: [PATCH] disable ut quickly (#27793)

* disable ut quickly

* fix some error

* fix some error

* install urllib2 package

* use requests package instead of urllib2

* fix error caused by windows regular parameter

* fix error on windows

* fix some error

* fix with format error

* show disable ut in log

* fix some error

* fix some error

* add the handling of error in executing get_quickly_disable_ut
---
 paddle/scripts/paddle_build.bat | 11 ++++++++-
 paddle/scripts/paddle_build.sh  | 28 ++++++++++++++++------
 tools/get_quick_disable_lt.py   | 41 +++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 8 deletions(-)
 create mode 100644 tools/get_quick_disable_lt.py

diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat
index 2f7afcf17df..afb3f360a9a 100644
--- a/paddle/scripts/paddle_build.bat
+++ b/paddle/scripts/paddle_build.bat
@@ -293,10 +293,19 @@ dir %THIRD_PARTY_PATH:/=\%\install\mklml\lib
 dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin
 dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin
 
+pip install requests
+python %work_dir%\tools\get_quick_disable_lt.py > Output
+if %errorlevel%==0 (
+    set /p disable_ut_quickly=<Output
+    DEL Output
+    ) else (
+    set disable_ut_quickly=''
+)
+
 set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;^
 %THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;^
 %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH%
-ctest.exe --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4
+ctest.exe -E "(%disable_ut_quickly%)" --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4
 goto:eof
 
 :unit_test_error
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index b5c44f3d55b..913af90de5a 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -555,12 +555,12 @@ EOF
         fi
         tmpfile_rand=`date +%s%N`
         tmpfile=$tmp_dir/$tmpfile_rand
-        set +e
+        set +ex
         ut_startTime_s=`date +%s`
-        ctest --output-on-failure -j $2 | tee $tmpfile
+        get_quickly_disable_ut||disable_ut_quickly='' # indicate whether the case was in quickly disable list 
+        ctest -E "($disable_ut_quickly)" --output-on-failure -j $2 | tee $tmpfile
         failed_test_lists=''
         collect_failed_tests
-        set +x
         mactest_error=0
         retry_unittests_record=''
         retry_time=3
@@ -892,6 +892,19 @@ function collect_failed_tests() {
     done
 }
 
+# getting qucik disable ut list 
+function get_quickly_disable_ut() {
+    python -m pip install requests
+    if disable_ut_quickly=$(python ${PADDLE_ROOT}/tools/get_quick_disable_lt.py); then
+        echo "========================================="
+        echo "The following unittests have been disabled:"
+        echo ${disable_ut_quickly}
+        echo "========================================="
+    else
+        disable_ut_quickly=''
+    fi
+}
+
 function card_test() {
     set -m
     case_count $1 $2
@@ -934,15 +947,15 @@ function card_test() {
         tmpfile=$tmp_dir/$tmpfile_rand"_"$i
         if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
             if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
-                (ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
+                (ctest -I $i,,$NUM_PROC -R "($testcases)" -E "($disable_ut_quickly)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
             else  
-                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
+                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -E "($disable_ut_quickly)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
             fi
         else
             if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
-                (ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
+                (ctest -I $i,,$NUM_PROC -R "($testcases)" -E "($disable_ut_quickly)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
             else
-                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
+                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -E "($disable_ut_quickly)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
             fi
         fi
     done
@@ -977,6 +990,7 @@ set +x
         is_exclusive=''           # indicate whether the case is exclusive type
         is_multicard=''           # indicate whether the case is multiple GPUs type
         is_nightly=''             # indicate whether the case will only run at night
+        get_quickly_disable_ut||disable_ut_quickly=''    # indicate whether the case was in quickly disable list 
         while read -r line; do
             if [[ "$line" == "" ]]; then
                 continue
diff --git a/tools/get_quick_disable_lt.py b/tools/get_quick_disable_lt.py
new file mode 100644
index 00000000000..9b41f5e7808
--- /dev/null
+++ b/tools/get_quick_disable_lt.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import ssl
+import requests
+
+
+def download_file():
+    """Get disabled unit tests"""
+    ssl._create_default_https_context = ssl._create_unverified_context
+    url = "https://sys-p0.bj.bcebos.com/prec/{}".format('disable_ut')
+    f = requests.get(url)
+    data = f.text
+    status_code = f.status_code
+    if len(data.strip()) == 0 or status_code != 200:
+        sys.exit(1)
+    else:
+        lt = data.strip().split('\n')
+        lt = '^' + '$|^'.join(lt) + '$'
+        print(lt)
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    try:
+        download_file()
+    except Exception as e:
+        print(e)
+        sys.exit(1)
-- 
GitLab