Optimize the process of obtaining prec_list on windows (#32123)

* test,test,notest,test=windows_ci

* test,notest,test=windows_ci

* test,notest,test=windows_ci

* test,notest,test=windows_ci

* remove test code

* delete some unnecessary logs

* fix format error

* turn on added ut check on windows

tools/check_added_ut.sh

@@ -50,19 +50,12 @@ cd prec_build
 if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
     bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >prebuild.log 2>&1
 elif [[ "$SYSTEM" == "Windows_NT" ]];then
-    bash $PADDLE_ROOT/win_cmake.sh
+    bash $PADDLE_ROOT/win_cmake.sh >prec_build.log 2>&1
 fi
 ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/br-ut
 cd $PADDLE_ROOT/build
 ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/pr-ut
 cd $PADDLE_ROOT
-echo "================================="
-echo "br-ut"
-cat $PADDLE_ROOT/br-ut
-echo "================================="
-echo "pr-ut"
-cat $PADDLE_ROOT/pr-ut
-echo "================================="
 grep -F -x -v -f br-ut pr-ut > $PADDLE_ROOT/added_ut
 if [[ "$SYSTEM" == 'Linux' ]];then
     sort pr-ut |uniq -d > $PADDLE_ROOT/duplicate_ut

tools/get_pr_ut.py

@@ -265,8 +265,7 @@ class PRChecker(object):
                         '.cu'):
                     if f.find('test_') != -1 or f.find('_test') != -1:
                         print('PREC {} need check new ut'.format(f))
-                        if current_system != "Windows":
-                            check_added_ut = True
+                        check_added_ut = True
                     elif self.is_only_comment(f):
                         ut_list.append('nomap_comment_placeholder')
                     else:

tools/windows/get_prec_ut_list.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""To get a list of prec ut """
+
+import sys
+
+
+def get_prec_ut_list(all_test_cases, prec_test_cases):
+    """Select the ut that needs to be executed"""
+    all_test_cases_list = all_test_cases.strip().split("\n")
+    prec_test_cases_list = prec_test_cases.strip().split("\n")
+    all_test_cases_list_new = [item.rstrip() for item in all_test_cases_list]
+    prec_test_cases_list_new = [item.rstrip() for item in prec_test_cases_list]
+
+    if len(prec_test_cases) == 0:
+        return "\n".join(all_test_cases_list)
+
+    case_to_run = ['test_prec_ut']
+    for case in all_test_cases_list_new:
+        if case in prec_test_cases_list_new:
+            case_to_run.append(case)
+        else:
+            print("{} will not run in PRECISION_TEST mode.".format(case))
+    for case in case_to_run:
+        print(case)
+
+
+if __name__ == '__main__':
+    all_test_cases = sys.argv[1]
+    prec_test_cases = sys.argv[2]
+    get_prec_ut_list(all_test_cases, prec_test_cases)

tools/windows/run_unittests.sh

@@ -224,29 +224,16 @@ if [ ${WITH_GPU:-OFF} == "ON" ];then
     if [ ${PRECISION_TEST:-OFF} == "ON" ]; then
         python ${PADDLE_ROOT}/tools/get_pr_ut.py
         if [[ -f "ut_list" ]]; then
-            set +x
             echo "PREC length: "`wc -l ut_list`
             precision_cases=`cat ut_list`
-            set -x
         fi
     fi
 
     set +e
     if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]];then
-        UT_list_prec=''
-        re=$(cat ut_list|awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}')
-        for case in $UT_list; do
-            flag=$(echo $case|grep -oE $re)
-            if [ -n "$flag" ];then
-                if [ -z "$UT_list_prec" ];then
-                    UT_list_prec=$case
-                else
-                    UT_list_prec=$UT_list_prec'\n'$case
-                fi
-            else
-                echo $case "won't run in PRECISION_TEST mode."
-            fi
-        done
+        UT_list_res=$(python ${PADDLE_ROOT}/tools/windows/get_prec_ut_list.py "$UT_list" "$precision_cases" )
+        UT_list_prec=$(echo "${UT_list_res}" | grep -v 'PRECISION_TEST')
+        UT_list_prec_info=$(echo "${UT_list_res}" | grep 'PRECISION_TEST')
         UT_list=$UT_list_prec
     fi
     set -e