test=pretest (#33573)

paddle/scripts/paddle_build.sh

@@ -1427,7 +1427,6 @@ EOF
 function insert_pile_to_h_cu_diff {
     # TODO get develop h/cu md5
     cd ${PADDLE_ROOT}
-    find ${PADDLE_ROOT} -name '*.h'| grep -v ${PADDLE_ROOT}/build >> ${PADDLE_ROOT}/tools/h_cu_files.log
     find ${PADDLE_ROOT} -name '*.cu'| grep -v ${PADDLE_ROOT}/build >> ${PADDLE_ROOT}/tools/h_cu_files.log
     python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'get_h_file_md5' ${PADDLE_ROOT}
     
@@ -1447,8 +1446,8 @@ function precise_card_test_single {
         cd ${PADDLE_ROOT}/build
         precise_card_test "^${case}$" $num
         # c++ 
-        if [ -d "${PADDLE_ROOT}/build/ut_map/$case" ];then
-            rm -rf ${PADDLE_ROOT}/build/ut_map/$case
+        if [ ! -d "${PADDLE_ROOT}/build/ut_map/$case" ];then
+            mkdir ${PADDLE_ROOT}/build/ut_map/$case
         fi
         set -x
         mkdir ${PADDLE_ROOT}/build/ut_map/$case
@@ -1460,7 +1459,9 @@ function precise_card_test_single {
         ls python-coverage.data.*
         if [[ $? == 0 ]]
         then
+            if [ ! -d "${PADDLE_ROOT}/build/pytest/$case" ];then
                 mkdir -p ${PADDLE_ROOT}/build/pytest/$case
+            fi
             mv python-coverage.data.* ${PADDLE_ROOT}/build/pytest/$case
         fi
         find paddle/fluid -name *.gcda | xargs rm -f #delete gcda
@@ -1571,26 +1572,38 @@ set -x
     precise_card_test_single "$single_card_tests_1" 1
     precise_card_test_single "$multiple_card_tests" 2
     precise_card_test_single "$exclusive_tests"
-
+    wait;
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT}
     
-    if [[ -f "${PADDLE_ROOT}/build/utNotSuccess" ]]; then
-        rerun_tests=`cat ${PADDLE_ROOT}/build/utNotSuccess`
-        precise_card_test_single "$rerun_tests"
-    fi
+    #analy h/cu to Map file
+    python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT}
+
     wait;
+    get_failedUts_precise_map_file
 
     #generate python coverage and generate python file to tests_map_file
     python ${PADDLE_ROOT}/tools/pyCov_multithreading.py ${PADDLE_ROOT}
+    wait;
 
-    #analy h/cu to Map file
-    python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT}
     #generate ut map
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT}
-    wait;
 }
 
+function get_failedUts_precise_map_file {
+    if [[ -f "${PADDLE_ROOT}/build/utNotSuccess" ]]; then
+        rerun_tests=`cat ${PADDLE_ROOT}/build/utNotSuccess`
+        #remove pile to full h/cu file
+        python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'remove_pile_from_h_file' ${PADDLE_ROOT}
+        cd ${PADDLE_ROOT}/build
+        cmake_base ${PYTHON_ABI:-""}
+        build ${parallel_number}
+        pip uninstall -y paddlepaddle-gpu
+        pip install ${PADDLE_ROOT}/build/python/dist/*whl
+        precise_card_test_single "$rerun_tests"
+        wait;
         
+    fi
+}
 
 function parallel_test_base_xpu() {
     mkdir -p ${PADDLE_ROOT}/build

tools/get_single_test_cov.py

@@ -46,15 +46,14 @@ def analysisFNDAFile(rootPath, test):
         if 'FNDA:' in message:
             message_list = message.split('\n')
             clazz_filename = message_list[0]
-            if not clazz_filename.endswith('.h'):  #filter .h's Analysis
+            #if not clazz_filename.endswith('.h'):  #filter .h's Analysis
             for i in range(1, len(message_list) - 1):
                 fn = message_list[i]
                 matchObj = re.match(
                     r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)',
                     fn, re.I)
                 if matchObj == None:
-                        os.system('echo %s >> %s' %
-                                  (clazz_filename, ut_map_file))
+                    os.system('echo %s >> %s' % (clazz_filename, ut_map_file))
                     break
     f.close()
 

tools/get_ut_file_map.py

@@ -139,48 +139,53 @@ def ut_file_map_supplement(rootPath):
         'cd /pre_test && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json --no-check-certificate'
     )
     ut_file_map_old = "/pre_test/ut_file_map.json"
-    ut_file_map_full = {}
     with open(ut_file_map_new, 'r') as load_f:
         load_dict_new = json.load(load_f)
     with open(ut_file_map_old, 'r') as f:
         load_dict_old = json.load(f)
 
-    for filename in load_dict_new:
-        ut_file_map_full[filename] = load_dict_new[filename]
-        if filename in load_dict_old:
-            for ut in load_dict_old[filename]:
-                if ut not in ut_file_map_full[filename]:
-                    ut_file_map_full[filename].append(ut)
+    all_uts_paddle = '%s/build/all_uts_paddle' % rootPath
+    with open(all_uts_paddle, 'r') as f:
+        all_uts_paddle_list = []
+        for ut in f.readlines():
+            all_uts_paddle_list.append(ut.strip())
+        f.close()
 
     for filename in load_dict_old:
         if filename not in load_dict_new:
-            ut_file_map_full[filename] = load_dict_old[filename]
+            if filename.endswith(('.h')):
+                load_dict_new[filename] = []
+            else:
+                load_dict_new[filename] = load_dict_old[filename]
 
     with open("/pre_test/ut_file_map.json", "w") as f:
-        json.dump(ut_file_map_full, f, indent=4)
-        print("ut_file_map_full success!!")
+        json.dump(load_dict_new, f, indent=4)
+        print("load_dict_new success!!")
 
-    all_uts_paddle = '%s/build/all_uts_paddle' % rootPath
-    with open(all_uts_paddle, 'r') as f:
-        all_uts_paddle_list = f.readlines()
-        f.close()
     os.system(
         'cd /pre_test && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta --no-check-certificate'
     )
     prec_delta_old = '/pre_test/prec_delta'
     prec_delta_new = "%s/build/prec_delta" % rootPath
     with open(prec_delta_old, 'r') as f:
-        prec_delta_old_list = f.readlines()
+        prec_delta_old_list = []
+        for ut in f.readlines():
+            prec_delta_old_list.append(ut.strip())
         f.close()
     with open(prec_delta_new, 'r') as f:
-        prec_delta_new_list = f.readlines()
+        prec_delta_new_list = []
+        for ut in f.readlines():
+            prec_delta_new_list.append(ut.strip())
         f.close()
     for ut in prec_delta_old_list:
-        if ut not in prec_delta_new_list and ut not in all_uts_paddle_list:
+        filename = '%s/build/ut_map/%s/coverage.info.tmp' % (rootPath, ut)
+        if ut in all_uts_paddle_list:
+            if not os.path.exists(filename) and ut not in prec_delta_new_list:
                 prec_delta_new_list.append(ut)
     prec_delta_file = open("/pre_test/prec_delta", 'w')
     for ut in prec_delta_new_list:
-        prec_delta_file.write(ut)
+        prec_delta_file.write(ut + '\n')
+    print("prec_delta_file success!!")
     prec_delta_file.close()
 
 

tools/handle_h_cu_file.py

@@ -66,6 +66,19 @@ def insert_pile_to_h_file(rootPath):
         os.system('echo "\n#endif" >> %s' % line)
 
 
+def remove_pile_from_h_file(rootPath):
+    h_cu_files = '%s/tools/h_cu_files.log' % rootPath
+    f = open(h_cu_files)
+    lines = f.readlines()
+    count = 12
+    for line in lines:
+        line = line.strip()
+        while count > 0:
+            os.system("sed -i '$d' %s" % line)
+            count = count - 1
+        count = 12
+
+
 def get_h_cu_file(file_path):
     rootPath = file_path[0]
     dir_path = file_path[1]
@@ -110,3 +123,6 @@ if __name__ == "__main__":
         dir_path = sys.argv[2]
         rootPath = sys.argv[3]
         main(rootPath, dir_path)
+    elif func == 'remove_pile_from_h_file':
+        rootPath = sys.argv[2]
+        remove_pile_from_h_file(rootPath)

tools/pyCov_multithreading.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import commands
 from xml.etree import ElementTree
 import re
 import time
@@ -51,7 +50,7 @@ def getPyCovResult(params):
     os.system('cd %s && coverage combine `ls python-coverage.data.*`' % path)
     os.system('cd %s && pwd && coverage xml -i -o python-coverage.xml' % path)
     xml_path = '%s/python-coverage.xml' % path
-    os.system("python %s/tools/analysisPyXml.py %s %s" %
+    os.system("python2.7 %s/tools/analysisPyXml.py %s %s" %
               (rootPath, rootPath, ut))
     endTime = int(time.time())
     print('pyCov Time: %s' % (endTime - startTime))