Add eager jit dir (#49369)

* add_eager_and_jit,test=ljd_test

* test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* add_eager_jit_dir,test=ljd_test

* fix conflict,test=ljd_test

* test,test=ljd_test

* get new precise_map,test=ljd_test

* test

paddle/scripts/paddle_build.sh

@@ -1861,12 +1861,12 @@ function precise_card_test_single {
             mkdir ${PADDLE_ROOT}/build/ut_map/$case
         fi
         set -x
-        find paddle/fluid -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case
-        find paddle/phi -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case
-        find paddle/utils -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case
-        find paddle/phi -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case
-        find paddle/utils -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case
-        find paddle/fluid -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case
+        find paddle/fluid -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
+        find paddle/phi -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
+        find paddle/utils -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
+        find paddle/phi -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
+        find paddle/utils -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
+        find paddle/fluid -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
         python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case &
 
         # python
@@ -1953,7 +1953,7 @@ function precise_card_test() {
     echo "****************************************************************"
 
     tmpfile=$tmp_dir/$testcases".log"
-    env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile
+    env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" -E "($disable_ut_quickly)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile
     set +m
 }
 
@@ -2010,7 +2010,7 @@ set +x
                 fi
             else
                 single_card_test_num=$(($single_card_test_num+1))
-                if [[ $single_card_test_num -gt 3000 ]];then
+                if [[ $single_card_test_num -gt 1000 ]];then
                     if [[ "$single_card_tests_1" == "" ]]; then
                         single_card_tests_1="^$testcase$"
                     else
@@ -2034,14 +2034,19 @@ set +x
 set -x
     mkdir -p ${PADDLE_ROOT}/build/ut_map
     mkdir -p ${PADDLE_ROOT}/build/pytest
-    #run all unittest to get the coverage information of .c and .h files
+    #get disabled tests,not run these disabled ut
+    get_quickly_disable_ut||disable_ut_quickly='disable_ut'
+
+    # run all unittest to get the coverage information of .c and .h files
     precise_card_test_single "^simple_precision_test$" 1
     wait;
     precise_card_test_single "$single_card_tests" 1
+    wait;
     precise_card_test_single "$single_card_tests_1" 1
     precise_card_test_single "$multiple_card_tests" 2
     precise_card_test_single "$exclusive_tests"
     wait;
+    
     #get notSuccessut including the failed uniitests and not executed unittests
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT}
 

tools/get_single_test_cov.py

@@ -17,6 +17,7 @@ import os
 import re
 import subprocess
 import sys
+import time
 
 
 def getFNDAFile(rootPath, test):
@@ -168,25 +169,28 @@ def getCovinfo(rootPath, test):
     ut_map_path = '%s/build/ut_map/%s' % (rootPath, test)
     print("start get fluid ===>")
     cmd_fluid = (
-        'cd %s && lcov --capture -d ./paddle/fluid/ -o ./paddle/fluid/coverage_fluid.info --rc lcov_branch_coverage=0'
+        'cd %s && lcov --capture -d paddle/fluid/ -o paddle/fluid/coverage_fluid.info --rc lcov_branch_coverage=0'
         % ut_map_path
     )
     p_fluid = subprocess.Popen(cmd_fluid, shell=True, stdout=subprocess.DEVNULL)
 
     print("start get phi ===>")
     cmd_phi = (
-        'cd %s && lcov --capture -d ./paddle/phi -o ./paddle/phi/coverage_phi.info --rc lcov_branch_coverage=0'
+        'cd %s && lcov --capture -d paddle/phi -o paddle/phi/coverage_phi.info --rc lcov_branch_coverage=0'
         % ut_map_path
     )
-    p_phi = subprocess.Popen(cmd_phi, shell=True, stdout=subprocess.DEVNULL)
+    if os.path.exists("%s/paddle/phi" % ut_map_path):
+        p_phi = subprocess.Popen(cmd_phi, shell=True, stdout=subprocess.DEVNULL)
 
     print("start get utils ===>")
     cmd_utils = (
-        'cd %s && lcov --capture -d ./paddle/utils -o ./paddle/utils/coverage_utils.info --rc lcov_branch_coverage=0'
+        'cd %s && lcov --capture -d paddle/utils -o paddle/utils/coverage_utils.info --rc lcov_branch_coverage=0'
         % ut_map_path
     )
-    p_utils = subprocess.Popen(cmd_utils, shell=True, stdout=subprocess.DEVNULL)
-
+    if os.path.exists("%s/paddle/utils" % ut_map_path):
+        p_utils = subprocess.Popen(
+            cmd_utils, shell=True, stdout=subprocess.DEVNULL
+        )
     print("start wait fluid ===>")
     p_fluid.wait()
     print("start wait phi ===>")
@@ -194,10 +198,22 @@ def getCovinfo(rootPath, test):
     print("start wait utils ===>")
     p_utils.wait()
     print("end wait...")
-    os.system(
-        'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -a paddle/utils/coverage_utils.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
-        % ut_map_path
+    coverage_utils_info_path = (
+        "%s/paddle/utils/coverage_utils.info" % ut_map_path
     )
+    if (
+        os.path.exists(coverage_utils_info_path)
+        and os.path.getsize(coverage_utils_info_path) != 0
+    ):
+        os.system(
+            'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -a paddle/utils/coverage_utils.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
+            % ut_map_path
+        )
+    else:
+        os.system(
+            'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
+            % ut_map_path
+        )
     coverage_info_path = ut_map_path + '/coverage.info'
     file_size = os.path.getsize(coverage_info_path)
     if file_size == 0:
@@ -206,7 +222,7 @@ def getCovinfo(rootPath, test):
     else:
         print("get coverage.info succesfully")
     os.system(
-        "cd %s && lcov --extract coverage.info '/paddle/paddle/phi/*' '/paddle/paddle/utils/*' '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1"
+        "cd %s && lcov --extract coverage.info '/paddle/paddle/phi/*' '/paddle/paddle/utils/*' '/paddle/paddle/fluid/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1"
         % ut_map_path
     )
     coverage_info_tmp = ut_map_path + '/coverage.info.tmp'
@@ -222,12 +238,24 @@ def getCovinfo(rootPath, test):
     if test == "simple_precision_test":
         getBaseFnda(rootPath, test)
     else:
+        start_getFNDAFile = time.time()
         getFNDAFile(rootPath, test)
+        end_getFNDAFile = time.time()
+        print("getFNDAFile time:", end_getFNDAFile - start_getFNDAFile)
+        start_analysisFNDAFile = time.time()
         analysisFNDAFile(rootPath, test)
+        end_analysisFNDAFile = time.time()
+        print(
+            "analysisFNDAFile time :",
+            end_analysisFNDAFile - start_analysisFNDAFile,
+        )
     os.system('rm -rf %s/coverage.info.tmp' % ut_map_path)
 
 
 if __name__ == "__main__":
     rootPath = sys.argv[1]
     case = sys.argv[2]
+    start_getCovinfo = time.time()
     getCovinfo(rootPath, case)
+    end_getCovinfo = time.time()
+    print("getConvinfo time :", end_getCovinfo - start_getCovinfo)

tools/get_ut_file_map.py

@@ -145,8 +145,11 @@ def notsuccessfuc(rootPath):
     ut_map_path = "%s/build/ut_map" % rootPath
     files = os.listdir(ut_map_path)
     count = 0
+
     # ut failed!!
     for ut in files:
+        if ut == 'simple_precise_test':
+            continue
         coverage_info = '%s/%s/fnda.tmp' % (ut_map_path, ut)
         if os.path.exists(coverage_info):
             pass
@@ -175,15 +178,15 @@ def notsuccessfuc(rootPath):
 
 def ut_file_map_supplement(rootPath):
     ut_file_map_new = "%s/build/ut_file_map.json" % rootPath
-    os.system('mkdir /pre_test_tmp')
+    precision_test_map_store_dir = "/precision_test_map_store"
+    os.system('mkdir %s' % precision_test_map_store_dir)
     os.system(
-        'cd /pre_test_tmp && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json --no-check-certificate'
+        'cd %s && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/tmp_test/ut_file_map.json --no-check-certificate'
+        % precision_test_map_store_dir
     )
-    ut_file_map_old = "/pre_test_tmp/ut_file_map.json"
+    ut_file_map_old = "%s/ut_file_map.json" % precision_test_map_store_dir
     with open(ut_file_map_new, 'r') as load_f:
         load_dict_new = json.load(load_f)
-    with open(ut_file_map_old, 'r') as f:
-        load_dict_old = json.load(f)
 
     all_uts_paddle = '%s/build/all_uts_paddle' % rootPath
 
@@ -193,35 +196,24 @@ def ut_file_map_supplement(rootPath):
             all_uts_paddle_list.append(ut.strip())
         f.close()
 
-    with open("/pre_test_tmp/ut_file_map.json", "w") as f:
+    with open("%s/ut_file_map.json" % precision_test_map_store_dir, "w") as f:
         json.dump(load_dict_new, f, indent=4)
         print("load_dict_new success!!")
 
     os.system(
-        'cd /pre_test_tmp && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta --no-check-certificate'
+        'cd %s && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/tmp_test/prec_delta --no-check-certificate'
+        % precision_test_map_store_dir
     )
-    prec_delta_old = '/pre_test_tmp/prec_delta'
     prec_delta_new = "%s/build/prec_delta" % rootPath
-    with open(prec_delta_old, 'r') as f:
-        prec_delta_old_list = []
-        for ut in f.readlines():
-            prec_delta_old_list.append(ut.strip())
-        f.close()
     with open(prec_delta_new, 'r') as f:
         prec_delta_new_list = []
         for ut in f.readlines():
             prec_delta_new_list.append(ut.strip())
         f.close()
-
-    for ut in prec_delta_old_list:
-        filename = '%s/build/ut_map/%s/fnda.tmp' % (rootPath, ut)
-        if ut in all_uts_paddle_list:
-            if not os.path.exists(filename) and ut not in prec_delta_new_list:
-                prec_delta_new_list.append(ut)
     prec_delta_new_list.append(
         'test_py_reader_error_msg'
     )  # add a python case for pycoverage
-    prec_delta_file = open("/pre_test_tmp/prec_delta", 'w')
+    prec_delta_file = open("%s/prec_delta" % precision_test_map_store_dir, 'w')
     for ut in prec_delta_new_list:
         prec_delta_file.write(ut + '\n')
     print("prec_delta_file success!!")

tools/handle_h_cu_file.py

@@ -66,7 +66,7 @@ def insert_pile_to_h_file(rootPath):
             % (func, line)
         )
         os.system(
-            'echo \'    printf("precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s'
+            'echo \'    fprintf(stderr,"precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s'
             % line
         )
         os.system('echo "\n#endif" >> %s' % line)