support coverage CI on x86 server (#3632)

* test=develop,test=coverage, support coverage CI on x86 server

* test=develop,test=coverage, fix the conflict

* test=develop,test=coverage, add target code_coverage

* test=develop,test=coverage, modify the coveralls.cmake

CMakeLists.txt

@@ -221,6 +221,7 @@ endif()
 if(LITE_WITH_MLU)
     include(mlu)
 endif()
+include(coveralls)
 
 include(external/mklml)     # download mklml package
 include(external/xbyak)     # download xbyak package

cmake/coveralls.cmake

@@ -20,6 +20,9 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
     # will be converted from the format "1;2;3" to "1 2 3".
     set(COVERAGE_SRCS "")
     foreach (SINGLE_SRC ${_COVERAGE_SRCS})
+        if ("${SINGLE_SRC}" MATCHES "/Paddle-Lite/third-party/*")
+            continue()
+        endif()
         set(COVERAGE_SRCS "${COVERAGE_SRCS}*${SINGLE_SRC}")
     endforeach()
 
@@ -62,7 +65,7 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
 endfunction()
 
 if(WITH_COVERAGE)
-    set(CMAKE_BUILD_TYPE "Debug")
+    #set(CMAKE_BUILD_TYPE "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
 
@@ -95,9 +98,11 @@ if(WITH_COVERAGE)
         set(PADDLE_SRCS "${PADDLE_SRCS};${PROJECT_SOURCE_DIR}/${PADDLE_SRC}")
     endforeach()
 
+    set(COVERALLS_UPLOAD ON)
     code_coverage(
         "${PADDLE_SRCS}"
         ${COVERALLS_UPLOAD}
         "${PROJECT_SOURCE_DIR}/cmake"
     )
 endif()
+

lite/api/CMakeLists.txt

@@ -174,6 +174,7 @@ set(LITE_DEMO_INSTALL_DIR "${THIRD_PARTY_PATH}/inference_demo" CACHE STRING
         "A path setting inference demo download directories.")
 
 if(WITH_TESTING)
+    if(NOT WITH_COVERAGE)
         lite_cc_test(test_cxx_api SRCS cxx_api_test.cc
            DEPS cxx_api mir_passes lite_api_test_helper
            ${ops} ${host_kernels}
@@ -193,6 +194,7 @@ if(WITH_TESTING)
            ARGS --model_dir=${LITE_MODEL_DIR}/lite_naive_model
                 --optimized_model=${LITE_MODEL_DIR}/lite_naive_model_opt SERIAL)
         add_dependencies(test_cxx_api extern_lite_download_lite_naive_model_tar_gz)
+    endif()
     if(NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK)
         if(LITE_WITH_X86)
             lite_cc_test(test_googlenet SRCS test_googlenet_lite.cc
@@ -331,7 +333,8 @@ bundle_static_library(paddle_api_light paddle_api_light_bundled bundle_light_api
 
 # These tests needs CLI arguments, and is not supported in ARM CI.
 # TODO(Superjomn) support latter.
-lite_cc_test(test_light_api SRCS light_api_test.cc
+if(NOT WITH_COVERAGE)
+    lite_cc_test(test_light_api SRCS light_api_test.cc
         DEPS light_api program mir_passes paddle_api_light
         CL_DEPS ${opencl_kernels}
         FPGA_DEPS ${fpga_kernels}
@@ -339,7 +342,7 @@ lite_cc_test(test_light_api SRCS light_api_test.cc
         BM_DEPS ${bm_kernels}
         ARGS --optimized_model=${LITE_MODEL_DIR}/lite_naive_model_opt SERIAL)
 
-lite_cc_test(test_apis SRCS apis_test.cc
+    lite_cc_test(test_apis SRCS apis_test.cc
         DEPS cxx_api light_api ${ops} paddle_api_light
         CL_DEPS ${opencl_kernels}
         X86_DEPS ${x86_kernels}
@@ -350,6 +353,7 @@ lite_cc_test(test_apis SRCS apis_test.cc
         MLU_DEPS ${mlu_kernels}
         ARGS --model_dir=${LITE_MODEL_DIR}/lite_naive_model
         --optimized_model=${LITE_MODEL_DIR}/lite_naive_model_opt SERIAL)
+endif()
 
 if (LITE_WITH_JAVA AND LITE_WITH_ARM)
     add_subdirectory(android)
@@ -375,7 +379,8 @@ if (LITE_ON_MODEL_OPTIMIZE_TOOL)
     add_dependencies(opt op_list_h kernel_list_h all_kernel_faked_cc supported_kernel_op_info_h)
 endif(LITE_ON_MODEL_OPTIMIZE_TOOL)
 
-lite_cc_test(test_paddle_api SRCS paddle_api_test.cc DEPS paddle_api_full paddle_api_light
+if(NOT WITH_COVERAGE)
+    lite_cc_test(test_paddle_api SRCS paddle_api_test.cc DEPS paddle_api_full paddle_api_light
       ${ops}
       ARM_DEPS ${arm_kernels}
       CV_DEPS paddle_cv_arm
@@ -389,8 +394,9 @@ lite_cc_test(test_paddle_api SRCS paddle_api_test.cc DEPS paddle_api_full paddle
       BM_DEPS ${bm_kernels}
       MLU_DEPS ${mlu_kernels}
       ARGS --model_dir=${LITE_MODEL_DIR}/lite_naive_model SERIAL)
-if (WITH_TESTING)
+    if (WITH_TESTING)
         add_dependencies(test_paddle_api extern_lite_download_lite_naive_model_tar_gz)
+    endif()
 endif()
 
 # Some bins

lite/model_parser/naive_buffer/CMakeLists.txt

@@ -13,7 +13,9 @@ set(naive_wrapper
     nb_op_desc nb_var_desc nb_param_desc nb_combined_params_desc
     nb_block_desc nb_program_desc PARENT_SCOPE)
 
-lite_cc_test(test_naive_buffer SRCS naive_buffer_test.cc DEPS naive_buffer)
+if(NOT WITH_COVERAGE)
+    lite_cc_test(test_naive_buffer SRCS naive_buffer_test.cc DEPS naive_buffer)
+endif()
 lite_cc_test(test_naive_buffer_wrapper SRCS naive_buffer_wrapper_test.cc 
              DEPS nb_op_desc nb_var_desc nb_param_desc nb_combined_params_desc 
              nb_block_desc nb_program_desc)

lite/tools/ci_build.sh

@@ -17,6 +17,7 @@ NUM_CORES_FOR_COMPILE=${LITE_BUILD_THREADS:-8}
 # global variables
 #whether to use emulator as adb devices,when USE_ADB_EMULATOR=ON we use emulator, else we will use connected mobile phone as adb devices.
 USE_ADB_EMULATOR=ON
+LITE_WITH_COVERAGE=OFF
 
 # if operating in mac env, we should expand the maximum file num
 os_nmae=`uname -s`
@@ -96,9 +97,14 @@ function check_need_ci {
     git log -1 --oneline | grep "test=develop" || exit -1
 }
 
+function check_coverage() {
+    bash ../tools/coverage/paddle_lite_coverage.sh
+}
+
 function cmake_x86 {
     prepare_workspace
-    cmake ..  -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON ${common_flags}
+    #cmake ..  -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON ${common_flags}
+    cmake ..  -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON  -DWITH_COVERAGE=$LITE_WITH_COVERAGE ${common_flags}
 }
 
 function cmake_opencl {
@@ -202,7 +208,7 @@ function build_opencl {
 function cmake_x86_for_CI {
     prepare_workspace # fake an empty __generated_code__.cc to pass cmake.
     cmake ..  -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON ${common_flags} -DLITE_WITH_PROFILE=ON -DWITH_MKL=ON \
-        -DLITE_BUILD_EXTRA=ON \
+        -DLITE_BUILD_EXTRA=ON -DWITH_COVERAGE=ON 
 
     # Compile and execute the gen_code related test, so it will generate some code, and make the compilation reasonable.
     # make test_gen_code -j$NUM_CORES_FOR_COMPILE
@@ -240,7 +246,9 @@ function build_single {
 
 function build {
     make lite_compile_deps -j$NUM_CORES_FOR_COMPILE
-
+    if [ $LITE_WITH_COVERAGE = "ON" ];then
+        make coveralls_generate -j	
+    fi 
     # test publish inference lib
     # make publish_inference
 }
@@ -1058,6 +1066,10 @@ function main {
                 USE_ADB_EMULATOR="${i#*=}"
                 shift
                 ;;
+            --lite_with_coverage=*)
+                LITE_WITH_COVERAGE="${i#*=}"
+                shift
+                ;;
             build)
                 build $TESTS_FILE
                 build $LIBS_FILE
@@ -1123,6 +1135,11 @@ function main {
                 build_test_server
                 shift
                 ;;
+            build_check_coverage)
+                build_test_server
+                check_coverage
+                shift
+                ;;
             build_test_xpu)
                 build_test_xpu
                 shift

tools/coverage/coverage_diff.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+usage: coverage_diff.py info_file diff_file > > coverage-diff.info
+"""
+
+import sys
+
+
+def get_diff_file_lines(diff_file):
+    """
+
+    :param diff_file:
+    :return:
+    """
+
+    diff_file_lines = {}
+
+    current_file = None
+    current_line = -1
+
+    with open(diff_file) as diff_file:
+        for line in diff_file:
+            line = line.strip()
+
+            if line.startswith('+++ '):
+                current_file = line.lstrip('+++ ')
+
+                diff_file_lines[current_file] = []
+
+                continue
+
+            elif line.startswith('@@ '):
+                current_line = line.split()[2]
+                current_line = current_line.lstrip('+').split(',')[0]
+                current_line = int(current_line)
+
+                continue
+
+            elif line.startswith('-'):
+                continue
+
+            elif line.startswith('+'):
+                diff_file_lines[current_file].append(current_line)
+
+            current_line += 1
+
+    return diff_file_lines
+
+
+def get_info_file_lines(info_file, diff_file):
+    """
+
+    :param info_file:
+    :param diff_file:
+    """
+
+    diff_file_lines = get_diff_file_lines(diff_file)
+    print diff_file_lines
+    
+    current_lines = []
+    current_lf = 0
+    current_lh = 0
+
+    with open(info_file) as info_file:
+        for line in info_file:
+            line = line.strip()
+
+            if line.startswith('SF:'):
+                current_file = line.lstrip('SF:')
+
+                if current_file.startswith('/Paddle-Lite/'):
+                    current_file = current_file[len('/Paddle-Lite/'):]
+
+                current_lines = diff_file_lines.get(current_file, [])
+
+            elif line.startswith('DA:'):
+                da = line.lstrip('DA:').split(',')
+
+                if int(da[0]) in current_lines:
+                    current_lf += 1
+
+                    if not line.endswith(',0'):
+                        current_lh += 1
+
+                    print(line)
+
+                continue
+
+            elif line.startswith('LF:'):
+                print 'LF:{}'.format(current_lf)
+
+                continue
+
+            elif line.startswith('LH:'):
+                print 'LH:{}'.format(current_lh)
+
+                continue
+
+            print(line)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        exit()
+
+    info_file = sys.argv[1]
+    diff_file = sys.argv[2]
+
+    get_info_file_lines(info_file, diff_file)

tools/coverage/coverage_lines.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+usage: coverage_lines.py info_file expected
+"""
+import os
+import sys
+
+
+def get_lines(info_file):
+    """
+
+    :param info_file:
+    :return:
+    """
+
+    hits = .0
+    total = .0
+
+    with open(info_file) as info_file:
+        for line in info_file:
+            line = line.strip()
+
+            if not line.startswith('DA:'):
+                continue
+
+            line = line[3:]
+
+            total += 1
+
+            if int(line.split(',')[1]) > 0:
+                hits += 1
+
+    if total == 0:
+        print 'no data found'
+        exit()
+
+    return hits / total
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        exit()
+
+    info_file = sys.argv[1]
+    expected = float(sys.argv[2])
+
+    if not os.path.isfile(info_file):
+        print 'info file {} is not exists, ignored'.format(info_file)
+        exit()
+
+    actual = get_lines(info_file)
+    actual = round(actual, 3)
+
+    if actual < expected:
+        print 'expected >= {} %, actual {} %, failed'.format(
+            round(expected * 100, 1),
+            round(actual * 100, 1))
+
+        exit(1)
+
+    print 'expected >= {} %, actual {} %, passed'.format(
+        round(expected * 100, 1),
+        round(actual * 100, 1))

tools/coverage/gcda_clean.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+usage: gcda_clean.py pull_id
+"""
+
+import os
+import sys
+
+from github import Github
+
+token = os.getenv('GITHUB_API_TOKEN', 'e51cb020919a6eef689257966e8fb6477981788a')
+
+
+def get_pull(pull_id):
+    """
+
+    :param pull_id:
+    :return: pull
+    """
+
+    github = Github(token, timeout=60)
+    repo = github.get_repo('PaddlePaddle/Paddle-Lite')
+    pull = repo.get_pull(pull_id)
+
+    return pull
+
+
+def get_files(pull_id):
+    """
+
+    :param args:
+    """
+
+    pull = get_pull(pull_id)
+
+    for file in pull.get_files():
+        yield file.filename
+
+
+def clean(pull_id):
+    """
+
+    :param pull_id:
+    :return:
+    """
+
+    changed = []
+
+    for file in get_files(pull_id):
+        changed.append('/Paddle-Lite/build/{}.gcda'.format(file))
+
+    for parent, dirs, files in os.walk('/Paddle-Lite/build/'):
+        for gcda in files:
+            if gcda.endswith('.gcda'):
+                trimmed = parent
+
+                # convert paddle/fluid/imperative/CMakeFiles/layer.dir/layer.cc.gcda
+                # to paddle/fluid/imperative/layer.cc.gcda
+
+                if trimmed.endswith('.dir'):
+                    trimmed = os.path.dirname(trimmed)
+
+                if trimmed.endswith('CMakeFiles'):
+                    trimmed = os.path.dirname(trimmed)
+
+                # remove no changed gcda
+
+                if os.path.join(trimmed, gcda) not in changed:
+                    gcda = os.path.join(parent, gcda)
+                    os.remove(gcda)
+
+
+if __name__ == '__main__':
+    pull_id = sys.argv[1]
+    pull_id = int(pull_id)
+
+    clean(pull_id)

tools/coverage/paddle_lite_coverage.sh

+et -xe
+
+PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
+
+# install lcov
+curl -o /lcov-1.14.tar.gz -s https://paddle-ci.gz.bcebos.com/coverage%2Flcov-1.14.tar.gz
+tar -xf /lcov-1.14.tar.gz -C /
+cd /lcov-1.14
+make install
+
+# run paddle coverage
+cd /Paddle-Lite/build
+
+python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID}
+
+lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0
+
+# full html report
+function gen_full_html_report() {
+    lcov --extract coverage.info \
+        '/Paddle-Lite/lite/api/*' \
+        '/Paddle-Lite/lite/backends/*' \
+        '/Paddle-Lite/lite/core/*' \
+        '/Paddle-Lite/lite/fluid/*' \
+        '/Paddle-Lite/lite/gen_code/*' \
+        '/Paddle-Lite/lite/kernels/*' \
+        '/Paddle-Lite/lite/model_parser/*' \
+        '/Paddle-Lite/lite/opreators/*' \
+        '/Paddle-Lite/lite/tests/*' \
+        '/Paddle-Lite/lite/tools/*' \
+        '/Paddle-Lite/lite/utils/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/Paddle-Lite/lite/demo*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+gen_full_html_report || true
+
+# diff html report
+function gen_diff_html_report() {
+    if [ "${GIT_PR_ID}" != "" ]; then
+        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+
+        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
+    fi
+
+    lcov --extract coverage-full.info \
+        ${COVERAGE_DIFF_PATTERN} \
+        -o coverage-diff.info \
+        --rc lcov_branch_coverage=0
+    
+    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
+
+    mv -f coverage-diff.tmp coverage-diff.info
+
+    genhtml -o coverage-diff -t 'Diff Coverage' --no-function-coverage --no-branch-coverage coverage-diff.info
+}
+
+gen_diff_html_report || true
+
+## python coverage
+#export COVERAGE_FILE=/Paddle-Lite/build/python-coverage.data
+#
+#set +x
+#coverage combine `ls python-coverage.data.*`
+#set -x
+#
+#coverage xml -i -o python-coverage.xml
+#
+#python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info
+#
+## python full html report
+##
+#function gen_python_full_html_report() {
+#    lcov --extract python-coverage.info \
+#        '/Paddle-Lite/python/*' \
+#        -o python-coverage-full.tmp \
+#        --rc lcov_branch_coverage=0
+#
+#    mv -f python-coverage-full.tmp python-coverage-full.info
+#
+#    lcov --remove python-coverage-full.info \
+#        '/*/tests/*' \
+#        -o python-coverage-full.tmp \
+#        --rc lcov_branch_coverage=0
+#
+#    mv -f python-coverage-full.tmp python-coverage-full.info
+#}
+#
+#gen_python_full_html_report || true
+#
+## python diff html report
+#function gen_python_diff_html_report() {
+#    if [ "${GIT_PR_ID}" != "" ]; then
+#        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+#
+#        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
+#    fi
+#
+#    lcov --extract python-coverage-full.info \
+#        ${COVERAGE_DIFF_PATTERN} \
+#        -o python-coverage-diff.info \
+#        --rc lcov_branch_coverage=0
+#
+#    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
+#
+#    mv -f python-coverage-diff.tmp python-coverage-diff.info
+#
+#    genhtml -o python-coverage-diff \
+#        -t 'Python Diff Coverage' \
+#        --no-function-coverage \
+#        --no-branch-coverage \
+#        --ignore-errors source \
+#        python-coverage-diff.info
+#}
+#
+#gen_python_diff_html_report || true
+
+# assert coverage lines
+echo "Assert Diff Coverage"
+python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
+
+#echo "Assert Python Diff Coverage"
+#python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
+
+#if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then
+if [ "$COVERAGE_LINES_ASSERT" = "1" ]; then
+    exit 9
+fi

tools/coverage/pull_request.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+usage: pull_request.py files pull_id
+       pull_request.py diff  pull_id
+"""
+
+import argparse
+import os
+
+from github import Github
+
+token = os.getenv('GITHUB_API_TOKEN', 'e51cb020919a6eef689257966e8fb6477981788a')
+
+
+def get_pull(pull_id):
+    """
+
+    :param pull_id:
+    :return: pull
+    """
+
+    github = Github(token, timeout=60)
+    repo = github.get_repo('PaddlePaddle/Paddle-Lite')
+    pull = repo.get_pull(pull_id)
+
+    return pull
+
+
+def get_files(args):
+    """
+
+    :param args:
+    """
+
+    pull = get_pull(args.pull_id)
+
+    for file in pull.get_files():
+        print '/Paddle-Lite/{}'.format(file.filename)
+
+
+def diff(args):
+    """
+
+    :param args:
+    """
+
+    pull = get_pull(args.pull_id)
+
+    for file in pull.get_files():
+        print '+++ {}'.format(file.filename)
+        print file.patch
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    files_parser = subparsers.add_parser('files')
+    files_parser.add_argument('pull_id', type=int)
+    files_parser.set_defaults(func=get_files)
+
+    diff_parser = subparsers.add_parser('diff')
+    diff_parser.add_argument('pull_id', type=int)
+    diff_parser.set_defaults(func=diff)
+
+    args = parser.parse_args()
+    args.func(args)