From 87dbc76866412dbb4291de89e9105bdbda0d2555 Mon Sep 17 00:00:00 2001 From: Chunwei Date: Sun, 16 Jun 2019 14:20:58 +0000 Subject: [PATCH] make lite CI compile faster --- .gitlab-ci.yml | 9 +++-- paddle/fluid/lite/CMakeLists.txt | 10 +++++- paddle/fluid/lite/gen_code/CMakeLists.txt | 12 +++---- .../lite/kernels/arm/split_compute_test.cc | 1 + paddle/fluid/lite/tools/build.sh | 33 ++++++++++--------- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01dc646326..a2f0644f21 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ before_script: - env + - export CI_USER_DIR=$(pwd) image: $SERVER_LITE_DOCKER_IMAGE @@ -14,6 +15,8 @@ check:prebuilt: stage: ci script: - rm -rf ~/.pip + - export http_proxy=$CI_PROXY + - export https_proxy=$CI_PROXY - pip install pre-commit - pre-commit install @@ -27,7 +30,7 @@ check:prebuilt: cache: key: check_style paths: - - /root/.cache + - $CI_USER_DIR/.cache build:server: tags: @@ -38,7 +41,7 @@ build:server: key: server_thirdparty paths: - build/third_party - - /root/.ccache + - ~/.ccache script: - apt install ccache - export http_proxy=$CI_PROXY @@ -73,7 +76,7 @@ build:mobile: paths: - $MOBILE_LITE_CACHE0 - $MOBILE_LITE_CACHE1 - - /root/.ccache + - ~/.ccache script: - apt install ccache - export http_proxy=$CI_PROXY diff --git a/paddle/fluid/lite/CMakeLists.txt b/paddle/fluid/lite/CMakeLists.txt index 978fb0eec8..fefc73c754 100644 --- a/paddle/fluid/lite/CMakeLists.txt +++ b/paddle/fluid/lite/CMakeLists.txt @@ -80,6 +80,9 @@ function (lite_deps TARGET) endfunction() +# A fake target to include all the libraries and tests the lite module depends. +add_custom_target(lite_compile_deps COMMAND echo 1) + # Add names for lite libraries for latter compile. We use this name list to avoid compiling # the whole fluid project to accelerate the compile speed. set(offline_lib_registry_file "${CMAKE_BINARY_DIR}/lite_libs.txt") @@ -112,6 +115,9 @@ function(lite_cc_library TARGET) cc_library(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS}) + # collect targets need to compile for lite + add_dependencies(lite_compile_deps ${TARGET}) + # register a library name. file(APPEND ${offline_lib_registry_file} "${TARGET}\n") endfunction() @@ -160,6 +166,9 @@ function(lite_cc_test TARGET) ) _lite_cc_test(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ARGS ${args_ARGS}) file(APPEND ${offline_test_registry_file} "${TARGET}\n") + + # collect targets need to compile for lite + add_dependencies(lite_compile_deps ${TARGET}) endfunction() add_subdirectory(operators) @@ -173,4 +182,3 @@ add_subdirectory(model_parser) add_subdirectory(utils) add_subdirectory(api) add_subdirectory(gen_code) - diff --git a/paddle/fluid/lite/gen_code/CMakeLists.txt b/paddle/fluid/lite/gen_code/CMakeLists.txt index d6e447a259..9c7e895ed5 100644 --- a/paddle/fluid/lite/gen_code/CMakeLists.txt +++ b/paddle/fluid/lite/gen_code/CMakeLists.txt @@ -18,11 +18,11 @@ if (NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK) DEPS scope_lite op_lite kernel_lite paddle_infer_gencode ) - # lite_cc_test(test_generated_code SRCS generated_code_test.cc DEPS __generated_code__ - # ${ops_lite} ${host_kernels} - # X86_DEPS ${x86_kernels} - # ) + lite_cc_test(test_generated_code SRCS generated_code_test.cc DEPS __generated_code__ + ${ops_lite} ${host_kernels} + X86_DEPS ${x86_kernels} + ) - # add_dependencies(__generated_code__ test_gen_code_lite) + add_dependencies(__generated_code__ test_gen_code_lite) + add_dependencies(__generated_code__ extern_lite_download_lite_naive_model_tar_gz) endif() - diff --git a/paddle/fluid/lite/kernels/arm/split_compute_test.cc b/paddle/fluid/lite/kernels/arm/split_compute_test.cc index 39632bee8d..35e2c7cded 100644 --- a/paddle/fluid/lite/kernels/arm/split_compute_test.cc +++ b/paddle/fluid/lite/kernels/arm/split_compute_test.cc @@ -14,6 +14,7 @@ #include "paddle/fluid/lite/kernels/arm/split_compute.h" #include +#include #include #include #include "paddle/fluid/lite/core/op_registry.h" diff --git a/paddle/fluid/lite/tools/build.sh b/paddle/fluid/lite/tools/build.sh index 392e9b82bb..a02cdc0385 100755 --- a/paddle/fluid/lite/tools/build.sh +++ b/paddle/fluid/lite/tools/build.sh @@ -6,6 +6,8 @@ LIBS_FILE="./lite_libs.txt" readonly common_flags="-DWITH_LITE=ON -DLITE_WITH_LIGHT_WEIGHT_FRAMEWORK=OFF -DWITH_PYTHON=OFF -DWITH_TESTING=ON -DLITE_WITH_ARM=OFF" +NUM_CORES_FOR_COMPILE=8 + # for code gen, a source file is generated after a test, but is dependended by some targets in cmake. # here we fake an empty file to make cmake works. function prepare_for_codegen { @@ -23,9 +25,17 @@ function cmake_x86 { cmake .. -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON ${common_flags} } +# This method is only called in CI. function cmake_x86_for_CI { - prepare_for_codegen + prepare_for_codegen # fake an empty __generated_code__.cc to pass cmake. cmake .. -DWITH_GPU=OFF -DWITH_MKLDNN=OFF -DLITE_WITH_X86=ON ${common_flags} -DLITE_WITH_PROFILE=ON + + # Compile and execute the gen_code related test, so it will generate some code, and make the compilation reasonable. + make test_gen_code_lite -j$NUM_CORES_FOR_COMPILE + make test_cxx_api_lite -j$NUM_CORES_FOR_COMPILE + ctest -R test_cxx_api_lite + ctest -R test_gen_code_lite + make test_generated_code -j$NUM_CORES_FOR_COMPILE } function cmake_gpu { @@ -61,14 +71,11 @@ function cmake_arm { function build_single { #make $1 -j$(expr $(nproc) - 2) - make $1 -j8 + make $1 -j$NUM_CORES_FOR_COMPILE } function build { - file=$1 - for _test in $(cat $file); do - build_single $_test - done + make lite_compile_deps -j $NUM_CORES_FOR_COMPILE } # It will eagerly test all lite related unittests. @@ -77,10 +84,6 @@ function test_lite { echo "file: ${file}" for _test in $(cat $file); do - # We move the build phase here to make the 'gen_code' test compiles after the - # corresponding test is executed and the C++ code generates. - #make $_test -j$(expr $(nproc) - 2) - make $_test -j8 ctest -R $_test -V done } @@ -91,10 +94,8 @@ function build_test_server { cd ./build export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/paddle/build/third_party/install/mklml/lib" cmake_x86_for_CI - # compile the tests and execute them. + build test_lite $TESTS_FILE - # build the remaining libraries to check compiling error. - build $LIBS_FILE } # test_arm_android @@ -135,6 +136,8 @@ function build_test_arm { echo -ne '\n' | ${ANDROID_HOME}/emulator/emulator -avd paddle-armv7 -noaudio -no-window -gpu off -verbose -port ${port_armv7} & sleep 1m + cur_dir=$(pwd) + for os in "android" "armlinux" ; do for abi in "arm64-v8a" "armeabi-v7a" "armeabi-v7a-hf" ; do # TODO(TJ): enable compile on v7-hf on andorid and all v7 on armlinux @@ -148,9 +151,10 @@ function build_test_arm { continue fi - build_dir=build.lite.${os}.${abi} + build_dir=$cur_dir/build.lite.${os}.${abi} mkdir -p $build_dir cd $build_dir + cmake_arm ${os} ${abi} build $TESTS_FILE @@ -179,7 +183,6 @@ function build_test_arm { test_arm_android $_test $port done fi - cd - done done adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done -- GitLab