From c9fd973d86d616d654c9951e8538bdb2a8d7943a Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 19 Jun 2019 09:51:38 +0000 Subject: [PATCH] speedup and split ci --- CMakeLists.txt | 4 +- .../kernels/arm/elementwise_compute_test.cc | 29 +++ .../lite/kernels/arm/softmax_compute_test.cc | 9 +- paddle/fluid/lite/tools/build.sh | 221 +++++++++++------- 4 files changed, 179 insertions(+), 84 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ac1f7d769..ae84f8eb31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,9 +44,9 @@ if(LITE_WITH_LIGHT_WEIGHT_FRAMEWORK) # check arch abi if(NOT DEFINED ARM_TARGET_LANG) - set(ARM_TARGET_LANG "clang" CACHE STRING "Choose ARM Target Language") + set(ARM_TARGET_LANG "gcc" CACHE STRING "Choose ARM Target Language") endif() - set(ARM_TARGET_LANG_LIST "gcc" "clang") + set(ARM_TARGET_LANG_LIST "gcc" "clang" "") set_property(CACHE ARM_TARGET_LANG PROPERTY STRINGS ${ARM_TARGET_LANG_LIST}) if (NOT ARM_TARGET_LANG IN_LIST ARM_TARGET_LANG_LIST) message(FATAL_ERROR "ARM_TARGET_LANG must be in one of ${ARM_TARGET_LANG_LIST}") diff --git a/paddle/fluid/lite/kernels/arm/elementwise_compute_test.cc b/paddle/fluid/lite/kernels/arm/elementwise_compute_test.cc index 4e242c8cc5..9f87e6628b 100644 --- a/paddle/fluid/lite/kernels/arm/elementwise_compute_test.cc +++ b/paddle/fluid/lite/kernels/arm/elementwise_compute_test.cc @@ -117,6 +117,19 @@ TEST(elementwise_add, compute) { operators::ElementwiseParam param; lite::Tensor x, y, output, output_ref; +#if 1 + for (auto n : {1, 3, 4}) { + for (auto c : {1, 3, 4}) { + for (auto h : {1, 3, 4}) { + for (auto w : {1, 3, 4}) { + for (auto axis : {-1, 0, 1, 3}) { + for (auto yd : + {std::vector({n}), std::vector({c}), + std::vector({h}), std::vector({w}), + std::vector({n, c}), std::vector({c, h}), + std::vector({c, h, w}), + std::vector({n, c, h, w})}) { +#else for (auto n : {1, 3, 4, 11}) { for (auto c : {1, 3, 4, 11}) { for (auto h : {1, 3, 4, 11}) { @@ -129,6 +142,7 @@ TEST(elementwise_add, compute) { std::vector({h, w}), std::vector({n, c, h}), std::vector({c, h, w}), std::vector({n, c, h, w})}) { +#endif auto x_dim = DDim(std::vector({n, c, h, w})); auto y_dim = DDim(yd); int axis_t = axis < 0 ? x_dim.size() - y_dim.size() : axis; @@ -192,6 +206,20 @@ TEST(fusion_elementwise_add_activation_arm, compute) { operators::FusionElementwiseActivationParam param; lite::Tensor x, y, output, output_ref; +#if 1 + for (auto act_type : {"relu"}) { + for (auto n : {1, 3, 4}) { + for (auto c : {1, 3, 4}) { + for (auto h : {1, 3, 4}) { + for (auto w : {1, 3, 4}) { + for (auto axis : {-1, 0, 1, 3}) { + for (auto yd : + {std::vector({n}), std::vector({c}), + std::vector({h}), std::vector({w}), + std::vector({n, c}), std::vector({h, w}), + std::vector({n, c, h}), + std::vector({n, c, h, w})}) { +#else for (auto act_type : {"relu"}) { for (auto n : {1, 3, 4, 11}) { for (auto c : {1, 3, 4, 11}) { @@ -206,6 +234,7 @@ TEST(fusion_elementwise_add_activation_arm, compute) { std::vector({n, c, h}), std::vector({c, h, w}), std::vector({n, c, h, w})}) { +#endif auto x_dim = DDim(std::vector({n, c, h, w})); auto y_dim = DDim(yd); int axis_t = axis < 0 ? x_dim.size() - y_dim.size() : axis; diff --git a/paddle/fluid/lite/kernels/arm/softmax_compute_test.cc b/paddle/fluid/lite/kernels/arm/softmax_compute_test.cc index 80a64f4eaf..a984a5b3ad 100644 --- a/paddle/fluid/lite/kernels/arm/softmax_compute_test.cc +++ b/paddle/fluid/lite/kernels/arm/softmax_compute_test.cc @@ -80,12 +80,19 @@ TEST(softmax_arm, compute) { lite::Tensor x; lite::Tensor output; lite::Tensor output_ref; - +#if 1 + for (auto n : {1, 3}) { + for (auto c : {1, 4}) { + for (auto h : {5, 1}) { + for (auto w : {1, 6}) { + for (auto axis : {-2, -1, 0, 1, 2}) { +#else for (auto n : {1, 3, 4, 11}) { for (auto c : {1, 3, 11, 4}) { for (auto h : {3, 1, 11, 4}) { for (auto w : {1, 3, 4, 12}) { for (auto axis : {-4, -3, -2, -1, 0, 1, 2, 3}) { +#endif x.Resize(DDim(std::vector({n, c, h, w}))); output.Resize(DDim(std::vector({n, c, h, w}))); output_ref.Resize(DDim(std::vector({n, c, h, w}))); diff --git a/paddle/fluid/lite/tools/build.sh b/paddle/fluid/lite/tools/build.sh index 29fa9d9ad0..70a23ecf69 100755 --- a/paddle/fluid/lite/tools/build.sh +++ b/paddle/fluid/lite/tools/build.sh @@ -54,22 +54,6 @@ function check_style { fi } -function cmake_arm { - # $1: ARM_TARGET_OS in "android" , "armlinux" - # $2: ARM_TARGET_ARCH_ABI in "armv8", "armv7" ,"armv7hf" - # $3: ARM_TARGET_LANG in "gcc" "clang" - cmake .. \ - -DWITH_GPU=OFF \ - -DWITH_MKL=OFF \ - -DWITH_LITE=ON \ - -DLITE_WITH_CUDA=OFF \ - -DLITE_WITH_X86=OFF \ - -DLITE_WITH_ARM=ON \ - -DLITE_WITH_LIGHT_WEIGHT_FRAMEWORK=ON \ - -DWITH_TESTING=ON \ - -DARM_TARGET_OS=$1 -DARM_TARGET_ARCH_ABI=$2 -DARM_TARGET_LANG=$3 -} - function build_single { #make $1 -j$(expr $(nproc) - 2) make $1 -j$NUM_CORES_FOR_COMPILE @@ -153,46 +137,102 @@ function test_arm_model { adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}" local adb_model_path="./${adb_work_dir}/`basename ${model_dir}`" adb -s emulator-${port} shell "./${adb_work_dir}/${test_name} --eval_model_dir=$adb_model_path" + } -# Build the code and run lite arm tests. This is executed in the CI system. -function build_test_arm { - # 1. Build goes first +function cmake_arm { + # $1: ARM_TARGET_OS in "android" , "armlinux" + # $2: ARM_TARGET_ARCH_ABI in "armv8", "armv7" ,"armv7hf" + # $3: ARM_TARGET_LANG in "gcc" "clang" + cmake .. \ + -DWITH_GPU=OFF \ + -DWITH_MKL=OFF \ + -DWITH_LITE=ON \ + -DLITE_WITH_CUDA=OFF \ + -DLITE_WITH_X86=OFF \ + -DLITE_WITH_ARM=ON \ + -DLITE_WITH_LIGHT_WEIGHT_FRAMEWORK=ON \ + -DWITH_TESTING=ON \ + -DARM_TARGET_OS=$1 -DARM_TARGET_ARCH_ABI=$2 -DARM_TARGET_LANG=$3 +} + +# $1: ARM_TARGET_OS in "android" , "armlinux" +# $2: ARM_TARGET_ARCH_ABI in "armv8", "armv7" ,"armv7hf" +# $3: ARM_TARGET_LANG in "gcc" "clang" +function build_arm { + os=$1 + abi=$2 + lang=$3 + cur_dir=$(pwd) - for lang in "gcc" "clang"; do - for os in "android" "armlinux" ; do - if [[ ${os} == "armlinux" && ${lang} == "clang" ]]; then - continue - fi - for abi in "armv8" "armv7" "armv7hf"; do - # TODO(hongming): enable compile armv7 and armv7hf on armlinux - if [[ ${abi} == "armv7hf" ]]; then - echo "armv7hf is not supported on both android and armlinux yet" - continue - fi - - # TODO(hongming): enable armv7 on armlinux - if [[ ${os} == "armlinux" && ${abi} == "armv7" ]]; then - echo "armv7 is not supported on armlinux yet" - continue - fi - - if [[ ${os} == "android" && ${abi} == "armv7hf" ]]; then - echo "android do not need armv7hf" - continue - fi - - build_dir=$cur_dir/build.lite.${os}.${abi}.${lang} - mkdir -p $build_dir - cd $build_dir - - cmake_arm ${os} ${abi} ${lang} - build $TESTS_FILE - done - done + if [[ ${os} == "armlinux" ]]; then + # TODO(hongming): enable compile armv7 and armv7hf on armlinux, and clang compile + if [[ ${lang} == "clang" ]]; then + echo "clang is not enabled on armlinux yet" + return 0 + fi + if [[ ${abi} == "armv7hf" ]]; then + echo "armv7hf is not supported on armlinux yet" + return 0 + fi + if [[ ${abi} == "armv7" ]]; then + echo "armv7 is not supported on armlinux yet" + return 0 + fi + fi + + if [[ ${os} == "android" && ${abi} == "armv7hf" ]]; then + echo "android do not need armv7hf" + return 0 + fi + + build_dir=$cur_dir/build.lite.${os}.${abi}.${lang} + mkdir -p $build_dir + cd $build_dir + + cmake_arm ${os} ${abi} ${lang} + build $TESTS_FILE +} + +# $1: ARM_TARGET_OS in "android" , "armlinux" +# $2: ARM_TARGET_ARCH_ABI in "armv8", "armv7" ,"armv7hf" +# $3: ARM_TARGET_LANG in "gcc" "clang" +# $4: android test port +# Note: test must be in build dir +function test_arm { + os=$1 + abi=$2 + lang=$3 + port=$4 + if [[ ${os} == "armlinux" ]]; then + # TODO(hongming): enable test armlinux on armv8, armv7 and armv7hf + echo "Skip test arm linux yet. armlinux must in another docker" + return 0 + fi + + if [[ ${os} == "android" && ${abi} == "armv7hf" ]]; then + echo "android do not need armv7hf" + return 0 + fi + + # TODO(yuanshuai): enable armv7 on android + if [[ ${abi} == "armv7" ]]; then + echo "skip android v7 test yet" + return 0 + fi + + echo "test file: ${TESTS_FILE}" + for _test in $(cat $TESTS_FILE); do + test_arm_android $_test $port done + # TODO(sangoly): refine this + test_arm_model "test_cxx_api_lite" $port "./third_party/install/mobilenet_v2_relu" +} - # 2. Then test +# Build the code and run lite arm tests. This is executed in the CI system. +function build_test_arm { + ######################################################################## + # job 1-4 must be in one runner port_armv8=5554 port_armv7=5556 @@ -206,39 +246,46 @@ function build_test_arm { echo -ne '\n' | ${ANDROID_HOME}/emulator/emulator -avd paddle-armv7 -noaudio -no-window -gpu off -verbose -port ${port_armv7} & sleep 1m - # now can only test android. - for lang in "gcc" "clang"; do - for abi in "armv8" "armv7" ; do - # TODO(yuanshuai): enable armv7 on android - if [[ ${abi} == "armv7" ]]; then - continue - fi - - build_dir=$cur_dir/build.lite.android.${abi}.${lang} - cd $build_dir - - local port= - if [[ ${abi} == "armv7" ]]; then - port=${port_armv7} - fi - - if [[ ${abi} == "armv8" ]]; then - port=${port_armv8} - fi - echo "test file: ${TESTS_FILE}" - for _test in $(cat $TESTS_FILE); do - test_arm_android $_test $port - done - # TODO(sangoly): refine this - test_arm_model "test_cxx_api_lite" $port "./third_party/install/mobilenet_v2_relu" - done - done + # job 1 + build_arm "android" "armv8" "gcc" + test_arm "android" "armv8" "gcc" ${port_armv8} + cd - + + # job 2 + build_arm "android" "armv8" "clang" + test_arm "android" "armv8" "clang" ${port_armv8} + cd - - # armlinux need in another docker - # TODO(hongming): enable test armlinux on armv8, armv7 and armv7hf + # job 3 + build_arm "android" "armv7" "gcc" + test_arm "android" "armv7" "gcc" ${port_armv7} + cd - + + # job 4 + build_arm "android" "armv7" "clang" + test_arm "android" "armv7" "clang" ${port_armv7} + cd - adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done echo "Done" + ######################################################################## + + # job 5 + build_arm "armlinux" "armv8" + test_arm "armlinux" "armv8" + cd - + + # job 6 + build_arm "armlinux" "armv7" + test_arm "armlinux" "armv7" + cd - + + # job 7 + build_arm "armlinux" "armv7hf" + test_arm "armlinux" "armv7hf" + cd - + + echo "Done" } ############################# MAIN ################################# @@ -279,6 +326,10 @@ function main { ARM_ABI="${i#*=}" shift ;; + --arm_lang=*) + ARM_LANG="${i#*=}" + shift + ;; --arm_port=*) ARM_PORT="${i#*=}" shift @@ -301,13 +352,21 @@ function main { shift ;; cmake_arm) - cmake_arm $ARM_OS $ARM_ABI + cmake_arm $ARM_OS $ARM_ABI $ARM_LANG + shift + ;; + build_arm) + build_arm $ARM_OS $ARM_ABI $ARM_LANG shift ;; test_server) test_lite $TESTS_FILE shift ;; + test_arm) + build_arm $ARM_OS $ARM_ABI $ARM_LANG $ARM_PORT + shift + ;; test_arm_android) test_arm_android $TEST_NAME $ARM_PORT shift -- GitLab