From 8dc7ba33ea3b6a4a2887a66beec5eb903077c6d6 Mon Sep 17 00:00:00 2001
From: Chunwei <yanchunwei@baidu.com>
Date: Thu, 20 Jun 2019 09:04:20 +0000
Subject: [PATCH] parallel CI

---
 .gitlab-ci.yml                   | 104 ++++++++++++++++++++++---------
 paddle/fluid/lite/tools/build.sh |  99 ++++++++++++++++++++++++-----
 2 files changed, 158 insertions(+), 45 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a2f0644f21..d3f5df342e 100755
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,20 @@ before_script:
   - env
   - export CI_USER_DIR=$(pwd)
 
+  # prepare ccache
+  - apt install ccache
+
+  # for proxy
+  - export http_proxy=$CI_PROXY
+  - export https_proxy=$CI_PROXY
+
+  # merge the latest code
+  - git config --global user.email "you@example.com"
+  - git config --global user.name "Your Name"
+  - git fetch origin incubate/lite
+  - git merge --no-ff origin/incubate/lite
+
+
 image: $SERVER_LITE_DOCKER_IMAGE
 
 stages:
@@ -14,19 +28,13 @@ check:prebuilt:
         - lite
     stage: ci
     script:
+        # prepare for pre-commit
         - rm -rf ~/.pip
-        - export http_proxy=$CI_PROXY
-        - export https_proxy=$CI_PROXY
         - pip install pre-commit
         - pre-commit install
 
-        # merge the latest code
-        - git config --global user.email "you@example.com"
-        - git config --global user.name "Your Name"
-        - git fetch origin incubate/lite
-        - git merge --no-ff origin/incubate/lite
-
         - ./paddle/fluid/lite/tools/build.sh check_style
+
     cache:
         key: check_style
         paths:
@@ -42,17 +50,11 @@ build:server:
         paths:
             - build/third_party
             - ~/.ccache
+            - $CI_PROJECT_DIR/_build_server_ccache
     script:
-        - apt install ccache
-        - export http_proxy=$CI_PROXY
-        - export https_proxy=$CI_PROXY
-
-        # merge the latest code
-        - git config --global user.email "you@example.com"
-        - git config --global user.name "Your Name"
-        - git fetch origin incubate/lite
-        - git merge --no-ff origin/incubate/lite
-
+        # customize ccache path for specifying runner cache
+        - export CCACHE_DIR=$CI_PROJECT_DIR/_build_server_ccache
+        # run build and test
         - mkdir -p build
         - cd build
         - ../paddle/fluid/lite/tools/build.sh cmake_x86
@@ -66,7 +68,27 @@ build:server:
     dependencies:
         - check:prebuilt
 
-build:mobile:
+build:mobile_android:
+    tags:
+        - lite
+    stage: build_mobile
+    image: $MOBILE_LITE_DOCKER_IMAGE
+    cache:
+        key: mobile_thirdparty
+        paths:
+            - $MOBILE_LITE_CACHE0
+            - $MOBILE_LITE_CACHE1
+            - ~/.ccache
+            - $CI_PROJECT_DIR/build_mobile_ccache
+    script:
+        - export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_ccache
+        - ./paddle/fluid/lite/tools/build.sh build_test_arm_subtask_android
+
+    dependencies:
+        - build:server
+
+
+build:mobile_armlinux:
     tags:
         - lite
     stage: build_mobile
@@ -77,17 +99,43 @@ build:mobile:
             - $MOBILE_LITE_CACHE0
             - $MOBILE_LITE_CACHE1
             - ~/.ccache
+            - $CI_PROJECT_DIR/build_mobile_ccache2
     script:
-        - apt install ccache
-        - export http_proxy=$CI_PROXY
-        - export https_proxy=$CI_PROXY
+        - export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_ccache2
+        - ./paddle/fluid/lite/tools/build.sh build_test_arm_subtask_armlinux
+
+    dependencies:
+        - build:server
 
-        # merge the latest code
-        - git config --global user.email "you@example.com"
-        - git config --global user.name "Your Name"
-        - git fetch origin incubate/lite
-        - git merge --no-ff origin/incubate/lite
+    cache:
+        key: mobile_thirdparty
+        paths:
+            - $MOBILE_LITE_CACHE0
+            - $MOBILE_LITE_CACHE1
+            - ~/.ccache
+
+build:mobile_model_mobilenetv2:
+    tags:
+        - lite
+    stage: build_mobile
+    image: $MOBILE_LITE_DOCKER_IMAGE
+    cache:
+        key: mobile_thirdparty
+        paths:
+            - $MOBILE_LITE_CACHE0
+            - $MOBILE_LITE_CACHE1
+            - ~/.ccache
+    script:
+        - export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_model1
+        - ./paddle/fluid/lite/tools/build.sh build_test_arm_model1
 
-        - ./paddle/fluid/lite/tools/build.sh build_test_arm
     dependencies:
         - build:server
+
+    cache:
+        key: mobile_thirdparty
+        paths:
+            - $MOBILE_LITE_CACHE0
+            - $MOBILE_LITE_CACHE1
+            - ~/.ccache
+            - $CI_PROJECT_DIR/build_mobile_model1
diff --git a/paddle/fluid/lite/tools/build.sh b/paddle/fluid/lite/tools/build.sh
index 70a23ecf69..b66efe8959 100755
--- a/paddle/fluid/lite/tools/build.sh
+++ b/paddle/fluid/lite/tools/build.sh
@@ -135,8 +135,8 @@ function test_arm_model {
     adb -s emulator-${port} push ${model_dir} ${adb_work_dir}
     adb -s emulator-${port} push ${testpath} ${adb_work_dir}
     adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}"
-    local adb_model_path="./${adb_work_dir}/`basename ${model_dir}`"
-    adb -s emulator-${port} shell "./${adb_work_dir}/${test_name} --eval_model_dir=$adb_model_path"
+    local adb_model_path="${adb_work_dir}/`basename ${model_dir}`"
+    adb -s emulator-${port} shell "${adb_work_dir}/${test_name} --eval_model_dir=$adb_model_path"
 
 }
 
@@ -225,16 +225,11 @@ function test_arm {
     for _test in $(cat $TESTS_FILE); do
         test_arm_android $_test $port
     done
-    # TODO(sangoly): refine this
-    test_arm_model "test_cxx_api_lite" $port "./third_party/install/mobilenet_v2_relu"
 }
 
-# Build the code and run lite arm tests. This is executed in the CI system.
-function build_test_arm {
-    ########################################################################
-    # job 1-4 must be in one runner
-    port_armv8=5554
-    port_armv7=5556
+function prepare_emulator {
+    local port_armv8=$1
+    local port_armv7=$2
 
     adb kill-server
     adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
@@ -245,6 +240,18 @@ function build_test_arm {
     echo n | avdmanager create avd -f -n paddle-armv7 -k "system-images;android-24;google_apis;armeabi-v7a"
     echo -ne '\n' | ${ANDROID_HOME}/emulator/emulator -avd paddle-armv7 -noaudio -no-window -gpu off -verbose -port ${port_armv7} &
     sleep 1m
+}
+
+
+# We split the arm unittest into several sub-tasks to parallel and reduce the overall CI timetime.
+# sub-task1
+function build_test_arm_subtask_android {
+    ########################################################################
+    # job 1-4 must be in one runner
+    port_armv8=5554
+    port_armv7=5556
+
+    prepare_emulator $port_armv8 $port_armv7
 
     # job 1
     build_arm "android" "armv8" "gcc"
@@ -252,9 +259,9 @@ function build_test_arm {
     cd -
 
     # job 2
-    build_arm "android" "armv8" "clang"
-    test_arm "android" "armv8" "clang" ${port_armv8}
-    cd -
+    #build_arm "android" "armv8" "clang"
+    #test_arm "android" "armv8" "clang" ${port_armv8}
+    #cd -
 
     # job 3
     build_arm "android" "armv7" "gcc"
@@ -262,13 +269,22 @@ function build_test_arm {
     cd -
 
     # job 4
-    build_arm "android" "armv7" "clang"
-    test_arm "android" "armv7" "clang" ${port_armv7}
-    cd -
+    #build_arm "android" "armv7" "clang"
+    #test_arm "android" "armv7" "clang" ${port_armv7}
+    #cd -
 
     adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
     echo "Done"
+}
+
+# sub-task2
+function build_test_arm_subtask_armlinux {
     ########################################################################
+    # job 1-4 must be in one runner
+    port_armv8=5554
+    port_armv7=5556
+
+    prepare_emulator $port_armv8 $port_armv7
 
     # job 5
     build_arm "armlinux" "armv8"
@@ -285,9 +301,47 @@ function build_test_arm {
     test_arm "armlinux" "armv7hf"
     cd -
 
+    adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
+    echo "Done"
+}
+
+# sub-task3
+function build_test_arm_subtask3_mobilenet_v2 {
+    local port_armv8=5554
+    local port_armv7=5556
+    # We just test following single one environment to limit the CI time.
+    local os=android
+    local abi=armv8
+    local lang=gcc
+
+    cur_dir=$(pwd)
+    build_dir=$cur_dir/build.lite.${os}.${abi}.${lang}
+    mkdir -p $build_dir
+    cd $build_dir
+    cmake_arm $os $abi $lang
+    make test_cxx_api_lite -j$NUM_CORES_FOR_COMPILE
+
+    prepare_emulator $port_armv8 $port_armv7
+
+    # just test the model on armv8
+    test_arm_model "test_cxx_api_lite" $port_armv8 "./third_party/install/mobilenet_v2_relu"
+
+    adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
     echo "Done"
 }
 
+# Build the code and run lite arm tests. This is executed in the CI system.
+function build_test_arm {
+    ########################################################################
+    # job 1-4 must be in one runner
+    port_armv8=5554
+    port_armv7=5556
+
+    build_test_arm_subtask_android
+    build_test_arm_subtask_armlinux
+}
+
+
 ############################# MAIN #################################
 function print_usage {
     echo -e "\nUSAGE:"
@@ -379,6 +433,18 @@ function main {
                 build_test_arm
                 shift
                 ;;
+            build_test_arm_subtask_android)
+                build_test_arm_subtask_android
+                shift
+                ;;
+            build_test_arm_subtask_armlinux)
+                build_test_arm_subtask_armlinux
+                shift
+                ;;
+            build_test_arm_model1)
+                build_test_arm_subtask3_mobilenet_v2
+                shift
+                ;;
             check_style)
                 check_style
                 shift
@@ -397,4 +463,3 @@ function main {
 }
 
 main $@
- 
-- 
GitLab