Skip to content

P
PaddleDetection

PaddlePaddle / PaddleDetection
大约 1 年 前同步成功

通知 695
Star 11112
Fork 2696
P
PaddleDetection
提交 8dc7ba33 编写于 作者: C Chunwei
浏览文件
操作

parallel CI

上级 b950d3a3
隐藏空白更改
内联 并排
Showing with 158 addition and 45 deletion
.gitlab-ci.yml
浏览文件 @ 8dc7ba33
......@@ -2,6 +2,20 @@ before_script:
- env
- export CI_USER_DIR=$(pwd)
# prepare ccache
- apt install ccache
# for proxy
- export http_proxy=$CI_PROXY
- export https_proxy=$CI_PROXY
# merge the latest code
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
- git fetch origin incubate/lite
- git merge --no-ff origin/incubate/lite
image: $SERVER_LITE_DOCKER_IMAGE
stages:
......@@ -14,19 +28,13 @@ check:prebuilt:
- lite
stage: ci
script:
# prepare for pre-commit
- rm -rf ~/.pip
- export http_proxy=$CI_PROXY
- export https_proxy=$CI_PROXY
- pip install pre-commit
- pre-commit install
# merge the latest code
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
- git fetch origin incubate/lite
- git merge --no-ff origin/incubate/lite
- ./paddle/fluid/lite/tools/build.sh check_style
cache:
key: check_style
paths:
......@@ -42,17 +50,11 @@ build:server:
paths:
- build/third_party
- ~/.ccache
- $CI_PROJECT_DIR/_build_server_ccache
script:
- apt install ccache
- export http_proxy=$CI_PROXY
- export https_proxy=$CI_PROXY
# merge the latest code
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
- git fetch origin incubate/lite
- git merge --no-ff origin/incubate/lite
# customize ccache path for specifying runner cache
- export CCACHE_DIR=$CI_PROJECT_DIR/_build_server_ccache
# run build and test
- mkdir -p build
- cd build
- ../paddle/fluid/lite/tools/build.sh cmake_x86
......@@ -66,7 +68,27 @@ build:server:
dependencies:
- check:prebuilt
build:mobile:
build:mobile_android:
tags:
- lite
stage: build_mobile
image: $MOBILE_LITE_DOCKER_IMAGE
cache:
key: mobile_thirdparty
paths:
- $MOBILE_LITE_CACHE0
- $MOBILE_LITE_CACHE1
- ~/.ccache
- $CI_PROJECT_DIR/build_mobile_ccache
script:
- export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_ccache
- ./paddle/fluid/lite/tools/build.sh build_test_arm_subtask_android
dependencies:
- build:server
build:mobile_armlinux:
tags:
- lite
stage: build_mobile
......@@ -77,17 +99,43 @@ build:mobile:
- $MOBILE_LITE_CACHE0
- $MOBILE_LITE_CACHE1
- ~/.ccache
- $CI_PROJECT_DIR/build_mobile_ccache2
script:
- apt install ccache
- export http_proxy=$CI_PROXY
- export https_proxy=$CI_PROXY
- export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_ccache2
- ./paddle/fluid/lite/tools/build.sh build_test_arm_subtask_armlinux
dependencies:
- build:server
# merge the latest code
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
- git fetch origin incubate/lite
- git merge --no-ff origin/incubate/lite
cache:
key: mobile_thirdparty
paths:
- $MOBILE_LITE_CACHE0
- $MOBILE_LITE_CACHE1
- ~/.ccache
build:mobile_model_mobilenetv2:
tags:
- lite
stage: build_mobile
image: $MOBILE_LITE_DOCKER_IMAGE
cache:
key: mobile_thirdparty
paths:
- $MOBILE_LITE_CACHE0
- $MOBILE_LITE_CACHE1
- ~/.ccache
script:
- export CCACHE_DIR=$CI_PROJECT_DIR/build_mobile_model1
- ./paddle/fluid/lite/tools/build.sh build_test_arm_model1
- ./paddle/fluid/lite/tools/build.sh build_test_arm
dependencies:
- build:server
cache:
key: mobile_thirdparty
paths:
- $MOBILE_LITE_CACHE0
- $MOBILE_LITE_CACHE1
- ~/.ccache
- $CI_PROJECT_DIR/build_mobile_model1
paddle/fluid/lite/tools/build.sh
浏览文件 @ 8dc7ba33
......@@ -135,8 +135,8 @@ function test_arm_model {
adb -s emulator-${port} push ${model_dir} ${adb_work_dir}
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}"
local adb_model_path="./${adb_work_dir}/`basename ${model_dir}`"
adb -s emulator-${port} shell "./${adb_work_dir}/${test_name} --eval_model_dir=$adb_model_path"
local adb_model_path="${adb_work_dir}/`basename ${model_dir}`"
adb -s emulator-${port} shell "${adb_work_dir}/${test_name} --eval_model_dir=$adb_model_path"
}
......@@ -225,16 +225,11 @@ function test_arm {
for _test in $(cat $TESTS_FILE); do
test_arm_android $_test $port
done
# TODO(sangoly): refine this
test_arm_model "test_cxx_api_lite" $port "./third_party/install/mobilenet_v2_relu"
}
# Build the code and run lite arm tests. This is executed in the CI system.
function build_test_arm {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
function prepare_emulator {
local port_armv8=$1
local port_armv7=$2
adb kill-server
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
......@@ -245,6 +240,18 @@ function build_test_arm {
echo n | avdmanager create avd -f -n paddle-armv7 -k "system-images;android-24;google_apis;armeabi-v7a"
echo -ne '\n' | ${ANDROID_HOME}/emulator/emulator -avd paddle-armv7 -noaudio -no-window -gpu off -verbose -port ${port_armv7} &
sleep 1m
}
# We split the arm unittest into several sub-tasks to parallel and reduce the overall CI timetime.
# sub-task1
function build_test_arm_subtask_android {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
prepare_emulator $port_armv8 $port_armv7
# job 1
build_arm "android" "armv8" "gcc"
......@@ -252,9 +259,9 @@ function build_test_arm {
cd -
# job 2
build_arm "android" "armv8" "clang"
test_arm "android" "armv8" "clang" ${port_armv8}
cd -
#build_arm "android" "armv8" "clang"
#test_arm "android" "armv8" "clang" ${port_armv8}
#cd -
# job 3
build_arm "android" "armv7" "gcc"
......@@ -262,13 +269,22 @@ function build_test_arm {
cd -
# job 4
build_arm "android" "armv7" "clang"
test_arm "android" "armv7" "clang" ${port_armv7}
cd -
#build_arm "android" "armv7" "clang"
#test_arm "android" "armv7" "clang" ${port_armv7}
#cd -
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
echo "Done"
}
# sub-task2
function build_test_arm_subtask_armlinux {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
prepare_emulator $port_armv8 $port_armv7
# job 5
build_arm "armlinux" "armv8"
......@@ -285,9 +301,47 @@ function build_test_arm {
test_arm "armlinux" "armv7hf"
cd -
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
echo "Done"
}
# sub-task3
function build_test_arm_subtask3_mobilenet_v2 {
local port_armv8=5554
local port_armv7=5556
# We just test following single one environment to limit the CI time.
local os=android
local abi=armv8
local lang=gcc
cur_dir=$(pwd)
build_dir=$cur_dir/build.lite.${os}.${abi}.${lang}
mkdir -p $build_dir
cd $build_dir
cmake_arm $os $abi $lang
make test_cxx_api_lite -j$NUM_CORES_FOR_COMPILE
prepare_emulator $port_armv8 $port_armv7
# just test the model on armv8
test_arm_model "test_cxx_api_lite" $port_armv8 "./third_party/install/mobilenet_v2_relu"
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
echo "Done"
}
# Build the code and run lite arm tests. This is executed in the CI system.
function build_test_arm {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
build_test_arm_subtask_android
build_test_arm_subtask_armlinux
}
############################# MAIN #################################
function print_usage {
echo -e "\nUSAGE:"
......@@ -379,6 +433,18 @@ function main {
build_test_arm
shift
;;
build_test_arm_subtask_android)
build_test_arm_subtask_android
shift
;;
build_test_arm_subtask_armlinux)
build_test_arm_subtask_armlinux
shift
;;
build_test_arm_model1)
build_test_arm_subtask3_mobilenet_v2
shift
;;
check_style)
check_style
shift
......@@ -397,4 +463,3 @@ function main {
}
main $@
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或