Skip to content

P
Paddle-Lite

PaddlePaddle / Paddle-Lite

通知 331
Star 4
Fork 1
P
Paddle-Lite
未验证 提交 381fa7f6 编写于 作者: H huzhiqiang 提交者： GitHub
浏览文件
操作

modify ci_build.sh to support ci test with actual mobilephone test=develop (#2525)

上级 8903c795
显示空白变更内容
内联 并排
Showing with 120 addition and 78 deletion
lite/tools/build.sh
浏览文件 @ 381fa7f6
......@@ -26,6 +26,12 @@ readonly THIRDPARTY_TAR=https://paddle-inference-dist.bj.bcebos.com/PaddleLite/t
readonly workspace=$PWD
# if operating in mac env, we should expand the maximum file num
os_nmae=`uname -s`
if [ ${os_nmae} == "Darwin" ]; then
ulimit -n 1024
fi
# for code gen, a source file is generated after a test, but is dependended by some targets in cmake.
# here we fake an empty file to make cmake works.
function prepare_workspace {
......
lite/tools/ci_build.sh
浏览文件 @ 381fa7f6
......@@ -14,6 +14,16 @@ readonly workspace=$PWD
NUM_CORES_FOR_COMPILE=${LITE_BUILD_THREADS:-8}
# global variables
#whether to use emulator as adb devices,when USE_ADB_EMULATOR=ON we use emulator, else we will use connected mobile phone as adb devices.
USE_ADB_EMULATOR=ON
# if operating in mac env, we should expand the maximum file num
os_nmae=`uname -s`
if [ ${os_nmae} == "Darwin" ]; then
ulimit -n 1024
fi
function prepare_thirdparty {
if [ ! -d $workspace/third-party -o -f $workspace/third-party-05b862.tar.gz ]; then
rm -rf $workspace/third-party
......@@ -27,6 +37,27 @@ function prepare_thirdparty {
fi
}
# prepare adb devices
# if USE_ADB_EMULATOR=ON , we create adb emulator port_armv8 and port_armv7 for usage, else we will use actual mobilephone according to adbindex.
function prepare_adb_devices {
port_armv8=5554
port_armv7=5556
if [ $USE_ADB_EMULATOR == "ON" ]; then
prepare_emulator $port_armv8 $port_armv7
device_armv8=emulator-$port_armv8
device_armv7=emulator-$port_armv7
else
adb_devices=($(adb devices |grep -v devices |grep device | awk -F " " '{print $1}'))
# adbindex is the env variable registered in ci agent to tell which mobile is to used as adb
if [ ${adbindex} > ${#adb_devices[@]}-1 ]; then
echo -e "Error: the adb devices on ci agent are not enough, at least ${adbindex} adb devices are needed."
exit 1
fi
echo ${adb_devices[${adbindex}]}
device_armv8=${adb_devices[${adbindex}]}
device_armv7=${adb_devices[${adbindex}]}
fi
}
# for code gen, a source file is generated after a test, but is dependended by some targets in cmake.
# here we fake an empty file to make cmake works.
......@@ -77,7 +108,7 @@ function cmake_opencl {
}
function run_gen_code_test {
local port=$1
local device=$1
local gen_code_file_name="__generated_code__.cc"
local gen_code_file_path="./lite/gen_code/${gen_code_file_path}"
local adb_work_dir="/data/local/tmp"
......@@ -87,20 +118,20 @@ function run_gen_code_test {
# 2. run test_cxx_api_lite in emulator to get opt model
local test_cxx_api_lite_path=$(find ./lite -name test_cxx_api)
adb -s emulator-${port} push "./third_party/install/lite_naive_model" ${adb_work_dir}
adb -s emulator-${port} push ${test_cxx_api_lite_path} ${adb_work_dir}
adb -s emulator-${port} shell "${adb_work_dir}/test_cxx_api --model_dir=${adb_work_dir}/lite_naive_model --optimized_model=${adb_work_dir}/lite_naive_model_opt"
adb -s ${device} push "./third_party/install/lite_naive_model" ${adb_work_dir}
adb -s ${device} push ${test_cxx_api_lite_path} ${adb_work_dir}
adb -s ${device} shell "${adb_work_dir}/test_cxx_api --model_dir=${adb_work_dir}/lite_naive_model --optimized_model=${adb_work_dir}/lite_naive_model_opt"
# 3. build test_gen_code
make test_gen_code -j$NUM_CORES_FOR_COMPILE
# 4. run test_gen_code_lite in emulator to get __generated_code__.cc
local test_gen_code_lite_path=$(find ./lite -name test_gen_code)
adb -s emulator-${port} push ${test_gen_code_lite_path} ${adb_work_dir}
adb -s emulator-${port} shell "${adb_work_dir}/test_gen_code --optimized_model=${adb_work_dir}/lite_naive_model_opt --generated_code_file=${adb_work_dir}/${gen_code_file_name}"
adb -s ${device} push ${test_gen_code_lite_path} ${adb_work_dir}
adb -s ${device} shell "${adb_work_dir}/test_gen_code --optimized_model=${adb_work_dir}/lite_naive_model_opt --generated_code_file=${adb_work_dir}/${gen_code_file_name}"
# 5. pull __generated_code__.cc down and mv to build real path
adb -s emulator-${port} pull "${adb_work_dir}/${gen_code_file_name}" .
adb -s ${device} pull "${adb_work_dir}/${gen_code_file_name}" .
mv ${gen_code_file_name} ${gen_code_file_path}
# 6. build test_generated_code
......@@ -324,12 +355,12 @@ function build_test_xpu {
# test_arm_android <some_test_name> <adb_port_number>
function test_arm_android {
local test_name=$1
local port=$2
local device=$2
if [[ "${test_name}x" == "x" ]]; then
echo "test_name can not be empty"
exit 1
fi
if [[ "${port}x" == "x" ]]; then
if [[ "${device}x" == "x" ]]; then
echo "Port can not be empty"
exit 1
fi
......@@ -344,20 +375,20 @@ function test_arm_android {
local testpath=$(find ./lite -name ${test_name})
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s emulator-${port} shell "cd ${adb_work_dir} && ./${test_name}"
adb -s emulator-${port} shell "rm ${adb_work_dir}/${test_name}"
adb -s ${device} push ${testpath} ${adb_work_dir}
adb -s ${device} shell "cd ${adb_work_dir} && ./${test_name}"
adb -s ${device} shell "rm ${adb_work_dir}/${test_name}"
}
# test_npu <some_test_name> <adb_port_number>
function test_npu {
local test_name=$1
local port=$2
local device=$2
if [[ "${test_name}x" == "x" ]]; then
echo "test_name can not be empty"
exit 1
fi
if [[ "${port}x" == "x" ]]; then
if [[ "${device}x" == "x" ]]; then
echo "Port can not be empty"
exit 1
fi
......@@ -373,33 +404,33 @@ function test_npu {
local testpath=$(find ./lite -name ${test_name})
# note the ai_ddk_lib is under paddle-lite root directory
adb -s emulator-${port} push ../ai_ddk_lib/lib64/* ${adb_work_dir}
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s ${device} push ../ai_ddk_lib/lib64/* ${adb_work_dir}
adb -s ${device} push ${testpath} ${adb_work_dir}
if [[ ${test_name} == "test_npu_pass" ]]; then
local model_name=mobilenet_v1
adb -s emulator-${port} push "./third_party/install/${model_name}" ${adb_work_dir}
adb -s emulator-${port} shell "rm -rf ${adb_work_dir}/${model_name}_opt "
adb -s emulator-${port} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; export GLOG_v=0; ./${test_name} --model_dir=./${model_name} --optimized_model=./${model_name}_opt"
adb -s ${device} push "./third_party/install/${model_name}" ${adb_work_dir}
adb -s ${device} shell "rm -rf ${adb_work_dir}/${model_name}_opt "
adb -s ${device} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; export GLOG_v=0; ./${test_name} --model_dir=./${model_name} --optimized_model=./${model_name}_opt"
elif [[ ${test_name} == "test_subgraph_pass" ]]; then
local model_name=mobilenet_v1
adb -s emulator-${port} push "./third_party/install/${model_name}" ${adb_work_dir}
adb -s emulator-${port} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; export GLOG_v=0; ./${test_name} --model_dir=./${model_name}"
adb -s ${device} push "./third_party/install/${model_name}" ${adb_work_dir}
adb -s ${device} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; export GLOG_v=0; ./${test_name} --model_dir=./${model_name}"
else
adb -s emulator-${port} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; ./${test_name}"
adb -s ${device} shell "cd ${adb_work_dir}; export LD_LIBRARY_PATH=./ ; ./${test_name}"
fi
}
function test_npu_model {
local test_name=$1
local port=$2
local device=$2
local model_dir=$3
if [[ "${test_name}x" == "x" ]]; then
echo "test_name can not be empty"
exit 1
fi
if [[ "${port}x" == "x" ]]; then
if [[ "${device}x" == "x" ]]; then
echo "Port can not be empty"
exit 1
fi
......@@ -412,17 +443,17 @@ function test_npu_model {
adb_work_dir="/data/local/tmp"
testpath=$(find ./lite -name ${test_name})
adb -s emulator-${port} push ../ai_ddk_lib/lib64/* ${adb_work_dir}
adb -s emulator-${port} push ${model_dir} ${adb_work_dir}
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}"
adb -s ${device} push ../ai_ddk_lib/lib64/* ${adb_work_dir}
adb -s ${device} push ${model_dir} ${adb_work_dir}
adb -s ${device} push ${testpath} ${adb_work_dir}
adb -s ${device} shell chmod +x "${adb_work_dir}/${test_name}"
local adb_model_path="${adb_work_dir}/`basename ${model_dir}`"
adb -s emulator-${port} shell "export LD_LIBRARY_PATH=${adb_work_dir}; ${adb_work_dir}/${test_name} --model_dir=$adb_model_path"
adb -s ${device} shell "export LD_LIBRARY_PATH=${adb_work_dir}; ${adb_work_dir}/${test_name} --model_dir=$adb_model_path"
}
# test the inference high level api
function test_arm_api {
local port=$1
local device=$1
local test_name="test_paddle_api"
make $test_name -j$NUM_CORES_FOR_COMPILE
......@@ -431,23 +462,23 @@ function test_arm_api {
local remote_model=${adb_work_dir}/paddle_api
local testpath=$(find ./lite -name ${test_name})
arm_push_necessary_file $port $model_path $remote_model
adb -s emulator-${port} shell mkdir -p $remote_model
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}"
adb -s emulator-${port} shell "${adb_work_dir}/${test_name} --model_dir $remote_model"
arm_push_necessary_file $device $model_path $remote_model
adb -s ${device} shell mkdir -p $remote_model
adb -s ${device} push ${testpath} ${adb_work_dir}
adb -s ${device} shell chmod +x "${adb_work_dir}/${test_name}"
adb -s ${device} shell "${adb_work_dir}/${test_name} --model_dir $remote_model"
}
function test_arm_model {
local test_name=$1
local port=$2
local device=$2
local model_dir=$3
if [[ "${test_name}x" == "x" ]]; then
echo "test_name can not be empty"
exit 1
fi
if [[ "${port}x" == "x" ]]; then
if [[ "${device}x" == "x" ]]; then
echo "Port can not be empty"
exit 1
fi
......@@ -460,11 +491,11 @@ function test_arm_model {
adb_work_dir="/data/local/tmp"
testpath=$(find ./lite -name ${test_name})
adb -s emulator-${port} push ${model_dir} ${adb_work_dir}
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s emulator-${port} shell chmod +x "${adb_work_dir}/${test_name}"
adb -s ${device} push ${model_dir} ${adb_work_dir}
adb -s ${device} push ${testpath} ${adb_work_dir}
adb -s ${device} shell chmod +x "${adb_work_dir}/${test_name}"
local adb_model_path="${adb_work_dir}/`basename ${model_dir}`"
adb -s emulator-${port} shell "${adb_work_dir}/${test_name} --model_dir=$adb_model_path"
adb -s ${device} shell "${adb_work_dir}/${test_name} --model_dir=$adb_model_path"
}
# function _test_model_optimize_tool {
......@@ -491,11 +522,11 @@ function test_model_optimize_tool_compile {
}
function _test_paddle_code_generator {
local port=$1
local device=$1
local test_name=paddle_code_generator
local remote_test=$ADB_WORK_DIR/$test_name
local remote_model=$ADB_WORK_DIR/lite_naive_model.opt
local adb="adb -s emulator-${port}"
local adb="adb -s ${device}"
make paddle_code_generator -j$NUM_CORES_FOR_COMPILE
local test_path=$(find . -name $test_name | head -n1)
......@@ -576,8 +607,6 @@ function build_arm {
cmake_arm ${os} ${abi} ${lang}
build $TESTS_FILE
# test publish inference lib
make publish_inference
}
# $1: ARM_TARGET_OS in "android"
......@@ -614,7 +643,7 @@ function test_arm {
os=$1
abi=$2
lang=$3
port=$4
device=$4
if [[ ${os} == "armlinux" ]]; then
# TODO(hongming): enable test armlinux on armv8, armv7 and armv7hf
......@@ -628,16 +657,16 @@ function test_arm {
fi
# prepare for CXXApi test
local adb="adb -s emulator-${port}"
local adb="adb -s ${device}"
$adb shell mkdir -p /data/local/tmp/lite_naive_model_opt
echo "test file: ${TESTS_FILE}"
for _test in $(cat $TESTS_FILE); do
test_arm_android $_test $port
test_arm_android $_test $device
done
# test finally
test_arm_api $port
test_arm_api $device
# _test_model_optimize_tool $port
# _test_paddle_code_generator $port
......@@ -661,11 +690,11 @@ function prepare_emulator {
}
function arm_push_necessary_file {
local port=$1
local device=$1
local testpath=$2
local adb_work_dir=$3
adb -s emulator-${port} push ${testpath} ${adb_work_dir}
adb -s ${device} push ${testpath} ${adb_work_dir}
}
function build_test_arm_opencl {
......@@ -688,15 +717,13 @@ function build_test_arm_opencl {
function build_test_arm_subtask_android {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
prepare_emulator $port_armv8 $port_armv7
prepare_adb_devices
# job 1
build_arm "android" "armv8" "gcc"
run_gen_code_test ${port_armv8}
test_arm "android" "armv8" "gcc" ${port_armv8}
adb -s $device_armv8 shell 'rm -rf /data/local/tmp/*'
run_gen_code_test ${device_armv8}
test_arm "android" "armv8" "gcc" ${device_armv8}
cd -
# job 2
......@@ -707,8 +734,9 @@ function build_test_arm_subtask_android {
# job 3
build_arm "android" "armv7" "gcc"
run_gen_code_test ${port_armv7}
test_arm "android" "armv7" "gcc" ${port_armv7}
adb -s $device_armv7 shell 'rm -rf /data/local/tmp/*'
run_gen_code_test ${device_armv7}
test_arm "android" "armv7" "gcc" ${device_armv7}
cd -
# job 4
......@@ -717,7 +745,9 @@ function build_test_arm_subtask_android {
#test_arm "android" "armv7" "clang" ${port_armv7}
#cd -
if [ $USE_ADB_EMULATOR == "ON" ]; then
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
fi
echo "Done"
}
......@@ -726,17 +756,17 @@ function build_test_arm_subtask_armlinux {
cur=$PWD
# job 5
build_arm "armlinux" "armv8" "gcc"
test_arm "armlinux" "armv8" "gcc" $port_armv8
test_arm "armlinux" "armv8" "gcc" $device_armv8
cd $cur
# job 6
build_arm "armlinux" "armv7" "gcc"
test_arm "armlinux" "armv7" "gcc" $port_armv8
test_arm "armlinux" "armv7" "gcc" $device_armv8
cd $cur
# job 7
build_arm "armlinux" "armv7hf" "gcc"
test_arm "armlinux" "armv7hf" "gcc" $port_armv8
test_arm "armlinux" "armv7hf" "gcc" $device_armv8
cd $cur
echo "Done"
......@@ -744,8 +774,6 @@ function build_test_arm_subtask_armlinux {
# sub-task-model
function build_test_arm_subtask_model {
local port_armv8=5554
local port_armv7=5556
# We just test following single one environment to limit the CI time.
local os=android
local abi=armv8
......@@ -761,12 +789,16 @@ function build_test_arm_subtask_model {
cmake_arm $os $abi $lang
make $test_name -j$NUM_CORES_FOR_COMPILE
prepare_emulator $port_armv8 $port_armv7
# prepare adb devices
prepare_adb_devices
adb -s $device_armv8 shell 'rm -rf /data/local/tmp/*'
# just test the model on armv8
test_arm_model $test_name $port_armv8 "./third_party/install/$model_name"
test_arm_model $test_name $device_armv8 "./third_party/install/$model_name"
if [ $USE_ADB_EMULATOR == "ON" ]; then
adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
fi
echo "Done"
cd -
rm -rf $build_dir
......@@ -775,16 +807,16 @@ function build_test_arm_subtask_model {
# this test load a model, optimize it and check the prediction result of both cxx and light APIS.
function test_arm_predict_apis {
local port=$1
local device=$1
local workspace=$2
local naive_model_path=$3
local api_test_path=$(find . -name "test_apis")
# the model is pushed to ./lite_naive_model
adb -s emulator-${port} push ${naive_model_path} ${workspace}
adb -s emulator-${port} push $api_test_path ${workspace}
adb -s ${device} push ${naive_model_path} ${workspace}
adb -s ${device} push $api_test_path ${workspace}
# test cxx_api first to store the optimized model.
adb -s emulator-${port} shell ./test_apis --model_dir ./lite_naive_model --optimized_model ./lite_naive_model_opt
adb -s ${device} shell ./test_apis --model_dir ./lite_naive_model --optimized_model ./lite_naive_model_opt
}
......@@ -792,9 +824,6 @@ function test_arm_predict_apis {
function build_test_arm {
########################################################################
# job 1-4 must be in one runner
port_armv8=5554
port_armv7=5556
build_test_arm_subtask_android
build_test_arm_subtask_armlinux
}
......@@ -816,16 +845,19 @@ function build_test_npu {
# just test the model on armv8
# prepare_emulator $port_armv8
prepare_emulator $port_armv8 $port_armv7
local device_armv8=emulator-$port_armv8
if [[ "${test_name}x" != "x" ]]; then
test_npu ${test_name} ${port_armv8}
test_npu ${test_name} ${device_armv8}
else
# run_gen_code_test ${port_armv8}
for _test in $(cat $TESTS_FILE | grep npu); do
test_npu $_test $port_armv8
test_npu $_test $device_armv8
done
fi
test_npu_model $test_model_name $port_armv8 "./third_party/install/$model_name"
test_npu_model $test_model_name $device_armv8 "./third_party/install/$model_name"
cd -
# just test the model on armv8
# adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
......@@ -910,6 +942,10 @@ function main {
ARM_PORT="${i#*=}"
shift
;;
--use_adb_emulator=*)
USE_ADB_EMULATOR="${i#*=}"
shift
;;
build)
build $TESTS_FILE
build $LIBS_FILE
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或