From f9748a17c612260326358ff062ae4dd723da9e71 Mon Sep 17 00:00:00 2001 From: yejianwu Date: Mon, 28 Jan 2019 15:32:49 +0800 Subject: [PATCH] update CI and update Dockerfile to support python3 --- .gitlab-ci.yml | 85 +++++++++++++++++++-------------- docker/gitlab-runner/Dockerfile | 3 ++ docker/mace-dev-lite/Dockerfile | 28 +++++++++-- docker/mace-dev/Dockerfile | 45 +++++++++++++++-- 4 files changed, 115 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab1a7ba4..3d6ad0cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,14 @@ stages: - ops_test_disable_neon - ops_test - api_test - - python_tools_tests - model_tests - quantization_tests - build_android_demo - ops_benchmark - extra_tests - so_size_check + - dynamic_link_test + - python3_test cpplint: stage: cpplint @@ -76,12 +77,9 @@ ndk_versions_compatible_tests: export ANDROID_NDK_HOME=$new_ndk_path; export PATH=$ANDROID_NDK_HOME:$PATH; echo "ndk path: $ANDROID_NDK_HOME"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64-v8a --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64-v8a --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*"; - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*"; + python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*" || exit 1; + python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*" || exit 1; + python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64-v8a --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*" || exit 1; fi done - export ANDROID_NDK_HOME=$DEFAULT_NDK_PATH @@ -96,7 +94,7 @@ ops_test_disable_neon: GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git DEVICE_CONF_FILE=generic-mobile-devices/devices.yml fi - - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS --enable_neon=false + - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS --enable_neon=false || exit 1; ops_test: stage: ops_test @@ -107,42 +105,23 @@ ops_test: GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git DEVICE_CONF_FILE=generic-mobile-devices/devices.yml fi - - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS + - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS || exit 1; api_test: stage: api_test script: - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS || exit 1; + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS || exit 1; + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS || exit 1; - > if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git DEVICE_CONF_FILE=generic-mobile-devices/devices.yml fi - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS - - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS - -python_tools_tests: - stage: python_tools_tests - script: - - pwd - - rm -rf mace-models - - GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@github.com:XiaoMi/mace-models.git - - CONF_FILE=mace-models/mobilenet-v2/mobilenet-v2.yml - - > - if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then - GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git - DEVICE_CONF_FILE=generic-mobile-devices/devices.yml - fi - - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi - - > - python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,arm64 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS || exit 1; + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS || exit 1; + - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS || exit 1; model_tests: stage: model_tests @@ -186,7 +165,6 @@ quantization_tests: do python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; done - rm -rf mace-models @@ -200,7 +178,7 @@ ops_benchmark: stage: ops_benchmark script: - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi - - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS --args="--filter=.*CONV.*" + - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS --args="--filter=.*SIGMOID.*" || exit 1; extra_tests: stage: extra_tests @@ -211,7 +189,7 @@ extra_tests: GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git DEVICE_CONF_FILE=generic-mobile-devices/devices.yml fi - - python tools/bazel_adb_run.py --target="//mace/utils:tuner_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS + - python tools/bazel_adb_run.py --target="//mace/utils:tuner_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS || exit 1; so_size_check: stage: so_size_check @@ -230,3 +208,36 @@ so_size_check: - rm -rf $TARGET_MACE_WORK_DIR - echo "The libmace.so size grows from $TARGET_LIBMACE_SO_SIZE(KB) to $CURRENT_LIBMACE_SO_SIZE(KB)" - if [ $TARGET_LIBMACE_SO_SIZE -lt `expr \( $CURRENT_LIBMACE_SO_SIZE - $TARGET_LIBMACE_SO_SIZE \) \* 10` ]; then echo "The libmace.so size grows larger than 10%"; exit 1; fi + +dynamic_link_test: + stage: dynamic_link_test + script: + - pwd + - rm -rf mace-models + - rm -rf generic-mobile-devices + - GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@github.com:XiaoMi/mace-models.git + - CONF_FILE=mace-models/mobilenet-v1/mobilenet-v1.yml + - > + if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then + GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git + DEVICE_CONF_FILE=generic-mobile-devices/devices.yml + fi + - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi + - > + python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file --cl_mem_type=buffer || exit 1; + python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --mace_lib_type=dynamic --target_abis=armeabi-v7a,arm64 --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; + - rm -rf mace-models + +python3_test: + stage: python3_test + script: + - pyenv local 3.6.3 + - rm -rf mace-models + - GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@github.com:XiaoMi/mace-models.git + - CONF_FILE=mace-models/mobilenet-v1/mobilenet-v1.yml + - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi + - > + python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1; + python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; + - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --run_target=False --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --validate || exit 1; + - pyenv local 2.7.14 diff --git a/docker/gitlab-runner/Dockerfile b/docker/gitlab-runner/Dockerfile index 2f4c3c6e..bfbcab23 100644 --- a/docker/gitlab-runner/Dockerfile +++ b/docker/gitlab-runner/Dockerfile @@ -7,4 +7,7 @@ RUN apt-get update -y RUN curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-ci-multi-runner/script.deb.sh | bash RUN apt-get install gitlab-ci-multi-runner +# set timezone +RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + ENTRYPOINT gitlab-runner run diff --git a/docker/mace-dev-lite/Dockerfile b/docker/mace-dev-lite/Dockerfile index f0753cd6..4169b2c2 100644 --- a/docker/mace-dev-lite/Dockerfile +++ b/docker/mace-dev-lite/Dockerfile @@ -19,8 +19,6 @@ RUN apt-get install -y --no-install-recommends \ libprotobuf-dev \ libzmq3-dev \ pkg-config \ - python-dev \ - python-pip \ protobuf-compiler \ libprotoc-dev \ rsync \ @@ -33,7 +31,27 @@ RUN apt-get install -y --no-install-recommends \ openssh-server \ wget \ bsdmainutils -RUN pip install --upgrade pip + +# install pyenv +RUN apt-get install -y make \ + libssl-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python-openssl +RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +ENV PATH /root/.pyenv/bin:/root/.pyenv/shims/:${PATH} +RUN eval "$(pyenv init -)" +RUN eval "$(pyenv virtualenv-init -)" +RUN pyenv install 2.7.14 && pyenv install 3.6.3 +RUN pyenv global 2.7.14 # Setup vim RUN apt-get install -y --no-install-recommends \ @@ -78,6 +96,7 @@ RUN mkdir -p /opt/sdk && \ unzip ${ANDROID_SDK_FILENAME} -d ${ANDROID_HOME} && \ rm ${ANDROID_SDK_FILENAME} && \ yes | android update sdk --no-ui -a --filter tools,platform-tools,${ANDROID_API_LEVELS},build-tools-${ANDROID_BUILD_TOOLS_VERSION} +RUN ${ANDROID_HOME}/tools/bin/sdkmanager "cmake;3.6.4111459" # Download NDK RUN cd /opt/ && \ @@ -102,7 +121,8 @@ RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com pyyaml \ sh==1.12.14 \ pycodestyle==2.4.0 \ - filelock + filelock \ + PTable # Install Tensorflow RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ diff --git a/docker/mace-dev/Dockerfile b/docker/mace-dev/Dockerfile index 1f73dcef..6ab7eb43 100644 --- a/docker/mace-dev/Dockerfile +++ b/docker/mace-dev/Dockerfile @@ -1,18 +1,58 @@ FROM registry.cn-hangzhou.aliyuncs.com/xiaomimace/mace-dev-lite:latest # Install tools +RUN apt-get update -y RUN apt-get install -y --no-install-recommends \ telnet \ net-tools \ inetutils-ping \ screen \ htop + RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ sphinx \ sphinx-autobuild \ sphinx_rtd_theme \ recommonmark +# ============ Install same deps for python3 ============ +RUN pyenv global 3.6.3 + +# Install tools +RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com setuptools +RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ + "numpy>=1.14.0" \ + scipy \ + jinja2 \ + pyyaml \ + sh==1.12.14 \ + pycodestyle==2.4.0 \ + filelock \ + PTable + +# Install Tensorflow +RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ + tensorflow==1.8.0 + +# Install pytorch (refer to: https://pytorch.org/get-started/locally/) +RUN pip install future==0.17.1 torchvision_nightly +RUN pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + +# Install ONNX +RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ + onnx==1.3.0 \ + onnx-tf==1.2.0 + +RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com \ + sphinx \ + sphinx-autobuild \ + sphinx_rtd_theme \ + recommonmark + +RUN pyenv global 2.7.14 +# ============ End installing same deps for python3 ============ + + # Customize vim RUN mkdir -p ~/.vim/autoload ~/.vim/bundle && \ curl -LSso ~/.vim/autoload/pathogen.vim https://tpo.pe/pathogen.vim @@ -34,11 +74,6 @@ RUN wget https://cmake.org/files/v3.11/cmake-3.11.3-Linux-x86_64.tar.gz -P /tmp/ rm -f /tmp/cmake-3.11.3-Linux-x86_64.tar.gz # mace-dev-lite image already included NDK r15c -## Download other NDK r12b -RUN cd /opt/ && \ - wget -q https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip && \ - unzip -q android-ndk-r12b-linux-x86_64.zip && \ - rm -f android-ndk-r12b-linux-x86_64.zip ## Download other NDK r16 RUN cd /opt/ && \ wget -q https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip && \ -- GitLab