#!/usr/bin/env bash # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #================================================= # Utils #================================================= set -ex if [ -z ${BRANCH} ]; then BRANCH="develop" fi function print_usage() { echo -e "\n${RED}Usage${NONE}: ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]" echo -e "\n${RED}Options${NONE}: ${BLUE}build${NONE}: run build for x86 platform ${BLUE}test${NONE}: run all unit tests ${BLUE}single_test${NONE}: run a single unit test ${BLUE}bind_test${NONE}: parallel tests bind to different GPU ${BLUE}doc${NONE}: generate paddle documents ${BLUE}gen_doc_lib${NONE}: generate paddle documents library ${BLUE}html${NONE}: convert C++ source code into HTML ${BLUE}dockerfile${NONE}: generate paddle release dockerfile ${BLUE}fluid_inference_lib${NONE}: deploy fluid inference library ${BLUE}check_style${NONE}: run code style check ${BLUE}cicheck${NONE}: run CI tasks on Linux ${BLUE}maccheck${NONE}: run CI tasks on Mac " } function init() { RED='\033[0;31m' BLUE='\033[0;34m' BOLD='\033[1m' NONE='\033[0m' PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" if [ -z "${SCRIPT_NAME}" ]; then SCRIPT_NAME=$0 fi ENABLE_MAKE_CLEAN=${ENABLE_MAKE_CLEAN:-ON} } function cmake_base() { # Build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true # Delete previous built whl packages rm -rf python/dist 2>/dev/null || true # `gym` is only used in unittest, it's not suitable to add in requirements.txt. # Add it dynamically. echo "gym" >> ${PADDLE_ROOT}/python/requirements.txt # Support build for all python versions, currently # including cp27-cp27m and cp27-cp27mu. PYTHON_FLAGS="" SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then echo "Using python abi: $1" if [[ "$1" == "cp27-cp27m" ]] || [[ "$1" == "" ]]; then if [ -d "/Library/Frameworks/Python.framework/Versions/2.7" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7 export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7 export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7 -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/lib/libpython2.7.dylib" pip install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp35-cp35m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.5" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib" pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp36-cp36m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.6" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib" pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp37-cp37m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib" pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi fi # delete `gym` to avoid modifying requirements.txt in *.whl sed -i .bak "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt else if [ "$1" != "" ]; then echo "using python abi: $1" if [ "$1" == "cp27-cp27m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:} export PATH=/opt/python/cp27-cp27m/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so" pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp27-cp27mu" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:} export PATH=/opt/python/cp27-cp27mu/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so" pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp27-cp27m-gcc82" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs4/lib:} export PATH=/opt/python/cp27-cp27m/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs2/lib/libpython2.7.so" pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp27-cp27mu-gcc82" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs2/lib:} export PATH=/opt/python/cp27-cp27mu/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs4/lib/libpython2.7.so" pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp35-cp35m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so" pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp36-cp36m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so" pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp37-cp37m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so" pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt fi else pip install -r ${PADDLE_ROOT}/python/requirements.txt fi # delete `gym` to avoid modifying requirements.txt in *.whl sed -i "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt fi if [ "$SYSTEM" == "Darwin" ]; then WITH_DISTRIBUTE=${WITH_DISTRIBUTE:-ON} WITH_AVX=${WITH_AVX:-ON} INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-~/.cache/inference_demo} else INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-/root/.cache/inference_demo} fi distibuted_flag=${WITH_DISTRIBUTE:-OFF} grpc_flag=${WITH_GRPC:-${distibuted_flag}} cat <&2 echo "Please use pre-commit to check what is wrong." 1>&2 exit 1 } function check_style() { trap 'abort' 0 set -e if [ -x "$(command -v gimme)" ]; then eval "$(GIMME_GO_VERSION=1.8.3 gimme)" fi pip install cpplint pylint pytest astroid isort # set up go environment for running gometalinter mkdir -p $GOPATH/src/github.com/PaddlePaddle/ ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle pre-commit install clang-format --version commit_files=on for file_name in `git diff --numstat upstream/$BRANCH |awk '{print $NF}'`;do if ! pre-commit run --files $file_name ; then git diff commit_files=off fi done if [ $commit_files == 'off' ];then echo "code format error" exit 1 fi trap : 0 } #================================================= # Build #================================================= function build_base() { set +e if [ "$SYSTEM" == "Linux" ];then if [ `nproc` -gt 16 ];then parallel_number=$(expr `nproc` - 8) else parallel_number=`nproc` fi else parallel_number=8 fi if [ "$1" != "" ]; then parallel_number=$1 fi if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then make clean fi make install -j ${parallel_number};build_error=$? if [ "$build_error" != 0 ];then exit 7; fi } function build_size() { cat < $spec_path # used to log op_register data_type op_type_path=${PADDLE_ROOT}/paddle/fluid/OP_TYPE_${spec_kind}.spec python ${PADDLE_ROOT}/tools/check_op_register_type.py > $op_type_path # print all ops desc in dict to op_desc_path op_desc_path=${PADDLE_ROOT}/paddle/fluid/OP_DESC_${spec_kind}.spec python ${PADDLE_ROOT}/tools/print_op_desc.py > $op_desc_path # print api and the md5 of source code of the api. api_source_md5_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.source.md5 python ${PADDLE_ROOT}/tools/count_api_without_core_ops.py -p paddle > $api_source_md5_path awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then # Use sed to make python2 and python3 sepc keeps the same sed -i 's/arg0: str/arg0: unicode/g' $spec_path sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path fi python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \ ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec deactivate } function check_approvals_of_unittest() { if [ "$GITHUB_API_TOKEN" == "" ] || [ "$GIT_PR_ID" == "" ]; then return 0 fi # approval_user_list: XiaoguangHu01 46782768,luotao1 6836917,phlrain 43953930,lanxianghit 47554610, zhouwei25 52485244, kolinwei 22165420 check_times=$1 if [ $check_times == 1 ]; then approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000` if [ "${approval_line}" != "" ]; then APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244` set +x echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}" if [ "${APPROVALS}" == "TRUE" ]; then echo "===================================" echo -e "\n current pr ${GIT_PR_ID} has got approvals. So, Pass CI directly!\n" echo "===================================" exit 0 fi fi elif [ $check_times == 2 ]; then unittest_spec_diff=`python ${PADDLE_ROOT}/tools/diff_unittest.py ${PADDLE_ROOT}/paddle/fluid/UNITTEST_DEV.spec ${PADDLE_ROOT}/paddle/fluid/UNITTEST_PR.spec` if [ "$unittest_spec_diff" != "" ]; then approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000` APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244` set +x echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}" if [ "${APPROVALS}" == "FALSE" ]; then echo "************************************" echo -e "It is forbidden to disable or delete the unit-test.\n" echo -e "If you must delete it temporarily, please add it to[https://github.com/PaddlePaddle/Paddle/wiki/Temporarily-disabled-Unit-Test]." echo -e "Then you must have one RD (kolinwei(recommended) or zhouwei25) approval for the deletion of unit-test. \n" echo -e "If you have any problems about deleting unit-test, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/Deleting-unit-test-is-forbidden]. \n" echo -e "Following unit-tests are deleted in this PR: \n ${unittest_spec_diff} \n" echo "************************************" exit 1 fi fi fi set -x } function check_change_of_unittest() { generate_unittest_spec "PR" fetch_upstream_develop_if_not_exist git reset --hard upstream/$BRANCH cmake_gen $1 generate_unittest_spec "DEV" check_approvals_of_unittest 2 } function check_sequence_op_unittest(){ /bin/bash ${PADDLE_ROOT}/tools/check_sequence_op.sh } function generate_unittest_spec() { spec_kind=$1 if [ "$spec_kind" == "DEV" ]; then cat < ${spec_path} } function assert_api_spec_approvals() { /bin/bash ${PADDLE_ROOT}/tools/check_api_approvals.sh;approval_error=$? if [ "$approval_error" != 0 ];then exit 6 fi } function check_coverage() { /bin/bash ${PADDLE_ROOT}/tools/coverage/paddle_coverage.sh } function single_test() { TEST_NAME=$1 if [ -z "${TEST_NAME}" ]; then echo -e "${RED}Usage:${NONE}" echo -e "${BOLD}${SCRIPT_NAME}${NONE} ${BLUE}single_test${NONE} [test_name]" exit 1 fi mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build if [ ${WITH_TESTING:-ON} == "ON" ] ; then cat < ${job}" if ! wait ${job} ; then echo "At least one test failed with exit code => $?" ; EXIT_CODE=1; fi done } function case_count(){ cat < 1 )); then cardnumber=$2 if (( $cardnumber > $CUDA_DEVICE_COUNT )); then cardnumber=$CUDA_DEVICE_COUNT fi else cardnumber=$CUDA_DEVICE_COUNT fi if [[ "$testcases" == "" ]]; then return 0 fi trap 'caught_error' CHLD tmpfile_rand=`date +%s%N` NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber] for (( i = 0; i < $NUM_PROC; i++ )); do # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest cuda_list=() for (( j = 0; j < cardnumber; j++ )); do if [ $j -eq 0 ]; then cuda_list=("$[i*cardnumber]") else cuda_list="$cuda_list,$[i*cardnumber+j]" fi done tmpfile=$tmp_dir/$tmpfile_rand"_"$i if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then (ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) & else (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) & fi else if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then (ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) & else (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) & fi fi done wait; # wait for all subshells to finish ut_endTime_s=`date +%s` if [ "$2" == "" ]; then echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s" else echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s" fi set +m } function parallel_test_base_gpu() { if [ ${WITH_TESTING:-ON} == "ON" ] ; then cat < ${PADDLE_ROOT}/build/Dockerfile < ENV HOME /root EOF if [[ ${WITH_GPU} == "ON" ]]; then NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true" else NCCL_DEPS="true" fi if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0" fi PADDLE_VERSION="paddle version" CMD='"paddle", "version"' cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.6.0.tgz RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ pip3.6 install opencv-python && wget ${ref_web}/${ref_paddle36} && ${ref_paddle36_mv1} pip3.6 install ${ref_paddle36_whl} ${ref_paddle36_mv2}; apt-get install -f -y && \ apt-get clean -y && \ rm -f ${ref_paddle36} && \ ldconfig EOF cat >> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.7.0.tgz RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ pip3.7 install opencv-python && wget ${ref_web}/${ref_paddle37} && pip3.7 install ${ref_paddle37_whl}; apt-get install -f -y && \ apt-get clean -y && \ rm -f ${ref_paddle37} && \ ldconfig EOF cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <