#!/usr/bin/env bash # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #================================================= # Utils #================================================= set -ex function print_usage() { echo -e "\n${RED}Usage${NONE}: ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]" echo -e "\n${RED}Options${NONE}: ${BLUE}build${NONE}: run build for x86 platform ${BLUE}test${NONE}: run all unit tests ${BLUE}single_test${NONE}: run a single unit test ${BLUE}bind_test${NONE}: parallel tests bind to different GPU ${BLUE}doc${NONE}: generate paddle documents ${BLUE}gen_doc_lib${NONE}: generate paddle documents library ${BLUE}html${NONE}: convert C++ source code into HTML ${BLUE}dockerfile${NONE}: generate paddle release dockerfile ${BLUE}fluid_inference_lib${NONE}: deploy fluid inference library ${BLUE}check_style${NONE}: run code style check ${BLUE}cicheck${NONE}: run CI tasks ${BLUE}assert_api_not_changed${NONE}: check api compability " } function init() { RED='\033[0;31m' BLUE='\033[0;34m' BOLD='\033[1m' NONE='\033[0m' PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" if [ -z "${SCRIPT_NAME}" ]; then SCRIPT_NAME=$0 fi } function cmake_base() { # build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true # delete previous built whl packages rm -rf python/dist 2>/dev/null || true # Support build for all python versions, currently # including cp27-cp27m and cp27-cp27mu. PYTHON_FLAGS="" SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then echo "Using python abi: $1" if [[ "$1" == "cp27-cp27m" ]] || [[ "$1" == "" ]]; then if [ -d "/Library/Frameworks/Python.framework/Versions/2.7" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7 export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7 export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7 -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/lib/libpython2.7.dylib" pip install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp35-cp35m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.5" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib" pip3.5 uninstall -y protobuf pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp36-cp36m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.6" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib" pip3.6 uninstall -y protobuf pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi elif [ "$1" == "cp37-cp37m" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/ export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/ export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/ -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib" pip3.7 uninstall -y protobuf pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt else exit 1 fi fi else if [ "$1" != "" ]; then echo "using python abi: $1" if [ "$1" == "cp27-cp27m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:} export PATH=/opt/python/cp27-cp27m/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so" pip uninstall -y protobuf pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp27-cp27mu" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:} export PATH=/opt/python/cp27-cp27mu/bin/:${PATH} PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so" pip uninstall -y protobuf pip install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp35-cp35m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so" pip3.5 uninstall -y protobuf pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp36-cp36m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so" pip3.6 uninstall -y protobuf pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "cp37-cp37m" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH} export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7 -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so" pip3.7 uninstall -y protobuf pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt fi else pip uninstall -y protobuf pip install -r ${PADDLE_ROOT}/python/requirements.txt fi fi if [ "$SYSTEM" == "Darwin" ]; then WITH_DISTRIBUTE=${WITH_DISTRIBUTE:-ON} WITH_AVX=${WITH_AVX:-ON} INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-~/.cache/inference_demo} else INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-/root/.cache/inference_demo} fi distibuted_flag=${WITH_DISTRIBUTE:-OFF} grpc_flag=${WITH_GRPC:-${distibuted_flag}} cat <&2 echo "Please use pre-commit to check what is wrong." 1>&2 exit 1 } function check_style() { trap 'abort' 0 set -e if [ -x "$(command -v gimme)" ]; then eval "$(GIMME_GO_VERSION=1.8.3 gimme)" fi pip install cpplint # set up go environment for running gometalinter mkdir -p $GOPATH/src/github.com/PaddlePaddle/ ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle mkdir -p ./build/go cp go/glide.* build/go cd build/go; glide install; cd - export PATH=/usr/bin:$PATH pre-commit install clang-format --version if ! pre-commit run -a ; then git diff exit 1 fi trap : 0 } #================================================= # Build #================================================= function build_base() { if [ "$SYSTEM" == "Linux" ];then parallel_number=`nproc` else parallel_number=8 fi if [ "$1" != "" ]; then parallel_number=$1 fi make clean make -j ${parallel_number} make install -j ${parallel_number} } function build() { mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build cat < new.spec if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then # Use sed to make python2 and python3 sepc keeps the same sed -i 's/arg0: str/arg0: unicode/g' new.spec sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" new.spec fi # ComposeNotAligned has significant difference between py2 and py3 sed -i '/.*ComposeNotAligned.*/d' new.spec python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API.spec new.spec # Currently, we only check in PR_CI python 2.7 if [ "$SYSTEM" != "Darwin" ]; then if [ "$1" == "" ] || [ "$1" == "cp27-cp27m" ] || [ "$1" == "cp27-cp27mu" ]; then python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_op_maker.spec fi fi deactivate } function assert_api_spec_approvals() { if [ -z ${BRANCH} ]; then BRANCH="develop" fi API_FILES=("CMakeLists.txt" "paddle/fluid/API.spec" "paddle/fluid/op_use_default_grad_op_maker.spec" "python/paddle/fluid/parallel_executor.py" "paddle/fluid/framework/operator.h" "paddle/fluid/framework/tensor.h" "paddle/fluid/framework/details/op_registry.h" "paddle/fluid/framework/grad_op_desc_maker.h" "paddle/fluid/framework/lod_tensor.h" "paddle/fluid/framework/selected_rows.h" "paddle/fluid/framework/op_desc.h" "paddle/fluid/framework/block_desc.h" "paddle/fluid/framework/var_desc.h" "paddle/fluid/framework/scope.h" "paddle/fluid/framework/ir/node.h" "paddle/fluid/framework/ir/graph.h" "paddle/fluid/framework/framework.proto" "python/paddle/fluid/compiler.py" "python/paddle/fluid/__init__.py" "paddle/fluid/operators/distributed/send_recv.proto.in") for API_FILE in ${API_FILES[*]}; do API_CHANGE=`git diff --name-only upstream/$BRANCH | grep "${API_FILE}" | grep -v "/CMakeLists.txt" || true` echo "checking ${API_FILE} change, PR: ${GIT_PR_ID}, changes: ${API_CHANGE}" if [ "${API_CHANGE}" ] && [ "${GIT_PR_ID}" != "" ]; then # NOTE: per_page=10000 should be ok for all cases, a PR review > 10000 is not human readable. # approval_user_list: XiaoguangHu01 46782768,chengduoZH 30176695,Xreki 12538138,luotao1 6836917,sneaxiy 32832641,tensor-tang 21351065,xsrobin 50069408,qingqing01 7845005,guoshengCS 14105589,heavengate 12605721,kuke 3064195,Superjomn 328693,lanxianghit 47554610,cyj1986 39645414,hutuxian 11195205,frankwhzhang 20274488,nepeplwu 45024560,Dianhai 5086632,JiabinYang 22361972,chenwhql 22561442. approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000` if [ "${API_FILE}" == "paddle/fluid/API.spec" ];then APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 2 7534971 14105589 12605721 3064195 328693 47554610 39645414 11195205 20274488 45024560 ` elif [ "${API_FILE}" == "CMakeLists.txt" ];then APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \ python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 6836917 46782768 30176695` elif [ "${API_FILE}" == "python/paddle/fluid/__init__.py" ];then APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 6836917 47554610` elif [ "${API_FILE}" == "python/requirements.txt" ];then APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 6836917 22361972` else APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \ python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 21351065 3048612 46782768 30176695 12538138 6836917 32832641` fi echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}" if [ "${APPROVALS}" == "FALSE" ]; then if [ "${API_FILE}" == "paddle/fluid/API.spec" ];then echo "You must have two RD (wanghaoshuang or guoshengCS or heavengate or kuke or Superjomn or lanxianghit or cyj1986 or hutuxian or frankwhzhang or nepeplwu) approval for the api change! ${API_FILE} for the management reason of API interface and API document." elif [ "${API_FILE}" == "CMakeLists.txt" ];then echo "You must have one RD (luotao1 or chengduoZH or XiaoguangHu01) approval for the cmakelist change! ${API_FILE} for the management reason of the Compilation parameter." elif [ "${API_FILE}" == "python/paddle/fluid/__init__.py" ];then echo "You must have one RD (lanxianghit (Recommend) or luotao1) approval for the python/paddle/fluid/init.py, which manages the environment variables." elif [ "${API_FILE}" == "python/requirements.txt" ];then echo "You must have one RD (JiabinYang (Recommend) or luotao1) approval for python/requirements.txt, which manages the third-party python package." else echo "You must have one RD (XiaoguangHu01,chengduoZH,Xreki,luotao1,sneaxiy,tensor-tang,jacquesqiao) approval for the api change! ${API_FILE} for the management reason of the underlying code for fluid." fi exit 1 fi fi done HAS_CONST_CAST=`git diff -U0 upstream/$BRANCH |grep -o -m 1 "const_cast" || true` if [ ${HAS_CONST_CAST} ] && [ "${GIT_PR_ID}" != "" ]; then APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \ python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 21351065 3048612 46782768 30176695 12538138 6836917 32832641` echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}" if [ "${APPROVALS}" == "FALSE" ]; then echo "You must have one RD (XiaoguangHu01,chengduoZH,Xreki,luotao1,sneaxiy,tensor-tang,jacquesqiao) approval for the api change! ${API_FILE} for the avoidance of the bad C++ code habits." exit 1 fi fi } function single_test() { TEST_NAME=$1 if [ -z "${TEST_NAME}" ]; then echo -e "${RED}Usage:${NONE}" echo -e "${BOLD}${SCRIPT_NAME}${NONE} ${BLUE}single_test${NONE} [test_name]" exit 1 fi mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build if [ ${WITH_TESTING:-ON} == "ON" ] ; then cat < ${job}" if ! wait ${job} ; then echo "At least one test failed with exit code => $?" ; EXIT_CODE=1; fi done } function card_test() { set -m # get the CUDA device count CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l) testcases=$1 if (( $# > 1 )); then cardnumber=$2 if (( $cardnumber > $CUDA_DEVICE_COUNT )); then cardnumber=$CUDA_DEVICE_COUNT fi else cardnumber=$CUDA_DEVICE_COUNT fi if [[ "$testcases" == "" ]]; then return 0 fi trap 'caught_error' CHLD NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber] for (( i = 0; i < $NUM_PROC; i++ )); do # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest cuda_list=() for (( j = 0; j < cardnumber; j++ )); do if [ $j -eq 0 ]; then cuda_list=("$[i*cardnumber]") else cuda_list="$cuda_list,$[i*cardnumber+j]" fi done if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then ctest -I $i,,$NUM_PROC -R "($testcases)" -V & else env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V & fi else if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure & else # echo "env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R \"($testcases)\" --output-on-failure &" env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure & fi fi done wait; # wait for all subshells to finish set +m } function parallel_test_base() { if [ ${WITH_TESTING:-ON} == "ON" ] ; then cat < ${PADDLE_ROOT}/build/Dockerfile < ENV HOME /root EOF if [[ ${WITH_GPU} == "ON" ]]; then NCCL_DEPS="apt-get install -y --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true" else NCCL_DEPS="true" fi PADDLE_VERSION="paddle version" CMD='"paddle", "version"' if [ "$1" == "cp35-cp35m" ]; then cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ make -j8 > /dev/null && make altinstall > /dev/null RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \ pip3.6 install opencv-python && pip3.6 install /*.whl; apt-get install -f -y && \ apt-get clean -y && \ rm -f /*.whl && \ ${PADDLE_VERSION} && \ ldconfig ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_CUBLAS_DSO} ${DOCKERFILE_GPU_ENV} EOF elif [ "$1" == "cp37-cp37m" ]; then cat >> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ make -j8 > /dev/null && make altinstall > /dev/null RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \ pip3.7 install opencv-python && pip3.7 install /*.whl; apt-get install -f -y && \ apt-get clean -y && \ rm -f /*.whl && \ ${PADDLE_VERSION} && \ ldconfig ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_CUBLAS_DSO} ${DOCKERFILE_GPU_ENV} EOF else cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <