From c7c475236899c5558ad2e0e9ca3b6ef93d7295e7 Mon Sep 17 00:00:00 2001 From: zhangchunle Date: Tue, 4 Aug 2020 11:08:26 +0800 Subject: [PATCH] cherrypick1.8 update paddle_build.sh (#25904) --- paddle/scripts/paddle_build.sh | 257 ++++++++++++++++++++++++++------- tools/print_signatures.py | 4 + tools/sampcd_processor.py | 16 +- 3 files changed, 223 insertions(+), 54 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 594c98e7fe3..0b6b006bbb2 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -59,11 +59,14 @@ function init() { } function cmake_base() { - # build script will not fail if *.deb does not exist + # Build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true - # delete previous built whl packages + # Delete previous built whl packages rm -rf python/dist 2>/dev/null || true + # `gym` is only used in unittest, it's not suitable to add in requirements.txt. + # Add it dynamically. + echo "gym" >> ${PADDLE_ROOT}/python/requirements.txt # Support build for all python versions, currently # including cp27-cp27m and cp27-cp27mu. PYTHON_FLAGS="" @@ -119,6 +122,8 @@ function cmake_base() { exit 1 fi fi + # delete `gym` to avoid modifying requirements.txt in *.whl + sed -i .bak "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt else if [ "$1" != "" ]; then echo "using python abi: $1" @@ -175,6 +180,8 @@ function cmake_base() { else pip install -r ${PADDLE_ROOT}/python/requirements.txt fi + # delete `gym` to avoid modifying requirements.txt in *.whl + sed -i "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt fi if [ "$SYSTEM" == "Darwin" ]; then @@ -193,15 +200,12 @@ function cmake_base() { Configuring cmake in /paddle/build ... -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} ${PYTHON_FLAGS} - -DWITH_DSO=ON -DWITH_GPU=${WITH_GPU:-OFF} -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} -DWITH_DISTRIBUTE=${distibuted_flag} -DWITH_MKL=${WITH_MKL:-ON} -DWITH_AVX=${WITH_AVX:-OFF} - -DWITH_GOLANG=${WITH_GOLANG:-OFF} -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} - -DCUDA_ARCH_BIN=${CUDA_ARCH_BIN} -DWITH_PYTHON=${WITH_PYTHON:-ON} -DCUDNN_ROOT=/usr/ -DWITH_TESTING=${WITH_TESTING:-ON} @@ -216,24 +220,23 @@ function cmake_base() { -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build} -DWITH_GRPC=${grpc_flag} -DWITH_LITE=${WITH_LITE:-OFF} + -DLITE_GIT_TAG=develop ======================================== EOF # Disable UNITTEST_USE_VIRTUALENV in docker because # docker environment is fully controlled by this script. # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option. + set +e cmake .. \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \ ${PYTHON_FLAGS} \ - -DWITH_DSO=ON \ -DWITH_GPU=${WITH_GPU:-OFF} \ -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} \ -DWITH_DISTRIBUTE=${distibuted_flag} \ -DWITH_MKL=${WITH_MKL:-ON} \ -DWITH_AVX=${WITH_AVX:-OFF} \ -DNOAVX_CORE_FILE=${NOAVX_CORE_FILE:-""} \ - -DWITH_GOLANG=${WITH_GOLANG:-OFF} \ -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \ - -DCUDA_ARCH_BIN=${CUDA_ARCH_BIN} \ -DWITH_PYTHON=${WITH_PYTHON:-ON} \ -DCUDNN_ROOT=/usr/ \ -DWITH_TESTING=${WITH_TESTING:-ON} \ @@ -246,8 +249,11 @@ EOF -DPY_VERSION=${PY_VERSION:-2.7} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build} \ -DWITH_GRPC=${grpc_flag} \ - -DWITH_LITE=${WITH_LITE:-OFF} - + -DLITE_GIT_TAG=develop \ + -DWITH_LITE=${WITH_LITE:-OFF};build_error=$? + if [ "$build_error" != 0 ];then + exit 7; + fi } function cmake_gen() { @@ -270,6 +276,7 @@ function check_style() { eval "$(GIMME_GO_VERSION=1.8.3 gimme)" fi + pip install cpplint pylint pytest astroid isort # set up go environment for running gometalinter mkdir -p $GOPATH/src/github.com/PaddlePaddle/ @@ -298,6 +305,7 @@ function check_style() { #================================================= function build_base() { + set +e if [ "$SYSTEM" == "Linux" ];then if [ `nproc` -gt 16 ];then parallel_number=$(expr `nproc` - 8) @@ -315,7 +323,36 @@ function build_base() { make clean fi - make install -j ${parallel_number} + make install -j ${parallel_number};build_error=$? + if [ "$build_error" != 0 ];then + exit 7; + fi +} + +function build_size() { + cat < $op_desc_path + # print api and the md5 of source code of the api. + api_source_md5_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.source.md5 + python ${PADDLE_ROOT}/tools/count_api_without_core_ops.py -p paddle > $api_source_md5_path + awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then @@ -612,9 +680,9 @@ EOF function assert_api_spec_approvals() { - /bin/bash ${PADDLE_ROOT}/tools/check_api_approvals.sh - if [ "$?" != 0 ];then - exit 1 + /bin/bash ${PADDLE_ROOT}/tools/check_api_approvals.sh;approval_error=$? + if [ "$approval_error" != 0 ];then + exit 6 fi } @@ -684,9 +752,42 @@ function caught_error() { done } +function case_count(){ + cat < 2: + if len(module.split('.')) > 1: filename = '../python/' module_py = '%s.py' % module.split('.')[-1] for i in range(0, len(module.split('.')) - 1): @@ -478,10 +478,16 @@ def get_filenames(): filename = filename + module_py else: filename = '' - print("\n----Exception in get api filename----\n") + print("\nWARNING:----Exception in get api filename----\n") print("\n" + api + ' module is ' + module + "\n") - if filename not in filenames: - filenames.append(filename) + if filename != '': + # rm contrib file + if filename.startswith( + '../python/paddle/fluid/contrib' + ) or filename == '../python/paddle/verison.py': + pass + elif filename not in filenames: + filenames.append(filename) # get all methods method = '' if inspect.isclass(eval(api)): @@ -492,7 +498,7 @@ def get_filenames(): name = '%s.%s' % (api.split('.')[-2], api.split('.')[-1]) else: name = '' - print("\n----Exception in get api methods----\n") + print("\nWARNING:----Exception in get api methods----\n") print("\n" + line + "\n") print("\n" + api + ' method is None!!!' + "\n") for j in range(2, len(module.split('.'))): -- GitLab