paddle_build.sh 67.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#!/usr/bin/env bash

# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#=================================================
#                   Utils
#=================================================

21 22
set -ex

23 24 25 26
if [ -z ${BRANCH} ]; then
    BRANCH="develop"
fi

27 28
function print_usage() {
    echo -e "\n${RED}Usage${NONE}:
29
    ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]"
Q
Qiyang Min 已提交
30

31 32 33
    echo -e "\n${RED}Options${NONE}:
    ${BLUE}build${NONE}: run build for x86 platform
    ${BLUE}test${NONE}: run all unit tests
34
    ${BLUE}single_test${NONE}: run a single unit test
35 36
    ${BLUE}bind_test${NONE}: parallel tests bind to different GPU
    ${BLUE}doc${NONE}: generate paddle documents
37
    ${BLUE}gen_doc_lib${NONE}: generate paddle documents library
38 39 40 41
    ${BLUE}html${NONE}: convert C++ source code into HTML
    ${BLUE}dockerfile${NONE}: generate paddle release dockerfile
    ${BLUE}fluid_inference_lib${NONE}: deploy fluid inference library
    ${BLUE}check_style${NONE}: run code style check
42 43
    ${BLUE}cicheck${NONE}: run CI tasks on Linux
    ${BLUE}maccheck${NONE}: run CI tasks on Mac
44 45 46 47
    "
}

function init() {
48 49 50 51 52
    RED='\033[0;31m'
    BLUE='\033[0;34m'
    BOLD='\033[1m'
    NONE='\033[0m'

53
    PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
54 55 56
    if [ -z "${SCRIPT_NAME}" ]; then
        SCRIPT_NAME=$0
    fi
57 58

    ENABLE_MAKE_CLEAN=${ENABLE_MAKE_CLEAN:-ON}
59 60
}

61
function cmake_base() {
62
    # Build script will not fail if *.deb does not exist
63
    rm *.deb 2>/dev/null || true
64
    # Delete previous built whl packages
65 66
    rm -rf python/dist 2>/dev/null || true

67 68 69
    # `gym` is only used in unittest, it's not suitable to add in requirements.txt.
    # Add it dynamically.
    echo "gym" >> ${PADDLE_ROOT}/python/requirements.txt
70 71 72
    # Support build for all python versions, currently
    # including cp27-cp27m and cp27-cp27mu.
    PYTHON_FLAGS=""
73
    SYSTEM=`uname -s`
J
Jiabin Yang 已提交
74
    if [ "$SYSTEM" == "Darwin" ]; then
V
velconia 已提交
75
        echo "Using python abi: $1"
J
Jiabin Yang 已提交
76
        if [[ "$1" == "cp27-cp27m" ]] || [[ "$1" == "" ]]; then
77 78 79
            if [ -d "/Library/Frameworks/Python.framework/Versions/2.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
J
Jiabin Yang 已提交
80
                export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:${PATH}
81 82
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7
J
Jiabin Yang 已提交
83
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/lib/libpython2.7.dylib"
84
                pip install --user -r ${PADDLE_ROOT}/python/requirements.txt
85 86 87
            else
                exit 1
            fi
88
        elif [ "$1" == "cp35-cp35m" ]; then
V
velconia 已提交
89
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.5" ]; then
90 91 92 93 94 95
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib"
96
                pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt
97 98 99
            else
                exit 1
            fi
M
minqiyang 已提交
100 101 102 103 104 105 106 107
        elif [ "$1" == "cp36-cp36m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.6" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib"
108
                pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
109 110 111 112 113 114 115 116 117 118 119
            else
                exit 1
            fi
        elif [ "$1" == "cp37-cp37m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib"
120
                pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
121 122 123
            else
                exit 1
            fi
G
guofei 已提交
124 125 126 127 128 129 130 131 132 133 134 135
        elif [ "$1" == "cp38-cp38" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.8" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/libpython3.8.dylib"
                pip3.8 install --user -r ${PADDLE_ROOT}/python/requirements.txt
            else
                exit 1
            fi
136
        fi
137 138
        # delete `gym` to avoid modifying requirements.txt in *.whl
        sed -i .bak "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt
139
    else
140
        if [ "$1" != "" ]; then
141
            echo "using python abi: $1"
142
            if [ "$1" == "cp27-cp27m" ]; then
143
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs4/lib:}
144 145 146
                export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
147
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs2/lib/libpython2.7.so"
148
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
149
            elif [ "$1" == "cp27-cp27mu" ]; then
150
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs2/lib:}
151 152 153
                export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
154
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs4/lib/libpython2.7.so"
155 156 157 158 159 160 161 162 163 164 165 166 167 168
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
            elif [ "$1" == "cp27-cp27m-gcc82" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs4/lib:}
                export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs2/lib/libpython2.7.so"
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
            elif [ "$1" == "cp27-cp27mu-gcc82" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs2/lib:}
                export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs4/lib/libpython2.7.so"
169
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
170 171 172 173
            elif [ "$1" == "cp35-cp35m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3
174 175
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so"
176
                pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt
177 178 179 180 181 182
            elif [ "$1" == "cp36-cp36m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so"
183
                pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt
184 185 186
            elif [ "$1" == "cp37-cp37m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH}
187
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7
188 189
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so"
190
                pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
G
guofei 已提交
191 192 193 194 195 196 197
            elif [ "$1" == "cp38-cp38" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.8.0/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.8.0/bin/python3.8
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.8.0/include/python3.8
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.8.0/lib/libpython3.so"
                pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt
198
           fi
199 200
        else
            pip install -r ${PADDLE_ROOT}/python/requirements.txt
201
        fi
202 203
        # delete `gym` to avoid modifying requirements.txt in *.whl
        sed -i "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt
204
    fi
205

J
JiabinYang 已提交
206
    if [ "$SYSTEM" == "Darwin" ]; then
207 208
        WITH_DISTRIBUTE=${WITH_DISTRIBUTE:-ON}
        WITH_AVX=${WITH_AVX:-ON}
J
Jiabin Yang 已提交
209
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-~/.cache/inference_demo}
J
JiabinYang 已提交
210
    else
J
Jiabin Yang 已提交
211
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-/root/.cache/inference_demo}
212
    fi
213

G
gongweibao 已提交
214 215 216
    distibuted_flag=${WITH_DISTRIBUTE:-OFF}
    grpc_flag=${WITH_GRPC:-${distibuted_flag}}

217 218 219 220 221 222
    if [ "$SYSTEM" == "Darwin" ]; then
        gloo_flag="OFF"
    else
        gloo_flag=${distibuted_flag}
    fi

223 224 225 226 227 228 229
    cat <<EOF
    ========================================
    Configuring cmake in /paddle/build ...
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
        ${PYTHON_FLAGS}
        -DWITH_GPU=${WITH_GPU:-OFF}
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF}
G
gongweibao 已提交
230
        -DWITH_DISTRIBUTE=${distibuted_flag}
231 232 233 234 235 236
        -DWITH_MKL=${WITH_MKL:-ON}
        -DWITH_AVX=${WITH_AVX:-OFF}
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All}
        -DWITH_PYTHON=${WITH_PYTHON:-ON}
        -DCUDNN_ROOT=/usr/
        -DWITH_TESTING=${WITH_TESTING:-ON}
237
        -DWITH_COVERAGE=${WITH_COVERAGE:-OFF}
238 239 240 241 242
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON}
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON}
243
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR}
244
        -DPY_VERSION=${PY_VERSION:-2.7}
245
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build}
G
gongweibao 已提交
246
        -DWITH_GRPC=${grpc_flag}
247
	    -DWITH_GLOO=${gloo_flag}
248
        -DWITH_LITE=${WITH_LITE:-OFF}
249
        -DLITE_GIT_TAG=develop
250
    ========================================
251
EOF
252 253 254
    # Disable UNITTEST_USE_VIRTUALENV in docker because
    # docker environment is fully controlled by this script.
    # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
255
    set +e
256 257 258 259 260
    cmake .. \
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
        ${PYTHON_FLAGS} \
        -DWITH_GPU=${WITH_GPU:-OFF} \
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} \
G
gongweibao 已提交
261
        -DWITH_DISTRIBUTE=${distibuted_flag} \
262 263
        -DWITH_MKL=${WITH_MKL:-ON} \
        -DWITH_AVX=${WITH_AVX:-OFF} \
264
        -DNOAVX_CORE_FILE=${NOAVX_CORE_FILE:-""} \
265 266 267 268
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \
        -DWITH_PYTHON=${WITH_PYTHON:-ON} \
        -DCUDNN_ROOT=/usr/ \
        -DWITH_TESTING=${WITH_TESTING:-ON} \
269
        -DWITH_COVERAGE=${WITH_COVERAGE:-OFF} \
270 271 272 273
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake \
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON} \
274
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR} \
275
        -DPY_VERSION=${PY_VERSION:-2.7} \
M
minqiyang 已提交
276
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build} \
277
        -DWITH_GRPC=${grpc_flag} \
278
	    -DWITH_GLOO=${gloo_flag} \
279
        -DLITE_GIT_TAG=develop \
280 281 282 283
        -DWITH_LITE=${WITH_LITE:-OFF};build_error=$?
    if [ "$build_error" != 0 ];then
        exit 7;
    fi
284 285
}

286 287 288 289 290 291
function cmake_gen() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cmake_base $1
}

292 293 294
function abort(){
    echo "Your change doesn't follow PaddlePaddle's code style." 1>&2
    echo "Please use pre-commit to check what is wrong." 1>&2
295
    exit 4
296 297 298 299 300 301
}

function check_style() {
    trap 'abort' 0
    set -e

302 303 304
    if [ -x "$(command -v gimme)" ]; then
    	eval "$(GIMME_GO_VERSION=1.8.3 gimme)"
    fi
305

306 307

    pip install cpplint pylint pytest astroid isort
308 309 310 311 312 313 314
    # set up go environment for running gometalinter
    mkdir -p $GOPATH/src/github.com/PaddlePaddle/
    ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle

    pre-commit install
    clang-format --version

315
    commit_files=on
316 317
    for file_name in `git diff --numstat upstream/$BRANCH |awk '{print $NF}'`;do
        if ! pre-commit run --files $file_name ; then
318
            commit_files=off
319 320
        fi
    done 
321 322 323
    
    if [ $commit_files == 'off' ];then
        echo "code format error"
324
        git diff 2>&1
325
        exit 4
326
    fi
327 328 329 330 331 332 333
    trap : 0
}

#=================================================
#              Build
#=================================================

334
function build_base() {
335
    set +e
336
    if [ "$SYSTEM" == "Linux" ];then
337
      if [ `nproc` -gt 16 ];then
338
          parallel_number=$(expr `nproc` - 8)
339 340 341
      else
          parallel_number=`nproc`
      fi
342 343 344 345
    else
      parallel_number=8
    fi
    if [ "$1" != "" ]; then
346 347
      parallel_number=$1
    fi
348 349 350 351 352

    if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then
        make clean
    fi

353 354 355 356
    make install -j ${parallel_number};build_error=$?
    if [ "$build_error" != 0 ];then
        exit 7;
    fi
357 358
}

359 360 361 362 363 364
function build_size() {
    cat <<EOF
    ============================================
    Calculate /paddle/build size and PR whl size
    ============================================
EOF
365
    if [ "$1" == "paddle_inference" ]; then
366
        cd ${PADDLE_ROOT}/build
367 368 369 370
        cp -r paddle_inference_install_dir paddle_inference
        tar -czf paddle_inference.tgz paddle_inference
        buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference.tgz |awk '{print $1}')
        echo "Paddle_Inference Size: $buildSize"
371 372 373 374 375 376 377 378 379 380 381 382 383 384
    else
        SYSTEM=`uname -s`
        if [ "$SYSTEM" == "Darwin" ]; then
            com='du -h -d 0'
        else
            com='du -h --max-depth=0'
        fi
        buildSize=$($com ${PADDLE_ROOT}/build |awk '{print $1}')
        echo "Build Size: $buildSize"
        PR_whlSize=$($com ${PADDLE_ROOT}/build/python/dist |awk '{print $1}')
        echo "PR whl Size: $PR_whlSize"
    fi
}

385 386 387 388 389 390 391 392
function build() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
393
    build_base $@
394 395 396 397 398 399 400 401 402 403 404 405
    current_branch=`git branch | grep \* | cut -d ' ' -f2`
    if [ "$current_branch" != "develop_base_pr" ];then
        build_size
    fi
}

function cmake_gen_and_build() {
    startTime_s=`date +%s`
    cmake_gen $1
    build $2
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
406 407
}

408
function build_mac() {
409
    set +e
410 411 412 413 414 415 416
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
417 418 419
    if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then
        make clean
    fi
420 421 422 423 424
    make install -j 8;build_error=$?
    if [ "$build_error" != 0 ];then
        exit 7;
    fi
    set -e
425 426 427 428 429 430 431 432 433
    build_size
}

function cmake_gen_and_build_mac() {
    startTime_s=`date +%s`
    cmake_gen $1
    build_mac
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
434 435
}

436 437 438 439 440 441 442 443 444
function run_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
445 446 447 448 449
        if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
            ctest -V
        else
            ctest --output-on-failure
        fi
450 451 452
    fi
}

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471

function combine_avx_noavx_build() {
    mkdir -p ${PADDLE_ROOT}/build.noavx
    cd ${PADDLE_ROOT}/build.noavx
    WITH_AVX=OFF
    cmake_base ${PYTHON_ABI:-""}
    build_base

    # build combined one
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    NOAVX_CORE_FILE=`find ${PADDLE_ROOT}/build.noavx/python/paddle/fluid/ -name "core_noavx.*"`
    WITH_AVX=ON

    cmake_base ${PYTHON_ABI:-""}
    build_base
}


G
gongweibao 已提交
472 473 474 475 476 477 478 479 480 481 482 483
function run_brpc_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [[ ${WITH_TESTING:-ON} == "ON" \
        && ${WITH_DISTRIBUTE:-OFF} == "ON" \
        && ${WITH_GRPC:-OFF} == "OFF" ]] ; then
    cat <<EOF
    ========================================
    Running brpc unit tests ...
    ========================================
EOF
        set +x
484 485
        declare -a other_tests=("test_listen_and_serv_op" "system_allocator_test" \
        "rpc_server_test" "varhandle_test" "collective_server_test" "brpc_serde_test")
G
gongweibao 已提交
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
        all_tests=`ctest -N`

        for t in "${other_tests[@]}"
        do
            if [[ ${all_tests} != *$t* ]]; then
                continue
            fi

            if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
                ctest -V -R $t
            else
                ctest --output-on-failure -R $t
            fi
        done
        set -x

        if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
            ctest -V -R test_dist_*
        else
            ctest --output-on-failure -R test_dist_*
        fi
    fi
}



512 513 514 515 516 517 518 519 520
function run_mac_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
521 522
        #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. 
        #see details: https://github.com/PaddlePaddle/Paddle/issues/24738
523
        my_proxy=$http_proxy
J
JiabinYang 已提交
524 525
        export http_proxy=
        export https_proxy=
526 527 528 529 530 531 532 533 534 535

        set +ex
        if [ "$1" == "cp27-cp27m" ]; then
            pip uninstall -y paddlepaddle
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 uninstall -y paddlepaddle
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 uninstall -y paddlepaddle
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 uninstall -y paddlepaddle
G
guofei 已提交
536 537
        elif [ "$1" == "cp38-cp38" ]; then
            pip3.8 uninstall -y paddlepaddle
538 539 540
        fi
        set -ex

541
        if [ "$1" == "cp27-cp27m" ]; then
542
            set -e
543
            pip install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
544
            python ${PADDLE_ROOT}/paddle/scripts/installation_validate.py
545 546 547 548 549 550
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
G
guofei 已提交
551 552
        elif [ "$1" == "cp38-cp38" ]; then
            pip3.8 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
553
        fi
554 555
        tmpfile_rand=`date +%s%N`
        tmpfile=$tmp_dir/$tmpfile_rand
Z
zhangchunle 已提交
556
        set +e
557
        ut_startTime_s=`date +%s`
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
        ctest --output-on-failure -j $2 | tee $tmpfile
        failed_test_lists=''
        collect_failed_tests
        set +x
        mactest_error=0
        retry_unittests_record=''
        retry_time=3
        exec_times=0
        exec_time_array=('first' 'second' 'third')
        if [ -n "$failed_test_lists" ];then
            mactest_error=1
            while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
                do
                    retry_unittests_record="$retry_unittests_record$failed_test_lists"
                    failed_test_lists_ult=`echo "${failed_test_lists}"`
                    read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
                    echo "========================================="
                    echo "This is the ${exec_time_array[$exec_times]} time to re-run"
                    echo "========================================="
                    echo "The following unittest will be re-run:"
                    echo "${retry_unittests}"
                    echo "========================================="

                    retry_unittests_regular=''
                    for line in ${retry_unittests[@]} ;
                        do
                            if [[ "$retry_unittests_regular" == "" ]];then
                                retry_unittests_regular="^$line$"
                            else
                                retry_unittests_regular="$retry_unittests_regular|^$line$"
                            fi
                        done
                    rm -f $tmp_dir/*
                    failed_test_lists=''
                    ctest -R "($retry_unittests_regular)" --output-on-failure -j $2 | tee $tmpfile
                    collect_failed_tests
                    exec_times=$[$exec_times+1]
                done
        fi
        #mactest_error=$?
598 599
        ut_endTime_s=`date +%s`
        echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
600
        paddle version
601 602 603
        # Recovery proxy to avoid failure in later steps
        export http_proxy=$my_proxy
        export https_proxy=$my_proxy
604
        if [ "$mactest_error" != 0 ];then
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
            if [[ "$failed_test_lists" == "" ]]; then
                echo "========================================"
                echo "There are failed tests, which have been successful after re-run:"
                echo "========================================"
                echo "The following tests have been re-ran:"
                echo "${retry_unittests_record}"
            else
                failed_test_lists_ult=`echo "${failed_test_lists}"`
                echo "========================================"
                echo "Summary Failed Tests... "
                echo "========================================"
                echo "The following tests FAILED: "
                echo "${failed_test_lists_ult}"
                exit 8;
            fi
620
        fi
621 622 623
    fi
}

624 625 626 627 628 629 630 631 632 633 634 635 636 637
function fetch_upstream_develop_if_not_exist() {
    UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle'
    origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` 
    if [ "$origin_upstream_url" == "" ]; then
        git remote add upstream $UPSTREAM_URL.git
    elif [ "$origin_upstream_url" != "$UPSTREAM_URL" ] \
            && [ "$origin_upstream_url" != "$UPSTREAM_URL.git" ]; then
        git remote remove upstream
        git remote add upstream $UPSTREAM_URL.git
    fi
    
    if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then 
        git fetch upstream # develop is not fetched
    fi
638
}
639 640 641

function generate_upstream_develop_api_spec() {
    fetch_upstream_develop_if_not_exist
642
    cur_branch=`git branch | grep \* | cut -d ' ' -f2`
643
    git checkout .
644 645 646
    git checkout -b develop_base_pr upstream/$BRANCH
    cmake_gen $1
    build $2
647
    cp ${PADDLE_ROOT}/python/requirements.txt /tmp
648

649
    git checkout $cur_branch
650
    generate_api_spec "$1" "DEV"
651
    git branch -D develop_base_pr
652 653
    ENABLE_MAKE_CLEAN="ON"
    rm -rf ${PADDLE_ROOT}/build/Makefile ${PADDLE_ROOT}/build/CMakeCache.txt
654 655 656
}

function generate_api_spec() {
T
tianshuo78520a 已提交
657
    set -e
658 659 660 661 662 663
    spec_kind=$2
    if [ "$spec_kind" != "PR" ] && [ "$spec_kind" != "DEV" ]; then
        echo "Not supported $2"
        exit 1
    fi

Y
yuyang18 已提交
664 665
    mkdir -p ${PADDLE_ROOT}/build/.check_api_workspace
    cd ${PADDLE_ROOT}/build/.check_api_workspace
666 667
    virtualenv .${spec_kind}_env
    source .${spec_kind}_env/bin/activate
668 669 670 671 672 673

    if [ "$spec_kind" == "DEV" ]; then
        pip install -r /tmp/requirements.txt
    else
        pip install -r ${PADDLE_ROOT}/python/requirements.txt
    fi
674
    pip --no-cache-dir install ${PADDLE_ROOT}/build/python/dist/*whl
675
    spec_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.spec
676
    python ${PADDLE_ROOT}/tools/print_signatures.py paddle > $spec_path
677

678 679 680
    # used to log op_register data_type
    op_type_path=${PADDLE_ROOT}/paddle/fluid/OP_TYPE_${spec_kind}.spec
    python ${PADDLE_ROOT}/tools/check_op_register_type.py > $op_type_path
681 682 683 684 685

    # print all ops desc in dict to op_desc_path
    op_desc_path=${PADDLE_ROOT}/paddle/fluid/OP_DESC_${spec_kind}.spec
    python ${PADDLE_ROOT}/tools/print_op_desc.py > $op_desc_path

686 687 688 689
    # print api and the md5 of source code of the api.
    api_source_md5_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.source.md5
    python ${PADDLE_ROOT}/tools/count_api_without_core_ops.py -p paddle > $api_source_md5_path

690 691
    awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc
    awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api
G
guofei 已提交
692
    if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ] || [ "$1" == "cp38-cp38" ]; then
693
        # Use sed to make python2 and python3 sepc keeps the same
694 695 696
        sed -i 's/arg0: str/arg0: unicode/g' $spec_path
        sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path
    fi   
697 698 699
    
    python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \
        ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec
S
sneaxiy 已提交
700

Y
yuyang18 已提交
701
    deactivate
702 703
}

704
function check_approvals_of_unittest() {
705 706 707
    if [ "$GITHUB_API_TOKEN" == "" ] || [ "$GIT_PR_ID" == "" ]; then
        return 0
    fi
708 709
    # approval_user_list: XiaoguangHu01 46782768,luotao1 6836917,phlrain 43953930,lanxianghit 47554610, zhouwei25 52485244, kolinwei 22165420
    check_times=$1
710 711
    if [ $check_times == 1 ]; then
        approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
712
        if [ "${approval_line}" != "" ]; then
713
            APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244`
714 715 716 717 718 719 720 721
            set +x
            echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
            if [ "${APPROVALS}" == "TRUE" ]; then
                echo "==================================="
                echo -e "\n current pr ${GIT_PR_ID} has got approvals. So, Pass CI directly!\n"
                echo "==================================="
                exit 0
            fi
722
        fi
723
    elif [ $check_times == 2 ]; then
724 725
        unittest_spec_diff=`python ${PADDLE_ROOT}/tools/diff_unittest.py ${PADDLE_ROOT}/paddle/fluid/UNITTEST_DEV.spec ${PADDLE_ROOT}/paddle/fluid/UNITTEST_PR.spec`
        if [ "$unittest_spec_diff" != "" ]; then
726
            approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
727
            APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244`
728
            set +x
729 730 731 732 733
            echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
            if [ "${APPROVALS}" == "FALSE" ]; then
                echo "************************************"
                echo -e "It is forbidden to disable or delete the unit-test.\n"
                echo -e "If you must delete it temporarily, please add it to[https://github.com/PaddlePaddle/Paddle/wiki/Temporarily-disabled-Unit-Test]."
734
                echo -e "Then you must have one RD (kolinwei(recommended) or zhouwei25) approval for the deletion of unit-test. \n"
735 736 737 738 739
                echo -e "If you have any problems about deleting unit-test, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/Deleting-unit-test-is-forbidden]. \n"
                echo -e "Following unit-tests are deleted in this PR: \n ${unittest_spec_diff} \n"
                echo "************************************"
                exit 1
            fi
740 741
        fi
    fi
742
    set -x
743 744 745 746 747 748 749 750 751
}

function check_change_of_unittest() {
    generate_unittest_spec "PR"
    fetch_upstream_develop_if_not_exist
    git reset --hard upstream/$BRANCH
    cmake_gen $1
    generate_unittest_spec "DEV"
    check_approvals_of_unittest 2
752 753
}

754 755 756 757
function check_sequence_op_unittest(){
    /bin/bash ${PADDLE_ROOT}/tools/check_sequence_op.sh
}

758 759 760 761
function generate_unittest_spec() {
    spec_kind=$1
    if [ "$spec_kind" == "DEV" ]; then
        cat <<EOF
762 763 764
        ============================================
        Generate unit tests.spec of develop.
        ============================================
765 766 767
EOF
    elif [ "$spec_kind" == "PR" ]; then
        cat <<EOF
768 769 770
        ============================================
        Generate unit tests.spec of this PR.
        ============================================
771 772 773 774 775
EOF
    else
        echo "Not supported $1"
        exit 1
    fi
776
    spec_path=${PADDLE_ROOT}/paddle/fluid/UNITTEST_${spec_kind}.spec
777 778 779
    ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' > ${spec_path}
}

780

781
function assert_api_spec_approvals() {
782 783 784
    /bin/bash ${PADDLE_ROOT}/tools/check_api_approvals.sh;approval_error=$?
    if [ "$approval_error" != 0 ];then
       exit 6
785
    fi
Y
yuyang18 已提交
786 787
}

Z
zhangchunle 已提交
788 789 790 791 792 793 794
function assert_file_diff_approvals() {
    /bin/bash ${PADDLE_ROOT}/tools/check_file_diff_approvals.sh;file_approval_error=$?
    if [ "$file_approval_error" != 0 ];then
       exit 6
    fi
}

Y
yuyang18 已提交
795

796 797 798 799 800
function check_coverage() {
    /bin/bash ${PADDLE_ROOT}/tools/coverage/paddle_coverage.sh
}


801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
function single_test() {
    TEST_NAME=$1
    if [ -z "${TEST_NAME}" ]; then
        echo -e "${RED}Usage:${NONE}"
        echo -e "${BOLD}${SCRIPT_NAME}${NONE} ${BLUE}single_test${NONE} [test_name]"
        exit 1
    fi
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running ${TEST_NAME} ...
    ========================================
EOF
        ctest --output-on-failure -R ${TEST_NAME}
    fi
}

820 821 822
function bind_test() {
    # the number of process to run tests
    NUM_PROC=6
Q
Qiyang Min 已提交
823

824 825 826
    # calculate and set the memory usage for each process
    MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
    export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
Q
Qiyang Min 已提交
827

828 829
    # get the CUDA device count
    CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
Q
Qiyang Min 已提交
830

831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
    for (( i = 0; i < $NUM_PROC; i++ )); do
        cuda_list=()
        for (( j = 0; j < $CUDA_DEVICE_COUNT; j++ )); do
            s=$[i+j]
            n=$[s%CUDA_DEVICE_COUNT]
            if [ $j -eq 0 ]; then
                cuda_list=("$n")
            else
                cuda_list="$cuda_list,$n"
            fi
        done
        echo $cuda_list
        # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
        # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
        env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC --output-on-failure &
    done
    wait
}

850 851 852 853 854 855 856 857 858 859 860
EXIT_CODE=0;
function caught_error() {
 for job in `jobs -p`; do
        # echo "PID => ${job}"
        if ! wait ${job} ; then
            echo "At least one test failed with exit code => $?" ;
            EXIT_CODE=1;
        fi
    done
}

861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
function case_count(){
    cat <<EOF
    ============================================
    Generating TestCases Count ... 
    ============================================
EOF
    testcases=$1
    num=$(echo $testcases|grep -o '\^'|wc -l)
    if [ "$2" == "" ]; then
        echo "exclusive TestCases count is $num"
    else
        echo "$2 card TestCases count is $num"
    fi
}

876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
failed_test_lists=''
tmp_dir=`mktemp -d`

function collect_failed_tests() {
    for file in `ls $tmp_dir`; do
        exit_code=0
        grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$?
        if [ $exit_code -ne 0 ]; then
            failuretest=''
        else
            failuretest=`grep -A 10000 'The following tests FAILED:' $tmp_dir/$file | sed 's/The following tests FAILED://g'|sed '/^$/d'`
            failed_test_lists="${failed_test_lists}
            ${failuretest}"
        fi
    done
}

893 894
function card_test() {
    set -m
895 896
    case_count $1 $2
    ut_startTime_s=`date +%s` 
897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
    # get the CUDA device count
    CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)

    testcases=$1
    if (( $# > 1 )); then
        cardnumber=$2
        if (( $cardnumber > $CUDA_DEVICE_COUNT )); then
            cardnumber=$CUDA_DEVICE_COUNT
        fi
    else
        cardnumber=$CUDA_DEVICE_COUNT
    fi

    if [[ "$testcases" == "" ]]; then
        return 0
    fi

    trap 'caught_error' CHLD
915
    tmpfile_rand=`date +%s%N`
916 917 918 919 920 921 922 923 924 925 926 927
    NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber]
    for (( i = 0; i < $NUM_PROC; i++ )); do
        # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
        # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
        cuda_list=()
        for (( j = 0; j < cardnumber; j++ )); do
            if [ $j -eq 0 ]; then
                    cuda_list=("$[i*cardnumber]")
                else
                    cuda_list="$cuda_list,$[i*cardnumber+j]"
            fi
        done
928
        tmpfile=$tmp_dir/$tmpfile_rand"_"$i
929 930
        if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
            if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
931 932 933
                (ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
            else  
                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
934 935 936
            fi
        else
            if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
937
                (ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
938
            else
939
                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
940 941 942 943
            fi
        fi
    done
    wait; # wait for all subshells to finish
944 945 946 947 948 949
    ut_endTime_s=`date +%s`
    if [ "$2" == "" ]; then
        echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
    else
        echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
    fi
950
    set +m
951 952
}

953
function parallel_test_base_gpu() {
954 955 956
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
957
    Running unit tests in parallel way ...
958 959 960 961
    ========================================
EOF

set +x
C
chalsliu 已提交
962 963 964 965
        precison_cases=""
        if [ ${PRECISION_TEST:-OFF} == "ON" ]; then
            precision_cases=`python $PADDLE_ROOT/tools/get_pr_ut.py`
        fi
966
        EXIT_CODE=0;
967 968 969 970 971 972
        test_cases=$(ctest -N -V) # get all test cases
        exclusive_tests=''        # cases list which would be run exclusively
        single_card_tests=''      # cases list which would take one graph card
        multiple_card_tests=''    # cases list which would take multiple GPUs, most cases would be two GPUs
        is_exclusive=''           # indicate whether the case is exclusive type
        is_multicard=''           # indicate whether the case is multiple GPUs type
973
        is_nightly=''             # indicate whether the case will only run at night
974 975 976 977 978 979 980 981
        while read -r line; do
            if [[ "$line" == "" ]]; then
                continue
            fi
                read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#')
                if [[ "$matchstr" == "" ]]; then
                    # Any test case with LABELS property would be parse here
                    # RUN_TYPE=EXCLUSIVE mean the case would run exclusively
982
                    # RUN_TYPE=DIST mean the case would take two graph GPUs during runtime
983
                    # RUN_TYPE=NIGHTLY or RUN_TYPE=DIST:NIGHTLY or RUN_TYPE=EXCLUSIVE:NIGHTLY means the case will ONLY run at night
984 985
                    read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE")
                    read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST")
986
                    read is_nightly <<< $(echo "$line"|grep -oEi "RUN_TYPE=NIGHTLY|RUN_TYPE=DIST:NIGHTLY|RUN_TYPE=EXCLUSIVE:NIGHTLY")
987 988 989 990
                    continue
                fi
                read testcase <<< $(echo "$line"|grep -oEi "\w+$")

991 992 993 994
                if [[ "$is_nightly" != "" ]] && [ ${NIGHTLY_MODE:-OFF} == "OFF" ]; then
                    echo $testcase" will only run at night."
                    continue
                fi
C
chalsliu 已提交
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
                if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]]; then
                    will_test="false"
                    for case in $precision_cases; do
                        if [[ $testcase == $case ]]; then
                            will_test="true"
                            break
                        fi
                    done
                    if [[ $will_test == "false" ]]; then
                        echo $testcase" won't run in PRECISION_TEST mode."
                        continue
                    fi
                fi
1008

1009
                if [[ "$is_multicard" == "" ]]; then
1010
                  # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs
1011
                  read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_")
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
                fi

                if [[ "$is_exclusive" != "" ]]; then
                    if [[ "$exclusive_tests" == "" ]]; then
                        exclusive_tests="^$testcase$"
                    else
                        exclusive_tests="$exclusive_tests|^$testcase$"
                    fi
                elif [[ "$is_multicard" != "" ]]; then
                    if [[ "$multiple_card_tests" == "" ]]; then
                        multiple_card_tests="^$testcase$"
                    else
                        multiple_card_tests="$multiple_card_tests|^$testcase$"
                    fi
                else
1027
                    if [[ "${#single_card_tests}" -gt 10000 ]];then
1028 1029 1030 1031 1032 1033 1034 1035
                        if [[ "$single_card_tests_1" == "" ]]; then 
                            single_card_tests_1="^$testcase$"
                        else
                            single_card_tests_1="$single_card_tests_1|^$testcase$"
                        fi
                        continue
                    fi

1036 1037 1038 1039 1040 1041 1042 1043
                    if [[ "$single_card_tests" == "" ]]; then
                        single_card_tests="^$testcase$"
                    else
                        single_card_tests="$single_card_tests|^$testcase$"
                    fi
                fi
                is_exclusive=''
                is_multicard=''
1044
                is_nightly=''
1045 1046 1047 1048
                matchstr=''
                testcase=''
        done <<< "$test_cases";

1049
        card_test "$single_card_tests" 1    # run cases with single GPU
1050
        card_test "$single_card_tests_1" 1    # run cases with single GPU
1051 1052
        card_test "$multiple_card_tests" 2  # run cases with two GPUs
        card_test "$exclusive_tests"        # run cases exclusively, in this cases would be run with 4/8 GPUs
1053 1054
        collect_failed_tests
        rm -f $tmp_dir/*
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
        exec_times=0
        retry_unittests_record=''
        retry_time=3
        exec_time_array=('first' 'second' 'third')
        if [ -n "$failed_test_lists" ];then
            while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
                do
                    
                    retry_unittests_record="$retry_unittests_record$failed_test_lists"
                    failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
1065
                    read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' )
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
                    echo "========================================="
                    echo "This is the ${exec_time_array[$exec_times]} time to re-run"
                    echo "========================================="
                    echo "The following unittest will be re-run:"
                    echo "${failed_test_lists_ult}"
                        
                    for line in ${retry_unittests[@]} ;
                        do

                            one_card_tests=$single_card_tests'|'$single_card_tests_1

                            read tmp_one_tmp <<< "$( echo $one_card_tests | grep -oEi $line )"
                            read tmp_mul_tmp <<< "$( echo $multiple_card_tests | grep -oEi $line )"
                            read exclusive_tmp <<< "$( echo $exclusive_tests | grep -oEi $line )"

                            if [[ "$tmp_one_tmp" != ""  ]]; then
                                if [[ "$one_card_retry" == "" ]]; then
                                    one_card_retry="^$line$"
                                else
                                    one_card_retry="$one_card_retry|^$line$"
                                fi
                            elif [[ "$tmp_mul_tmp" != "" ]]; then
                                if [[ "$multiple_card_retry" == "" ]]; then
                                    multiple_card_retry="^$line$"
                                else
                                    multiple_card_retry="$multiple_card_retry|^$line$"
                                fi
                            else
                                if [[ "$exclusive_retry" == "" ]];then
                                    exclusive_retry="^$line$"
                                else
                                    exclusive_retry="$exclusive_retry|^$line$"
                                fi
                            fi

                        done

                    if [[ "$one_card_retry" != "" ]]; then
                        card_test "$one_card_retry" 1
                    fi

                    if [[ "$multiple_card_retry" != "" ]]; then
                        card_test "$multiple_card_retry" 2
                    fi

                    if [[ "$exclusive_retry" != "" ]]; then
                        card_test "$exclusive_retry"
                    fi
                    
                    exec_times=$[$exec_times+1]
                    failed_test_lists=''
                    collect_failed_tests
                    rm -f $tmp_dir/*
                    one_card_retry=''
                    multiple_card_retry=''
                    exclusive_retry=''
                    retry_unittests=''
                done
        fi

1126
        if [[ "$EXIT_CODE" != "0" ]]; then
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
            if [[ "$failed_test_lists" == "" ]]; then
                echo "========================================"
                echo "There are failed tests, which have been successful after re-run:"
                echo "========================================"
                echo "The following tests have been re-ran:"
                echo "${retry_unittests_record}"
            else
                failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
                echo "========================================"
                echo "Summary Failed Tests... "
                echo "========================================"
                echo "The following tests FAILED: "
                echo "${failed_test_lists_ult}"
                exit 8;
            fi
1142 1143 1144 1145 1146
        fi
set -ex
    fi
}

1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
function parallel_test_base_cpu() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit cpu tests ...
    ========================================
EOF
        ut_startTime_s=`date +%s`
        ctest --output-on-failure -j $1
        ut_endTime_s=`date +%s`
        echo "CPU testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
        if [[ "$EXIT_CODE" != "0" ]]; then
            exit 8;
        fi
    fi
}

T
tianshuo78520a 已提交
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
function parallel_test_base_xpu() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit cpu tests ...
    ========================================
EOF
        ut_startTime_s=`date +%s`
        ctest -R *xpu 
        ut_endTime_s=`date +%s`
        echo "XPU testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
        if [[ "$EXIT_CODE" != "0" ]]; then
            exit 8;
        fi
    fi
    
}

1186
function parallel_test() {
1187
    ut_total_startTime_s=`date +%s`
1188 1189
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
1190
    pip install ${PADDLE_ROOT}/build/python/dist/*whl
1191 1192 1193
    if [ "$WITH_GPU" == "ON" ];then
        parallel_test_base_gpu
    else
T
tianshuo78520a 已提交
1194 1195 1196 1197 1198
        if [ "$WITH_XPU" == "ON" ];then
            parallel_test_base_xpu
        else
            parallel_test_base_cpu ${PROC_RUN:-1}
        fi
1199
    fi
1200 1201
    ut_total_endTime_s=`date +%s`
    echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s"
1202 1203
}

1204 1205 1206 1207 1208 1209 1210
function enable_unused_var_check() {
    # NOTE(zhiqiu): Set FLAGS_enable_unused_var_check=1 here to enable unused_var_check,
    # which checks if an operator has unused input variable(s).
    # Currently, use it in coverage CI job.
    export FLAGS_enable_unused_var_check=1
}

1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
function gen_doc_lib() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ========================================
    Building documentation library ...
    In /paddle/build
    ========================================
EOF
    cmake .. \
        -DCMAKE_BUILD_TYPE=Release \
        -DWITH_GPU=OFF \
        -DWITH_MKL=OFF \

    local LIB_TYPE=$1
    case $LIB_TYPE in
      full)
        # Build full Paddle Python module. Will timeout without caching 'copy_paddle_pybind' first
1229
        make -j `nproc` framework_py_proto copy_paddle_pybind paddle_python
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
        ;;
      pybind)
        # Build paddle pybind library. Takes 49 minutes to build. Might timeout
        make -j `nproc` copy_paddle_pybind
        ;;
      proto)
        # Even smaller library.
        make -j `nproc` framework_py_proto
        ;;
      *)
        exit 0
        ;;
      esac
1243 1244 1245 1246 1247
}


function gen_dockerfile() {
    # Set BASE_IMAGE according to env variables
1248 1249
    CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1).$(echo $CUDA_VERSION | cut -d '.' -f 2)"
    CUDNN_MAJOR=$(echo $CUDNN_VERSION | cut -d '.' -f 1)
1250
    if [[ ${WITH_GPU} == "ON" ]]; then
1251
        BASE_IMAGE="nvidia/cuda:${CUDA_MAJOR}-cudnn${CUDNN_MAJOR}-devel-ubuntu16.04"
1252
    else
1253
        BASE_IMAGE="ubuntu:16.04"
1254 1255 1256 1257
    fi

    DOCKERFILE_GPU_ENV=""
    DOCKERFILE_CUDNN_DSO=""
1258
    DOCKERFILE_CUBLAS_DSO=""
1259 1260
    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:\${LD_LIBRARY_PATH}"
1261
        DOCKERFILE_CUDNN_DSO="RUN ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.${CUDNN_MAJOR} /usr/lib/x86_64-linux-gnu/libcudnn.so"
1262
        DOCKERFILE_CUBLAS_DSO="RUN ln -sf /usr/local/cuda/targets/x86_64-linux/lib/libcublas.so.${CUDA_MAJOR} /usr/lib/x86_64-linux-gnu/libcublas.so"
1263 1264 1265 1266
    fi

    cat <<EOF
    ========================================
1267
    Generate ${PADDLE_ROOT}/build/Dockerfile ...
1268 1269
    ========================================
EOF
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
    
    ref_CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1)"
    if [[ ${WITH_GPU} == "ON"  ]]; then
        ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR}
    else
        ref_gpu=cpu
    fi
    if [[ ${WITH_GPU} == "ON"  ]]; then
        install_gpu="_gpu"
    else
        install_gpu=""
    fi
    if [[ ${WITH_MKL} == "ON" ]]; then
        ref_mkl=mkl
    else
1285
        ref_mkl=openblas
1286
    fi
1287

1288 1289 1290 1291 1292 1293
    ref_web=https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}

    ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl
    ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
    ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
    ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl
G
guofei 已提交
1294
    ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl
1295 1296 1297 1298 1299

    ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl
    ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
    ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
    ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl
G
guofei 已提交
1300
    ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl
1301

1302
    if [[ ${PADDLE_BRANCH} != "0.0.0" && ${WITH_MKL} == "ON" && ${WITH_GPU} == "ON" ]]; then
1303 1304 1305 1306
        ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
        ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
        ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
G
guofei 已提交
1307
        ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl
1308 1309 1310 1311
        ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
        ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
G
guofei 已提交
1312
        ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
    fi

    #ref_paddle2_mv1=""
    #ref_paddle2_mv2=""
    ref_paddle35_mv1=""
    ref_paddle35_mv2=""
    ref_paddle36_mv1=""
    ref_paddle36_mv2=""
    #ref_paddle37_mv1=""
    #ref_paddle37_mv2=""
1323
    if [[ ${PADDLE_BRANCH} == "0.0.0" && ${WITH_GPU} == "ON" ]]; then
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
        #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle_gpu-1.5.1-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle_gpu-1.5.1-cp36-cp36m-linux_x86_64.whl
        #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl
        #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&"
        #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2"
        ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&"
        ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}"
        ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&"
        ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}"
        #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&"
        #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37"
    fi
1337
    if [[ ${PADDLE_BRANCH} == "0.0.0" && ${WITH_GPU} != "ON" ]]; then
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
        #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle-1.5.1-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle-1.5.1-cp36-cp36m-linux_x86_64.whl
        #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl
        #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&"
        #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2"
        ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&"
        ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}"
        ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&"
        ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}"
        #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&"
        #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37"
    fi
    
1352 1353 1354 1355 1356 1357 1358
    cat > ${PADDLE_ROOT}/build/Dockerfile <<EOF
    FROM ${BASE_IMAGE}
    MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
    ENV HOME /root
EOF

    if [[ ${WITH_GPU} == "ON"  ]]; then
1359
        NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true"
1360
    else
1361
        NCCL_DEPS="true"
1362 1363
    fi

1364 1365 1366 1367
    if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then 
        NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0"
    fi

1368 1369
    PADDLE_VERSION="paddle version"
    CMD='"paddle", "version"'
1370 1371
    
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1372 1373 1374
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python3 python3-pip libgtk2.0-dev dmidecode python3-tk && \
1375
        pip3 install opencv-python py-cpuinfo==5.0.0 && wget ${ref_web}/${ref_paddle35} && ${ref_paddle35_mv1} pip3 install ${ref_paddle35_whl} ${ref_paddle35_mv2}; apt-get install -f -y && \
1376
        apt-get clean -y && \
1377
        rm -f ${ref_paddle35} && \
1378 1379 1380 1381
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_CUBLAS_DSO}
    ${DOCKERFILE_GPU_ENV}
1382
EOF
1383
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1384 1385 1386 1387 1388 1389 1390
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \
        tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \
1391
        ./configure -prefix=/usr/local && make install -j8 && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \
1392 1393 1394
        wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \
        tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
1395 1396 1397
        make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.6.0.tgz
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
        pip3.6 install opencv-python && wget ${ref_web}/${ref_paddle36} && ${ref_paddle36_mv1} pip3.6 install ${ref_paddle36_whl} ${ref_paddle36_mv2}; apt-get install -f -y && \
1398
        apt-get clean -y && \
1399
        rm -f ${ref_paddle36} && \
1400 1401
        ldconfig
EOF
1402
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1403 1404 1405 1406 1407 1408 1409 1410
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \
        tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
1411 1412 1413
        make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.7.0.tgz
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
        pip3.7 install opencv-python && wget ${ref_web}/${ref_paddle37} && pip3.7 install ${ref_paddle37_whl}; apt-get install -f -y && \
1414
        apt-get clean -y && \
1415
        rm -f ${ref_paddle37} && \
1416
        ldconfig
G
guofei 已提交
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
EOF
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \
        tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
        make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.8.0.tgz
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
        pip3.8 install opencv-python && wget ${ref_web}/${ref_paddle38} && pip3.8 install ${ref_paddle38_whl}; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f ${ref_paddle38} && \
        ldconfig
1433
EOF
1434
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1435
    # run paddle version to install python packages first
1436 1437
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python-pip python-opencv libgtk2.0-dev dmidecode python-tk && easy_install -U pip && \
1438
        wget ${ref_web}/${ref_paddle2} && pip install ${ref_paddle2_whl}; apt-get install -f -y && \
1439
        apt-get clean -y && \
1440
        rm -f ${ref_paddle2} && \
1441 1442
        ${PADDLE_VERSION} && \
        ldconfig
1443
EOF
1444

1445
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1446 1447 1448 1449 1450
    # default command shows the paddle version and exit
    CMD [${CMD}]
EOF
}

1451
function gen_fluid_lib() {
L
Lei Wang 已提交
1452 1453
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
1454
    cat <<EOF
1455
    ========================================
1456
    Generating fluid library for train and inference ...
1457 1458
    ========================================
EOF
1459 1460 1461 1462
    parallel_number=`nproc`
    if [[ "$1" != "" ]]; then
      parallel_number=$1
    fi
1463 1464 1465 1466 1467 1468 1469 1470
    startTime_s=`date +%s`
    set +e
    cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto};build_error=$?
    make -j ${parallel_number} fluid_lib_dist;build_error=$?
    make -j ${parallel_number} inference_lib_dist;build_error=$?
    if [ "$build_error" != 0 ];then
        exit 7;
    fi
1471 1472
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
1473
    build_size "paddle_inference"
T
Tao Luo 已提交
1474 1475
}

1476
function tar_fluid_lib() {
1477
    cat <<EOF
T
Tao Luo 已提交
1478
    ========================================
1479
    Taring fluid library for train and inference ...
T
Tao Luo 已提交
1480 1481
    ========================================
EOF
1482
    cd ${PADDLE_ROOT}/build
1483
    cp -r paddle_install_dir fluid
1484
    tar -czf fluid.tgz fluid
1485 1486
    cp -r paddle_inference_install_dir paddle_inference
    tar -czf paddle_inference.tgz paddle_inference
1487 1488
}

1489
function test_fluid_lib() {
1490
    cat <<EOF
1491
    ========================================
1492
    Testing fluid library for inference ...
1493 1494
    ========================================
EOF
1495
    fluid_startTime_s=`date +%s`
1496 1497 1498 1499
    cd ${PADDLE_ROOT}/paddle/fluid/inference/api/demo_ci
    ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON} ${WITH_GPU:-OFF} ${INFERENCE_DEMO_INSTALL_DIR} \
             ${TENSORRT_INCLUDE_DIR:-/usr/local/TensorRT/include} \
             ${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib}
1500
    EXIT_CODE=$?
1501 1502
    fluid_endTime_s=`date +%s`
    echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"          
1503
    ./clean.sh
1504 1505 1506
    if [[ "$EXIT_CODE" != "0" ]]; then
        exit 8;
    fi
1507 1508
}

1509 1510 1511 1512 1513 1514
function test_fluid_lib_train() {
    cat <<EOF
    ========================================
    Testing fluid library for training ...
    ========================================
EOF
1515
    fluid_train_startTime_s=`date +%s`
1516 1517
    cd ${PADDLE_ROOT}/paddle/fluid/train/demo
    ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON}
1518
    EXIT_CODE=$?
1519 1520
    fluid_train_endTime_s=`date +%s`
    echo "test_fluid_lib_train Total Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s"
1521
    ./clean.sh
1522 1523 1524
    if [[ "$EXIT_CODE" != "0" ]]; then
        exit 8;
    fi
1525
}
1526

T
tianshuo78520a 已提交
1527

1528 1529 1530 1531 1532
function build_document_preview() {
    sh /paddle/tools/document_preview.sh ${PORT}
}


1533
function example() {
1534
    pip install ${PADDLE_ROOT}/build/python/dist/*.whl
1535
    paddle version
1536
    cd ${PADDLE_ROOT}/tools
Z
zhangchunle 已提交
1537 1538
    python sampcd_processor.py cpu;example_error=$?
    if [ "$example_error" != "0" ];then
1539
      echo "Code instance execution failed" >&2
Z
zhangchunle 已提交
1540
      exit 5
1541 1542 1543
    fi
}

1544 1545 1546 1547
function summary_check_problems() {
    set +x
    local check_style_code=$1
    local example_code=$2
1548 1549
    local check_style_info=$3
    local example_info=$4
1550 1551 1552
    if [ $check_style_code -ne 0 -o $example_code -ne 0 ];then
      echo "========================================"
      echo "summary problems:"
1553 1554 1555 1556 1557 1558
      if [ $check_style_code -ne 0 -a $example_code -ne 0 ];then
        echo "There are 2 errors: Code format error and Example code error."
      else
        [ $check_style_code -ne 0 ] && echo "There is 1 error: Code format error."
        [ $example_code -ne 0 ] && echo "There is 1 error: Example code error."
      fi
1559 1560
      echo "========================================"
      if [ $check_style_code -ne 0 ];then
1561 1562
        echo "*****Code format error***** Please fix it according to the diff information:"
        echo "$check_style_info" | grep "code format error" -A $(echo "$check_style_info" | wc -l)
1563 1564
      fi
      if [ $example_code -ne 0 ];then
1565 1566
        echo "*****Example code error***** Please fix the error listed in the information:"
        echo "$example_info" | grep "API check -- Example Code" -A $(echo "$example_info" | wc -l)
1567 1568 1569 1570 1571 1572 1573
      fi
      [ $check_style_code -ne 0 ] && exit $check_style_code
      [ $example_code -ne 0 ] && exit $example_code
    fi
    set -x
}

1574
function main() {
1575
    local CMD=$1 
1576
    local parallel_number=$2
1577
    init
1578 1579
    if [ "$CMD" != "assert_file_approvals" ];then
      python ${PADDLE_ROOT}/tools/summary_env.py
1580
      bash ${PADDLE_ROOT}/tools/get_cpu_info.sh
1581
    fi
1582
    case $CMD in
1583
      build_only)
1584
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1585 1586
        ;;
      build_and_check)
1587
        set +e
1588
        check_style_info=$(check_style)
1589
        check_style_code=$?
1590
        generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
1591
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1592 1593
        check_sequence_op_unittest
        generate_api_spec ${PYTHON_ABI:-""} "PR"
1594 1595
        set +e
        example_info=$(example)
1596
        example_code=$?
1597
        summary_check_problems $check_style_code $example_code "$check_style_info" "$example_info"
1598
        assert_api_spec_approvals
1599
        ;;
1600 1601
      build)
        cmake_gen ${PYTHON_ABI:-""}
1602
        build ${parallel_number}
1603
        gen_dockerfile ${PYTHON_ABI:-""}
1604
        assert_api_spec_approvals
1605
        ;;
1606 1607
      combine_avx_noavx)
        combine_avx_noavx_build
1608
        gen_dockerfile ${PYTHON_ABI:-""}
1609 1610 1611
        ;;
      combine_avx_noavx_build_and_test)
        combine_avx_noavx_build
1612
        gen_dockerfile ${PYTHON_ABI:-""}
1613 1614
        parallel_test_base
        ;;
1615
      test)
1616
        parallel_test
1617
        ;;
1618 1619 1620
      single_test)
        single_test $2
        ;;
1621 1622 1623
      bind_test)
        bind_test
        ;;
1624 1625 1626
      gen_doc_lib)
        gen_doc_lib $2
        ;;
1627
      dockerfile)
1628
        gen_dockerfile ${PYTHON_ABI:-""}
1629 1630 1631
        ;;
      fluid_inference_lib)
        cmake_gen ${PYTHON_ABI:-""}
1632
        gen_fluid_lib ${parallel_number}
1633 1634
        tar_fluid_lib
        test_fluid_lib
1635 1636 1637 1638
        ;;
      check_style)
        check_style
        ;;
L
Lei Wang 已提交
1639
      cicheck)
1640 1641 1642 1643 1644 1645
        cmake_gen ${PYTHON_ABI:-""}
        build ${parallel_number}
        enable_unused_var_check
        parallel_test
        ;;
      cicheck_coverage)
1646
        check_approvals_of_unittest 1
1647
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1648
        enable_unused_var_check
1649
        parallel_test
1650
        check_coverage
1651
        check_change_of_unittest ${PYTHON_ABI:-""}
L
Lei Wang 已提交
1652
        ;;
G
gongweibao 已提交
1653 1654
      cicheck_brpc)
        cmake_gen ${PYTHON_ABI:-""}
1655
        build ${parallel_number}
G
gongweibao 已提交
1656 1657
        run_brpc_test
        ;;
1658
      assert_api)
1659
        generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
Y
Yu Yang 已提交
1660
        assert_api_spec_approvals
1661 1662
        ;;
      test_inference)
1663
        gen_fluid_lib ${parallel_number}
1664
        test_fluid_lib
1665
        #test_fluid_lib_train
1666 1667 1668 1669
        ;;
      test_train)
        gen_fluid_lib ${parallel_number}
        test_fluid_lib_train
1670 1671 1672 1673
        ;;
      assert_api_approvals)
        assert_api_spec_approvals
        ;;
Z
zhangchunle 已提交
1674 1675 1676
      assert_file_approvals)
        assert_file_diff_approvals
        ;; 
1677
      maccheck)
1678
        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
1679
        run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
1680
        ;;
1681
      maccheck_py35)
1682
        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
1683
        run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
1684
        check_change_of_unittest ${PYTHON_ABI:-""}
1685
        ;;
J
JiabinYang 已提交
1686 1687 1688 1689
      macbuild)
        cmake_gen ${PYTHON_ABI:-""}
        build_mac
        ;;
M
minqiyang 已提交
1690
      cicheck_py35)
1691
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1692
        parallel_test
M
minqiyang 已提交
1693
        ;;
T
tianshuo78520a 已提交
1694 1695 1696
      check_xpu)
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
        parallel_test
T
tianshuo78520a 已提交
1697
        ;;
Y
Yu Yang 已提交
1698 1699 1700 1701
      cmake_gen)
        cmake_gen ${PYTHON_ABI:-""}
        ;;
      gen_fluid_lib)
1702
        gen_fluid_lib ${parallel_number}
Y
Yu Yang 已提交
1703 1704 1705 1706
        ;;
      test_fluid_lib)
        test_fluid_lib
        ;;
1707 1708 1709 1710 1711
      document)
        cmake_gen ${PYTHON_ABI:-""}
        build ${parallel_number}
        build_document_preview
        ;;
1712 1713 1714
      api_example)
        example
        ;;
1715 1716
      *)
        print_usage
1717
        exit 1
1718 1719
        ;;
      esac
1720
      echo "paddle_build script finished as expected"
1721 1722 1723
}

main $@
新手
引导
客服 返回
顶部