paddle_build.sh 53.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#!/usr/bin/env bash

# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#=================================================
#                   Utils
#=================================================

21 22
set -ex

Z
Zeng Jinle 已提交
23 24 25 26
if [ -z ${BRANCH} ]; then
    BRANCH="develop"
fi

27 28
function print_usage() {
    echo -e "\n${RED}Usage${NONE}:
29
    ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]"
Q
Qiyang Min 已提交
30

31 32 33
    echo -e "\n${RED}Options${NONE}:
    ${BLUE}build${NONE}: run build for x86 platform
    ${BLUE}test${NONE}: run all unit tests
34
    ${BLUE}single_test${NONE}: run a single unit test
35 36
    ${BLUE}bind_test${NONE}: parallel tests bind to different GPU
    ${BLUE}doc${NONE}: generate paddle documents
37
    ${BLUE}gen_doc_lib${NONE}: generate paddle documents library
38 39 40 41
    ${BLUE}html${NONE}: convert C++ source code into HTML
    ${BLUE}dockerfile${NONE}: generate paddle release dockerfile
    ${BLUE}fluid_inference_lib${NONE}: deploy fluid inference library
    ${BLUE}check_style${NONE}: run code style check
42 43
    ${BLUE}cicheck${NONE}: run CI tasks on Linux
    ${BLUE}maccheck${NONE}: run CI tasks on Mac
44 45 46 47
    "
}

function init() {
48 49 50 51 52
    RED='\033[0;31m'
    BLUE='\033[0;34m'
    BOLD='\033[1m'
    NONE='\033[0m'

53
    PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
54 55 56
    if [ -z "${SCRIPT_NAME}" ]; then
        SCRIPT_NAME=$0
    fi
Z
Zeng Jinle 已提交
57 58

    ENABLE_MAKE_CLEAN=${ENABLE_MAKE_CLEAN:-ON}
59 60
}

61
function cmake_base() {
R
randytli 已提交
62
    # Build script will not fail if *.deb does not exist
63
    rm *.deb 2>/dev/null || true
R
randytli 已提交
64
    # Delete previous built whl packages
65 66 67 68 69
    rm -rf python/dist 2>/dev/null || true

    # Support build for all python versions, currently
    # including cp27-cp27m and cp27-cp27mu.
    PYTHON_FLAGS=""
70
    SYSTEM=`uname -s`
J
Jiabin Yang 已提交
71
    if [ "$SYSTEM" == "Darwin" ]; then
V
velconia 已提交
72
        echo "Using python abi: $1"
J
Jiabin Yang 已提交
73
        if [[ "$1" == "cp27-cp27m" ]] || [[ "$1" == "" ]]; then
74 75 76
            if [ -d "/Library/Frameworks/Python.framework/Versions/2.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
J
Jiabin Yang 已提交
77
                export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:${PATH}
78 79
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7
J
Jiabin Yang 已提交
80
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/lib/libpython2.7.dylib"
81
                pip install --user -r ${PADDLE_ROOT}/python/requirements.txt
82 83 84
            else
                exit 1
            fi
V
velconia 已提交
85
        elif [ "$1" == "cp35-cp35m" ]; then
V
velconia 已提交
86
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.5" ]; then
V
velconia 已提交
87 88 89 90 91 92
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib"
93
                pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt
V
velconia 已提交
94 95 96
            else
                exit 1
            fi
M
minqiyang 已提交
97 98 99 100 101 102 103 104
        elif [ "$1" == "cp36-cp36m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.6" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib"
105
                pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
106 107 108 109 110 111 112 113 114 115 116
            else
                exit 1
            fi
        elif [ "$1" == "cp37-cp37m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib"
117
                pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
118 119 120
            else
                exit 1
            fi
121
        fi
M
minqiyang 已提交
122
    else
123
        if [ "$1" != "" ]; then
M
minqiyang 已提交
124
            echo "using python abi: $1"
125 126 127 128 129 130
            if [ "$1" == "cp27-cp27m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:}
                export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so"
131
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
132 133 134 135 136 137
            elif [ "$1" == "cp27-cp27mu" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:}
                export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so"
138 139 140 141 142 143 144 145 146 147 148 149 150 151
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
            elif [ "$1" == "cp27-cp27m-gcc82" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs4/lib:}
                export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs2/lib/libpython2.7.so"
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
            elif [ "$1" == "cp27-cp27mu-gcc82" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.15-ucs2/lib:}
                export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.15-ucs4/lib/libpython2.7.so"
152
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
153 154 155 156
            elif [ "$1" == "cp35-cp35m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3
M
minqiyang 已提交
157 158
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so"
159
                pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt
160 161 162 163 164 165
            elif [ "$1" == "cp36-cp36m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so"
166
                pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt
167 168 169
            elif [ "$1" == "cp37-cp37m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH}
170
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7
171 172
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so"
173
                pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
174
           fi
175 176
        else
            pip install -r ${PADDLE_ROOT}/python/requirements.txt
177 178
        fi
    fi
M
minqiyang 已提交
179

J
JiabinYang 已提交
180
    if [ "$SYSTEM" == "Darwin" ]; then
181 182
        WITH_DISTRIBUTE=${WITH_DISTRIBUTE:-ON}
        WITH_AVX=${WITH_AVX:-ON}
J
Jiabin Yang 已提交
183
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-~/.cache/inference_demo}
J
JiabinYang 已提交
184
    else
J
Jiabin Yang 已提交
185
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-/root/.cache/inference_demo}
186
    fi
M
minqiyang 已提交
187

G
gongweibao 已提交
188 189 190
    distibuted_flag=${WITH_DISTRIBUTE:-OFF}
    grpc_flag=${WITH_GRPC:-${distibuted_flag}}

191 192 193 194 195 196 197
    cat <<EOF
    ========================================
    Configuring cmake in /paddle/build ...
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
        ${PYTHON_FLAGS}
        -DWITH_GPU=${WITH_GPU:-OFF}
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF}
G
gongweibao 已提交
198
        -DWITH_DISTRIBUTE=${distibuted_flag}
199 200 201 202 203 204
        -DWITH_MKL=${WITH_MKL:-ON}
        -DWITH_AVX=${WITH_AVX:-OFF}
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All}
        -DWITH_PYTHON=${WITH_PYTHON:-ON}
        -DCUDNN_ROOT=/usr/
        -DWITH_TESTING=${WITH_TESTING:-ON}
205
        -DWITH_COVERAGE=${WITH_COVERAGE:-OFF}
206 207 208 209 210
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON}
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON}
211
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR}
212
        -DPY_VERSION=${PY_VERSION:-2.7}
W
Wu Yi 已提交
213
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build}
G
gongweibao 已提交
214
        -DWITH_GRPC=${grpc_flag}
石晓伟 已提交
215
        -DWITH_LITE=${WITH_LITE:-OFF}
216
    ========================================
217
EOF
218 219 220 221 222 223 224 225
    # Disable UNITTEST_USE_VIRTUALENV in docker because
    # docker environment is fully controlled by this script.
    # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
    cmake .. \
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
        ${PYTHON_FLAGS} \
        -DWITH_GPU=${WITH_GPU:-OFF} \
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} \
G
gongweibao 已提交
226
        -DWITH_DISTRIBUTE=${distibuted_flag} \
227 228
        -DWITH_MKL=${WITH_MKL:-ON} \
        -DWITH_AVX=${WITH_AVX:-OFF} \
229
        -DNOAVX_CORE_FILE=${NOAVX_CORE_FILE:-""} \
230 231 232 233
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \
        -DWITH_PYTHON=${WITH_PYTHON:-ON} \
        -DCUDNN_ROOT=/usr/ \
        -DWITH_TESTING=${WITH_TESTING:-ON} \
234
        -DWITH_COVERAGE=${WITH_COVERAGE:-OFF} \
235 236 237 238
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake \
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON} \
Z
zhangchunle 已提交
239
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR} \
W
Wu Yi 已提交
240
        -DPY_VERSION=${PY_VERSION:-2.7} \
M
minqiyang 已提交
241
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build} \
石晓伟 已提交
242 243
        -DWITH_GRPC=${grpc_flag} \
        -DWITH_LITE=${WITH_LITE:-OFF}
J
JiabinYang 已提交
244

245 246
}

247 248 249 250 251 252
function cmake_gen() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cmake_base $1
}

253 254 255 256 257 258 259 260 261 262
function abort(){
    echo "Your change doesn't follow PaddlePaddle's code style." 1>&2
    echo "Please use pre-commit to check what is wrong." 1>&2
    exit 1
}

function check_style() {
    trap 'abort' 0
    set -e

263 264 265
    if [ -x "$(command -v gimme)" ]; then
    	eval "$(GIMME_GO_VERSION=1.8.3 gimme)"
    fi
266

267 268

    pip install cpplint pylint pytest astroid isort
269 270 271 272 273 274 275
    # set up go environment for running gometalinter
    mkdir -p $GOPATH/src/github.com/PaddlePaddle/
    ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle

    pre-commit install
    clang-format --version

T
tianshuo78520a 已提交
276
    commit_files=on
277 278 279
    for file_name in `git diff --numstat upstream/$BRANCH |awk '{print $NF}'`;do
        if ! pre-commit run --files $file_name ; then
            git diff
T
tianshuo78520a 已提交
280
            commit_files=off
281 282
        fi
    done 
T
tianshuo78520a 已提交
283 284 285 286 287
    
    if [ $commit_files == 'off' ];then
        echo "code format error"
        exit 1
    fi
288 289 290 291 292 293 294
    trap : 0
}

#=================================================
#              Build
#=================================================

295
function build_base() {
296
    if [ "$SYSTEM" == "Linux" ];then
297
      if [ `nproc` -gt 16 ];then
298
          parallel_number=$(expr `nproc` - 8)
299 300 301
      else
          parallel_number=`nproc`
      fi
302 303 304 305
    else
      parallel_number=8
    fi
    if [ "$1" != "" ]; then
306 307
      parallel_number=$1
    fi
Z
Zeng Jinle 已提交
308 309 310 311 312

    if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then
        make clean
    fi

313
    make install -j ${parallel_number}
314 315
}

Z
zhangchunle 已提交
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
function build_size() {
    cat <<EOF
    ============================================
    Calculate /paddle/build size and PR whl size
    ============================================
EOF
    if [ "$1" == "fluid_inference" ]; then
        cd ${PADDLE_ROOT}/build
        cp -r fluid_inference_install_dir fluid_inference
        tar -czf fluid_inference.tgz fluid_inference
        buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/fluid_inference.tgz |awk '{print $1}')
        echo "FLuid_Inference Size: $buildSize"
    else
        SYSTEM=`uname -s`
        if [ "$SYSTEM" == "Darwin" ]; then
            com='du -h -d 0'
        else
            com='du -h --max-depth=0'
        fi
        buildSize=$($com ${PADDLE_ROOT}/build |awk '{print $1}')
        echo "Build Size: $buildSize"
        PR_whlSize=$($com ${PADDLE_ROOT}/build/python/dist |awk '{print $1}')
        echo "PR whl Size: $PR_whlSize"
    fi
}

342 343 344 345 346 347 348 349
function build() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
Z
Zeng Jinle 已提交
350
    build_base $@
Z
zhangchunle 已提交
351 352 353 354 355 356 357 358 359 360 361 362
    current_branch=`git branch | grep \* | cut -d ' ' -f2`
    if [ "$current_branch" != "develop_base_pr" ];then
        build_size
    fi
}

function cmake_gen_and_build() {
    startTime_s=`date +%s`
    cmake_gen $1
    build $2
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
363 364
}

365 366 367 368 369 370 371 372
function build_mac() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
Z
Zeng Jinle 已提交
373 374 375
    if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then
        make clean
    fi
J
JiabinYang 已提交
376
    make install -j 8
Z
zhangchunle 已提交
377 378 379 380 381 382 383 384 385
    build_size
}

function cmake_gen_and_build_mac() {
    startTime_s=`date +%s`
    cmake_gen $1
    build_mac
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
386 387
}

388 389 390 391 392 393 394 395 396
function run_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
397 398 399 400 401
        if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
            ctest -V
        else
            ctest --output-on-failure
        fi
402 403 404
    fi
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423

function combine_avx_noavx_build() {
    mkdir -p ${PADDLE_ROOT}/build.noavx
    cd ${PADDLE_ROOT}/build.noavx
    WITH_AVX=OFF
    cmake_base ${PYTHON_ABI:-""}
    build_base

    # build combined one
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    NOAVX_CORE_FILE=`find ${PADDLE_ROOT}/build.noavx/python/paddle/fluid/ -name "core_noavx.*"`
    WITH_AVX=ON

    cmake_base ${PYTHON_ABI:-""}
    build_base
}


G
gongweibao 已提交
424 425 426 427 428 429 430 431 432 433 434 435
function run_brpc_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [[ ${WITH_TESTING:-ON} == "ON" \
        && ${WITH_DISTRIBUTE:-OFF} == "ON" \
        && ${WITH_GRPC:-OFF} == "OFF" ]] ; then
    cat <<EOF
    ========================================
    Running brpc unit tests ...
    ========================================
EOF
        set +x
436 437
        declare -a other_tests=("test_listen_and_serv_op" "system_allocator_test" \
        "rpc_server_test" "varhandle_test" "collective_server_test" "brpc_serde_test")
G
gongweibao 已提交
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
        all_tests=`ctest -N`

        for t in "${other_tests[@]}"
        do
            if [[ ${all_tests} != *$t* ]]; then
                continue
            fi

            if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
                ctest -V -R $t
            else
                ctest --output-on-failure -R $t
            fi
        done
        set -x

        if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
            ctest -V -R test_dist_*
        else
            ctest --output-on-failure -R test_dist_*
        fi
    fi
}



464 465 466 467 468 469 470 471 472
function run_mac_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
473 474
        #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. 
        #see details: https://github.com/PaddlePaddle/Paddle/issues/24738
475
        my_proxy=$http_proxy
J
JiabinYang 已提交
476 477
        export http_proxy=
        export https_proxy=
478 479 480 481 482 483 484 485 486 487 488 489 490

        set +ex
        if [ "$1" == "cp27-cp27m" ]; then
            pip uninstall -y paddlepaddle
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 uninstall -y paddlepaddle
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 uninstall -y paddlepaddle
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 uninstall -y paddlepaddle
        fi
        set -ex

491
        if [ "$1" == "cp27-cp27m" ]; then
492
            set -e
493
            pip install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
J
JiabinYang 已提交
494
            python ${PADDLE_ROOT}/paddle/scripts/installation_validate.py
495 496 497 498 499 500 501
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        fi
Z
zhangchunle 已提交
502
        ut_startTime_s=`date +%s`
503
        ctest --output-on-failure -j $2
Z
zhangchunle 已提交
504 505
        ut_endTime_s=`date +%s`
        echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
T
Tao Luo 已提交
506
        paddle version
507 508 509
        # Recovery proxy to avoid failure in later steps
        export http_proxy=$my_proxy
        export https_proxy=$my_proxy
510 511 512
    fi
}

Z
Zeng Jinle 已提交
513 514 515 516 517 518 519 520 521 522 523 524 525 526
function fetch_upstream_develop_if_not_exist() {
    UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle'
    origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` 
    if [ "$origin_upstream_url" == "" ]; then
        git remote add upstream $UPSTREAM_URL.git
    elif [ "$origin_upstream_url" != "$UPSTREAM_URL" ] \
            && [ "$origin_upstream_url" != "$UPSTREAM_URL.git" ]; then
        git remote remove upstream
        git remote add upstream $UPSTREAM_URL.git
    fi
    
    if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then 
        git fetch upstream # develop is not fetched
    fi
Z
Zeng Jinle 已提交
527
}
Z
Zeng Jinle 已提交
528 529 530

function generate_upstream_develop_api_spec() {
    fetch_upstream_develop_if_not_exist
Z
Zeng Jinle 已提交
531
    cur_branch=`git branch | grep \* | cut -d ' ' -f2`
Z
Zeng Jinle 已提交
532 533 534
    git checkout -b develop_base_pr upstream/$BRANCH
    cmake_gen $1
    build $2
Z
Zeng Jinle 已提交
535

Z
Zeng Jinle 已提交
536
    git checkout $cur_branch
Z
Zeng Jinle 已提交
537
    generate_api_spec "$1" "DEV"
538
    git branch -D develop_base_pr
539 540
    ENABLE_MAKE_CLEAN="ON"
    rm -rf ${PADDLE_ROOT}/build/Makefile ${PADDLE_ROOT}/build/CMakeCache.txt
Z
Zeng Jinle 已提交
541 542 543 544 545 546 547 548 549
}

function generate_api_spec() {
    spec_kind=$2
    if [ "$spec_kind" != "PR" ] && [ "$spec_kind" != "DEV" ]; then
        echo "Not supported $2"
        exit 1
    fi

Y
yuyang18 已提交
550 551
    mkdir -p ${PADDLE_ROOT}/build/.check_api_workspace
    cd ${PADDLE_ROOT}/build/.check_api_workspace
Z
Zeng Jinle 已提交
552 553
    virtualenv .${spec_kind}_env
    source .${spec_kind}_env/bin/activate
Y
yuyang18 已提交
554
    pip install ${PADDLE_ROOT}/build/python/dist/*whl
555
    spec_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.spec
Z
Zeng Jinle 已提交
556
    python ${PADDLE_ROOT}/tools/print_signatures.py paddle > $spec_path
557

558 559 560
    # used to log op_register data_type
    op_type_path=${PADDLE_ROOT}/paddle/fluid/OP_TYPE_${spec_kind}.spec
    python ${PADDLE_ROOT}/tools/check_op_register_type.py > $op_type_path
561 562 563 564 565

    # print all ops desc in dict to op_desc_path
    op_desc_path=${PADDLE_ROOT}/paddle/fluid/OP_DESC_${spec_kind}.spec
    python ${PADDLE_ROOT}/tools/print_op_desc.py > $op_desc_path

T
tianshuo78520a 已提交
566 567
    awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc
    awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api
Z
Zeng Jinle 已提交
568
    if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then 
M
minqiyang 已提交
569
        # Use sed to make python2 and python3 sepc keeps the same
Z
Zeng Jinle 已提交
570 571 572
        sed -i 's/arg0: str/arg0: unicode/g' $spec_path
        sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path
    fi   
Z
Zeng Jinle 已提交
573 574 575
    
    python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \
        ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec
S
sneaxiy 已提交
576

Y
yuyang18 已提交
577
    deactivate
578 579
}

580
function check_approvals_of_unittest() {
581 582 583
    if [ "$GITHUB_API_TOKEN" == "" ] || [ "$GIT_PR_ID" == "" ]; then
        return 0
    fi
584 585
    # approval_user_list: XiaoguangHu01 46782768,luotao1 6836917,phlrain 43953930,lanxianghit 47554610, zhouwei25 52485244, kolinwei 22165420
    check_times=$1
586 587
    if [ $check_times == 1 ]; then
        approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
588
        if [ "${approval_line}" != "" ]; then
589
            APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244`
590 591 592 593 594 595 596 597
            set +x
            echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
            if [ "${APPROVALS}" == "TRUE" ]; then
                echo "==================================="
                echo -e "\n current pr ${GIT_PR_ID} has got approvals. So, Pass CI directly!\n"
                echo "==================================="
                exit 0
            fi
598
        fi
599
    elif [ $check_times == 2 ]; then
600 601
        unittest_spec_diff=`python ${PADDLE_ROOT}/tools/diff_unittest.py ${PADDLE_ROOT}/paddle/fluid/UNITTEST_DEV.spec ${PADDLE_ROOT}/paddle/fluid/UNITTEST_PR.spec`
        if [ "$unittest_spec_diff" != "" ]; then
602
            approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
603
            APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244`
604
            set +x
605 606 607 608 609
            echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
            if [ "${APPROVALS}" == "FALSE" ]; then
                echo "************************************"
                echo -e "It is forbidden to disable or delete the unit-test.\n"
                echo -e "If you must delete it temporarily, please add it to[https://github.com/PaddlePaddle/Paddle/wiki/Temporarily-disabled-Unit-Test]."
610
                echo -e "Then you must have one RD (kolinwei(recommended) or zhouwei25) approval for the deletion of unit-test. \n"
611 612 613 614 615
                echo -e "If you have any problems about deleting unit-test, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/Deleting-unit-test-is-forbidden]. \n"
                echo -e "Following unit-tests are deleted in this PR: \n ${unittest_spec_diff} \n"
                echo "************************************"
                exit 1
            fi
616 617
        fi
    fi
618
    set -x
619 620 621 622 623 624 625 626 627
}

function check_change_of_unittest() {
    generate_unittest_spec "PR"
    fetch_upstream_develop_if_not_exist
    git reset --hard upstream/$BRANCH
    cmake_gen $1
    generate_unittest_spec "DEV"
    check_approvals_of_unittest 2
628 629
}

630 631 632 633
function check_sequence_op_unittest(){
    /bin/bash ${PADDLE_ROOT}/tools/check_sequence_op.sh
}

634 635 636 637
function generate_unittest_spec() {
    spec_kind=$1
    if [ "$spec_kind" == "DEV" ]; then
        cat <<EOF
638 639 640
        ============================================
        Generate unit tests.spec of develop.
        ============================================
641 642 643
EOF
    elif [ "$spec_kind" == "PR" ]; then
        cat <<EOF
644 645 646
        ============================================
        Generate unit tests.spec of this PR.
        ============================================
647 648 649 650 651
EOF
    else
        echo "Not supported $1"
        exit 1
    fi
652
    spec_path=${PADDLE_ROOT}/paddle/fluid/UNITTEST_${spec_kind}.spec
653 654 655
    ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' > ${spec_path}
}

Z
Zeng Jinle 已提交
656

657
function assert_api_spec_approvals() {
658 659 660
    /bin/bash ${PADDLE_ROOT}/tools/check_api_approvals.sh
    if [ "$?" != 0 ];then
       exit 1
661
    fi
Y
yuyang18 已提交
662 663 664
}


L
Liu Xudong 已提交
665 666 667 668 669
function check_coverage() {
    /bin/bash ${PADDLE_ROOT}/tools/coverage/paddle_coverage.sh
}


670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
function single_test() {
    TEST_NAME=$1
    if [ -z "${TEST_NAME}" ]; then
        echo -e "${RED}Usage:${NONE}"
        echo -e "${BOLD}${SCRIPT_NAME}${NONE} ${BLUE}single_test${NONE} [test_name]"
        exit 1
    fi
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running ${TEST_NAME} ...
    ========================================
EOF
        ctest --output-on-failure -R ${TEST_NAME}
    fi
}

689 690 691
function bind_test() {
    # the number of process to run tests
    NUM_PROC=6
Q
Qiyang Min 已提交
692

693 694 695
    # calculate and set the memory usage for each process
    MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
    export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
Q
Qiyang Min 已提交
696

697 698
    # get the CUDA device count
    CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
Q
Qiyang Min 已提交
699

700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
    for (( i = 0; i < $NUM_PROC; i++ )); do
        cuda_list=()
        for (( j = 0; j < $CUDA_DEVICE_COUNT; j++ )); do
            s=$[i+j]
            n=$[s%CUDA_DEVICE_COUNT]
            if [ $j -eq 0 ]; then
                cuda_list=("$n")
            else
                cuda_list="$cuda_list,$n"
            fi
        done
        echo $cuda_list
        # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
        # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
        env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC --output-on-failure &
    done
    wait
}

719 720 721 722 723 724 725 726 727 728 729
EXIT_CODE=0;
function caught_error() {
 for job in `jobs -p`; do
        # echo "PID => ${job}"
        if ! wait ${job} ; then
            echo "At least one test failed with exit code => $?" ;
            EXIT_CODE=1;
        fi
    done
}

Z
zhangchunle 已提交
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
function case_count(){
    cat <<EOF
    ============================================
    Generating TestCases Count ... 
    ============================================
EOF
    testcases=$1
    num=$(echo $testcases|grep -o '\^'|wc -l)
    if [ "$2" == "" ]; then
        echo "exclusive TestCases count is $num"
    else
        echo "$2 card TestCases count is $num"
    fi
}

Z
zhangchunle 已提交
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
failed_test_lists=''
tmp_dir=`mktemp -d`

function collect_failed_tests() {
    for file in `ls $tmp_dir`; do
        exit_code=0
        grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$?
        if [ $exit_code -ne 0 ]; then
            failuretest=''
        else
            failuretest=`grep -A 10000 'The following tests FAILED:' $tmp_dir/$file | sed 's/The following tests FAILED://g'|sed '/^$/d'`
            failed_test_lists="${failed_test_lists}
            ${failuretest}"
        fi
    done
}

762 763
function card_test() {
    set -m
Z
zhangchunle 已提交
764 765
    case_count $1 $2
    ut_startTime_s=`date +%s` 
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
    # get the CUDA device count
    CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)

    testcases=$1
    if (( $# > 1 )); then
        cardnumber=$2
        if (( $cardnumber > $CUDA_DEVICE_COUNT )); then
            cardnumber=$CUDA_DEVICE_COUNT
        fi
    else
        cardnumber=$CUDA_DEVICE_COUNT
    fi

    if [[ "$testcases" == "" ]]; then
        return 0
    fi

    trap 'caught_error' CHLD
Z
zhangchunle 已提交
784
    tmpfile_rand=`date +%s%N`
785 786 787 788 789 790 791 792 793 794 795 796
    NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber]
    for (( i = 0; i < $NUM_PROC; i++ )); do
        # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
        # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
        cuda_list=()
        for (( j = 0; j < cardnumber; j++ )); do
            if [ $j -eq 0 ]; then
                    cuda_list=("$[i*cardnumber]")
                else
                    cuda_list="$cuda_list,$[i*cardnumber+j]"
            fi
        done
Z
zhangchunle 已提交
797
        tmpfile=$tmp_dir/$tmpfile_rand"_"$i
798 799
        if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
            if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
Z
zhangchunle 已提交
800 801 802
                (ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
            else  
                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
803 804 805
            fi
        else
            if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
Z
zhangchunle 已提交
806
                (ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
807
            else
Z
zhangchunle 已提交
808
                (env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure | tee $tmpfile; test ${PIPESTATUS[0]} -eq 0) &
809 810 811 812
            fi
        fi
    done
    wait; # wait for all subshells to finish
Z
zhangchunle 已提交
813 814 815 816 817 818
    ut_endTime_s=`date +%s`
    if [ "$2" == "" ]; then
        echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
    else
        echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
    fi
819
    set +m
820 821
}

T
tianshuo78520a 已提交
822
function parallel_test_base_gpu() {
823 824 825
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
826
    Running unit tests in parallel way ...
827 828 829 830 831
    ========================================
EOF

set +x
        EXIT_CODE=0;
832 833 834 835 836 837
        test_cases=$(ctest -N -V) # get all test cases
        exclusive_tests=''        # cases list which would be run exclusively
        single_card_tests=''      # cases list which would take one graph card
        multiple_card_tests=''    # cases list which would take multiple GPUs, most cases would be two GPUs
        is_exclusive=''           # indicate whether the case is exclusive type
        is_multicard=''           # indicate whether the case is multiple GPUs type
838 839 840 841 842 843 844 845
        while read -r line; do
            if [[ "$line" == "" ]]; then
                continue
            fi
                read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#')
                if [[ "$matchstr" == "" ]]; then
                    # Any test case with LABELS property would be parse here
                    # RUN_TYPE=EXCLUSIVE mean the case would run exclusively
846
                    # RUN_TYPE=DIST mean the case would take two graph GPUs during runtime
847 848 849 850 851 852 853
                    read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE")
                    read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST")
                    continue
                fi
                read testcase <<< $(echo "$line"|grep -oEi "\w+$")

                if [[ "$is_multicard" == "" ]]; then
854
                  # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs
855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
                  read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist")
                fi

                if [[ "$is_exclusive" != "" ]]; then
                    if [[ "$exclusive_tests" == "" ]]; then
                        exclusive_tests="^$testcase$"
                    else
                        exclusive_tests="$exclusive_tests|^$testcase$"
                    fi
                elif [[ "$is_multicard" != "" ]]; then
                    if [[ "$multiple_card_tests" == "" ]]; then
                        multiple_card_tests="^$testcase$"
                    else
                        multiple_card_tests="$multiple_card_tests|^$testcase$"
                    fi
                else
871 872 873 874 875 876 877 878 879
                    if [[ "${#single_card_tests}" -gt 3000 ]];then
                        if [[ "$single_card_tests_1" == "" ]]; then 
                            single_card_tests_1="^$testcase$"
                        else
                            single_card_tests_1="$single_card_tests_1|^$testcase$"
                        fi
                        continue
                    fi

880 881 882 883 884 885 886 887 888 889 890 891
                    if [[ "$single_card_tests" == "" ]]; then
                        single_card_tests="^$testcase$"
                    else
                        single_card_tests="$single_card_tests|^$testcase$"
                    fi
                fi
                is_exclusive=''
                is_multicard=''
                matchstr=''
                testcase=''
        done <<< "$test_cases";

892
        card_test "$single_card_tests" 1    # run cases with single GPU
893
        card_test "$single_card_tests_1" 1    # run cases with single GPU
894 895
        card_test "$multiple_card_tests" 2  # run cases with two GPUs
        card_test "$exclusive_tests"        # run cases exclusively, in this cases would be run with 4/8 GPUs
Z
zhangchunle 已提交
896 897 898 899 900 901 902 903 904 905
        collect_failed_tests
        if [ -n "${failed_test_lists}" ];then
            failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
            echo "========================================"
            echo "Summary Failed Tests... "
            echo "========================================"
            echo "The following tests FAILED: "
            echo "${failed_test_lists_ult}"
        fi
        rm -f $tmp_dir/*
906
        if [[ "$EXIT_CODE" != "0" ]]; then
907
            exit 8;
908 909 910 911 912
        fi
set -ex
    fi
}

T
tianshuo78520a 已提交
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
function parallel_test_base_cpu() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit cpu tests ...
    ========================================
EOF
        ut_startTime_s=`date +%s`
        ctest --output-on-failure -j $1
        ut_endTime_s=`date +%s`
        echo "CPU testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
        if [[ "$EXIT_CODE" != "0" ]]; then
            exit 8;
        fi
    fi
}

932
function parallel_test() {
Z
zhangchunle 已提交
933
    ut_total_startTime_s=`date +%s`
934 935
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
T
tianshuo78520a 已提交
936 937 938 939 940
    if [ "$WITH_GPU" == "ON" ];then
        parallel_test_base_gpu
    else
        parallel_test_base_cpu ${PROC_RUN:-1}
    fi
Z
zhangchunle 已提交
941 942
    ut_total_endTime_s=`date +%s`
    echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s"
943 944
}

945 946 947 948 949 950 951
function enable_unused_var_check() {
    # NOTE(zhiqiu): Set FLAGS_enable_unused_var_check=1 here to enable unused_var_check,
    # which checks if an operator has unused input variable(s).
    # Currently, use it in coverage CI job.
    export FLAGS_enable_unused_var_check=1
}

952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
function gen_doc_lib() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ========================================
    Building documentation library ...
    In /paddle/build
    ========================================
EOF
    cmake .. \
        -DCMAKE_BUILD_TYPE=Release \
        -DWITH_GPU=OFF \
        -DWITH_MKL=OFF \

    local LIB_TYPE=$1
    case $LIB_TYPE in
      full)
        # Build full Paddle Python module. Will timeout without caching 'copy_paddle_pybind' first
970
        make -j `nproc` framework_py_proto copy_paddle_pybind paddle_python
971 972 973 974 975 976 977 978 979 980 981 982 983
        ;;
      pybind)
        # Build paddle pybind library. Takes 49 minutes to build. Might timeout
        make -j `nproc` copy_paddle_pybind
        ;;
      proto)
        # Even smaller library.
        make -j `nproc` framework_py_proto
        ;;
      *)
        exit 0
        ;;
      esac
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
}

function gen_html() {
    cat <<EOF
    ========================================
    Converting C++ source code into HTML ...
    ========================================
EOF
    export WOBOQ_OUT=${PADDLE_ROOT}/build/woboq_out
    mkdir -p $WOBOQ_OUT
    cp -rv /woboq/data $WOBOQ_OUT/../data
    /woboq/generator/codebrowser_generator \
    	-b ${PADDLE_ROOT}/build \
    	-a \
    	-o $WOBOQ_OUT \
    	-p paddle:${PADDLE_ROOT}
    /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
}

function gen_dockerfile() {
    # Set BASE_IMAGE according to env variables
1005 1006
    CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1).$(echo $CUDA_VERSION | cut -d '.' -f 2)"
    CUDNN_MAJOR=$(echo $CUDNN_VERSION | cut -d '.' -f 1)
1007
    if [[ ${WITH_GPU} == "ON" ]]; then
1008
        BASE_IMAGE="nvidia/cuda:${CUDA_MAJOR}-cudnn${CUDNN_MAJOR}-devel-ubuntu16.04"
1009
    else
1010
        BASE_IMAGE="ubuntu:16.04"
1011 1012 1013 1014
    fi

    DOCKERFILE_GPU_ENV=""
    DOCKERFILE_CUDNN_DSO=""
X
Xi Chen 已提交
1015
    DOCKERFILE_CUBLAS_DSO=""
1016 1017
    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:\${LD_LIBRARY_PATH}"
X
Xi Chen 已提交
1018
        DOCKERFILE_CUDNN_DSO="RUN ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.${CUDNN_MAJOR} /usr/lib/x86_64-linux-gnu/libcudnn.so"
X
Xi Chen 已提交
1019
        DOCKERFILE_CUBLAS_DSO="RUN ln -sf /usr/local/cuda/targets/x86_64-linux/lib/libcublas.so.${CUDA_MAJOR} /usr/lib/x86_64-linux-gnu/libcublas.so"
1020 1021 1022 1023
    fi

    cat <<EOF
    ========================================
1024
    Generate ${PADDLE_ROOT}/build/Dockerfile ...
1025 1026
    ========================================
EOF
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
    
    ref_CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1)"
    if [[ ${WITH_GPU} == "ON"  ]]; then
        ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR}
    else
        ref_gpu=cpu
    fi
    if [[ ${WITH_GPU} == "ON"  ]]; then
        install_gpu="_gpu"
    else
        install_gpu=""
    fi
    if [[ ${WITH_MKL} == "ON" ]]; then
        ref_mkl=mkl
    else
1042
        ref_mkl=openblas
1043
    fi
1044

1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
    ref_web=https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}

    ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl
    ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
    ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
    ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl

    ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl
    ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
    ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
    ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl

1057
    if [[ ${PADDLE_BRANCH} != "0.0.0" && ${WITH_MKL} == "ON" && ${WITH_GPU} == "ON" ]]; then
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
        ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
        ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
        ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
        ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
        ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
    fi

    #ref_paddle2_mv1=""
    #ref_paddle2_mv2=""
    ref_paddle35_mv1=""
    ref_paddle35_mv2=""
    ref_paddle36_mv1=""
    ref_paddle36_mv2=""
    #ref_paddle37_mv1=""
    #ref_paddle37_mv2=""
1076
    if [[ ${PADDLE_BRANCH} == "0.0.0" && ${WITH_GPU} == "ON" ]]; then
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
        #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle_gpu-1.5.1-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle_gpu-1.5.1-cp36-cp36m-linux_x86_64.whl
        #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl
        #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&"
        #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2"
        ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&"
        ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}"
        ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&"
        ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}"
        #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&"
        #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37"
    fi
1090
    if [[ ${PADDLE_BRANCH} == "0.0.0" && ${WITH_GPU} != "ON" ]]; then
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
        #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl
        ref_paddle35_whl=paddlepaddle-1.5.1-cp35-cp35m-linux_x86_64.whl
        ref_paddle36_whl=paddlepaddle-1.5.1-cp36-cp36m-linux_x86_64.whl
        #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl
        #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&"
        #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2"
        ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&"
        ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}"
        ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&"
        ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}"
        #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&"
        #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37"
    fi
    
1105 1106 1107 1108 1109 1110 1111
    cat > ${PADDLE_ROOT}/build/Dockerfile <<EOF
    FROM ${BASE_IMAGE}
    MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
    ENV HOME /root
EOF

    if [[ ${WITH_GPU} == "ON"  ]]; then
1112
        NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true"
1113
    else
1114
        NCCL_DEPS="true"
1115 1116
    fi

1117 1118 1119 1120
    if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then 
        NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0"
    fi

T
Tao Luo 已提交
1121 1122
    PADDLE_VERSION="paddle version"
    CMD='"paddle", "version"'
1123 1124
    
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1125 1126 1127
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python3 python3-pip libgtk2.0-dev dmidecode python3-tk && \
1128
        pip3 install opencv-python py-cpuinfo==5.0.0 && wget ${ref_web}/${ref_paddle35} && ${ref_paddle35_mv1} pip3 install ${ref_paddle35_whl} ${ref_paddle35_mv2}; apt-get install -f -y && \
1129
        apt-get clean -y && \
1130
        rm -f ${ref_paddle35} && \
1131 1132 1133 1134
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_CUBLAS_DSO}
    ${DOCKERFILE_GPU_ENV}
1135
EOF
1136
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1137 1138 1139 1140 1141 1142 1143
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \
        tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \
T
Tao Luo 已提交
1144
        ./configure -prefix=/usr/local && make install -j8 && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \
1145 1146 1147
        wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \
        tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
1148 1149 1150
        make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.6.0.tgz
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
        pip3.6 install opencv-python && wget ${ref_web}/${ref_paddle36} && ${ref_paddle36_mv1} pip3.6 install ${ref_paddle36_whl} ${ref_paddle36_mv2}; apt-get install -f -y && \
1151
        apt-get clean -y && \
1152
        rm -f ${ref_paddle36} && \
1153 1154
        ldconfig
EOF
1155
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1156 1157 1158 1159 1160 1161 1162 1163
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \
        tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
1164 1165 1166
        make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.7.0.tgz
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
        pip3.7 install opencv-python && wget ${ref_web}/${ref_paddle37} && pip3.7 install ${ref_paddle37_whl}; apt-get install -f -y && \
1167
        apt-get clean -y && \
1168
        rm -f ${ref_paddle37} && \
1169
        ldconfig
1170
EOF
1171
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1172
    # run paddle version to install python packages first
1173 1174
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python-pip python-opencv libgtk2.0-dev dmidecode python-tk && easy_install -U pip && \
1175
        wget ${ref_web}/${ref_paddle2} && pip install ${ref_paddle2_whl}; apt-get install -f -y && \
1176
        apt-get clean -y && \
1177
        rm -f ${ref_paddle2} && \
1178 1179
        ${PADDLE_VERSION} && \
        ldconfig
1180
EOF
1181

1182
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
1183 1184 1185 1186 1187
    # default command shows the paddle version and exit
    CMD [${CMD}]
EOF
}

1188
function gen_fluid_lib() {
L
Lei Wang 已提交
1189 1190
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
T
Tao Luo 已提交
1191
    cat <<EOF
1192
    ========================================
1193
    Generating fluid library for train and inference ...
1194 1195
    ========================================
EOF
1196 1197 1198 1199
    parallel_number=`nproc`
    if [[ "$1" != "" ]]; then
      parallel_number=$1
    fi
Z
zhangchunle 已提交
1200
    startTime_s=`date +%s` 
R
randytli 已提交
1201
    cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto}
1202

1203 1204
    make -j ${parallel_number} fluid_lib_dist
    make -j ${parallel_number} inference_lib_dist
Z
zhangchunle 已提交
1205 1206 1207
    endTime_s=`date +%s`
    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
    build_size "fluid_inference"
T
Tao Luo 已提交
1208 1209
}

1210
function tar_fluid_lib() {
T
Tao Luo 已提交
1211
    cat <<EOF
T
Tao Luo 已提交
1212
    ========================================
1213
    Taring fluid library for train and inference ...
T
Tao Luo 已提交
1214 1215
    ========================================
EOF
T
Tao Luo 已提交
1216 1217 1218 1219 1220
    cd ${PADDLE_ROOT}/build
    cp -r fluid_install_dir fluid
    tar -czf fluid.tgz fluid
    cp -r fluid_inference_install_dir fluid_inference
    tar -czf fluid_inference.tgz fluid_inference
1221 1222
}

1223
function test_fluid_lib() {
T
Tao Luo 已提交
1224
    cat <<EOF
1225
    ========================================
1226
    Testing fluid library for inference ...
1227 1228
    ========================================
EOF
Z
zhangchunle 已提交
1229
    fluid_startTime_s=`date +%s`
T
Tao Luo 已提交
1230 1231 1232 1233
    cd ${PADDLE_ROOT}/paddle/fluid/inference/api/demo_ci
    ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON} ${WITH_GPU:-OFF} ${INFERENCE_DEMO_INSTALL_DIR} \
             ${TENSORRT_INCLUDE_DIR:-/usr/local/TensorRT/include} \
             ${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib}
Z
zhangchunle 已提交
1234 1235
    fluid_endTime_s=`date +%s`
    echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"          
T
Tao Luo 已提交
1236
    ./clean.sh
1237 1238
}

1239 1240 1241 1242 1243 1244
function test_fluid_lib_train() {
    cat <<EOF
    ========================================
    Testing fluid library for training ...
    ========================================
EOF
Z
zhangchunle 已提交
1245
    fluid_train_startTime_s=`date +%s`
1246 1247
    cd ${PADDLE_ROOT}/paddle/fluid/train/demo
    ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON}
Z
zhangchunle 已提交
1248 1249
    fluid_train_endTime_s=`date +%s`
    echo "test_fluid_lib_train Total Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s"
1250 1251
    ./clean.sh
}
T
tianshuo78520a 已提交
1252 1253 1254 1255 1256 1257

function build_document_preview() {
    sh /paddle/tools/document_preview.sh ${PORT}
}


T
tianshuo78520a 已提交
1258
function example() {
Z
Zeng Jinle 已提交
1259
    pip install ${PADDLE_ROOT}/build/python/dist/*.whl
T
tianshuo78520a 已提交
1260
    paddle version
1261
    cd ${PADDLE_ROOT}/tools
1262
    python sampcd_processor.py cpu 
T
tianshuo78520a 已提交
1263 1264 1265 1266 1267 1268
    if [ "$?" != "0" ];then
      echo "Code instance execution failed"
      exit 1
    fi
}

1269
function main() {
Z
Zeng Jinle 已提交
1270
    local CMD=$1 
1271
    local parallel_number=$2
1272 1273
    init
    case $CMD in
1274
      build_only)
Z
zhangchunle 已提交
1275
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1276 1277
        ;;
      build_and_check)
1278
        check_style
Z
Zeng Jinle 已提交
1279
        generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
Z
zhangchunle 已提交
1280
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1281 1282
        check_sequence_op_unittest
        generate_api_spec ${PYTHON_ABI:-""} "PR"
1283
        example
T
tianshuo78520a 已提交
1284
        assert_api_spec_approvals
1285
        ;;
1286 1287
      build)
        cmake_gen ${PYTHON_ABI:-""}
1288
        build ${parallel_number}
1289
        gen_dockerfile ${PYTHON_ABI:-""}
1290
        assert_api_spec_approvals
1291
        ;;
1292 1293
      combine_avx_noavx)
        combine_avx_noavx_build
1294
        gen_dockerfile ${PYTHON_ABI:-""}
1295 1296 1297
        ;;
      combine_avx_noavx_build_and_test)
        combine_avx_noavx_build
1298
        gen_dockerfile ${PYTHON_ABI:-""}
1299 1300
        parallel_test_base
        ;;
1301
      test)
1302
        parallel_test
1303
        ;;
1304 1305 1306
      single_test)
        single_test $2
        ;;
1307 1308 1309
      bind_test)
        bind_test
        ;;
1310 1311 1312
      gen_doc_lib)
        gen_doc_lib $2
        ;;
1313 1314 1315 1316
      html)
        gen_html
        ;;
      dockerfile)
1317
        gen_dockerfile ${PYTHON_ABI:-""}
1318 1319 1320
        ;;
      fluid_inference_lib)
        cmake_gen ${PYTHON_ABI:-""}
1321
        gen_fluid_lib ${parallel_number}
1322 1323
        tar_fluid_lib
        test_fluid_lib
1324 1325 1326 1327
        ;;
      check_style)
        check_style
        ;;
L
Lei Wang 已提交
1328
      cicheck)
1329 1330 1331 1332 1333 1334
        cmake_gen ${PYTHON_ABI:-""}
        build ${parallel_number}
        enable_unused_var_check
        parallel_test
        ;;
      cicheck_coverage)
1335
        check_approvals_of_unittest 1
Z
zhangchunle 已提交
1336
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1337
        enable_unused_var_check
1338
        parallel_test
L
Liu Xudong 已提交
1339
        check_coverage
1340
        check_change_of_unittest ${PYTHON_ABI:-""}
L
Lei Wang 已提交
1341
        ;;
G
gongweibao 已提交
1342 1343
      cicheck_brpc)
        cmake_gen ${PYTHON_ABI:-""}
1344
        build ${parallel_number}
G
gongweibao 已提交
1345 1346
        run_brpc_test
        ;;
1347
      assert_api)
Z
Zeng Jinle 已提交
1348
        generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
Y
Yu Yang 已提交
1349
        assert_api_spec_approvals
1350 1351
        ;;
      test_inference)
1352
        gen_fluid_lib ${parallel_number}
1353
        test_fluid_lib
1354 1355 1356 1357 1358
        test_fluid_lib_train
        ;;
      test_train)
        gen_fluid_lib ${parallel_number}
        test_fluid_lib_train
1359 1360 1361 1362
        ;;
      assert_api_approvals)
        assert_api_spec_approvals
        ;;
1363
      maccheck)
Z
zhangchunle 已提交
1364
        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
L
lujun 已提交
1365
        run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
1366
        ;;
1367
      maccheck_py35)
Z
zhangchunle 已提交
1368
        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
1369
        run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
1370
        check_change_of_unittest ${PYTHON_ABI:-""}
1371
        ;;
J
JiabinYang 已提交
1372 1373 1374 1375
      macbuild)
        cmake_gen ${PYTHON_ABI:-""}
        build_mac
        ;;
M
minqiyang 已提交
1376
      cicheck_py35)
Z
zhangchunle 已提交
1377
        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
1378
        parallel_test
M
minqiyang 已提交
1379
        ;;
Y
Yu Yang 已提交
1380 1381 1382 1383
      cmake_gen)
        cmake_gen ${PYTHON_ABI:-""}
        ;;
      gen_fluid_lib)
1384
        gen_fluid_lib ${parallel_number}
Y
Yu Yang 已提交
1385 1386 1387 1388
        ;;
      test_fluid_lib)
        test_fluid_lib
        ;;
T
tianshuo78520a 已提交
1389 1390 1391 1392 1393
      document)
        cmake_gen ${PYTHON_ABI:-""}
        build ${parallel_number}
        build_document_preview
        ;;
T
tianshuo78520a 已提交
1394 1395 1396
      api_example)
        example
        ;;
1397 1398
      *)
        print_usage
1399
        exit 1
1400 1401
        ;;
      esac
1402
      echo "paddle_build script finished as expected"
1403 1404 1405
}

main $@