paddle_build.sh 31.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#!/usr/bin/env bash

# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#=================================================
#                   Utils
#=================================================

21 22
set -ex

23 24
function print_usage() {
    echo -e "\n${RED}Usage${NONE}:
25
    ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]"
Q
Qiyang Min 已提交
26

27 28 29
    echo -e "\n${RED}Options${NONE}:
    ${BLUE}build${NONE}: run build for x86 platform
    ${BLUE}test${NONE}: run all unit tests
30
    ${BLUE}single_test${NONE}: run a single unit test
31 32
    ${BLUE}bind_test${NONE}: parallel tests bind to different GPU
    ${BLUE}doc${NONE}: generate paddle documents
33
    ${BLUE}gen_doc_lib${NONE}: generate paddle documents library
34 35 36 37
    ${BLUE}html${NONE}: convert C++ source code into HTML
    ${BLUE}dockerfile${NONE}: generate paddle release dockerfile
    ${BLUE}fluid_inference_lib${NONE}: deploy fluid inference library
    ${BLUE}check_style${NONE}: run code style check
L
Lei Wang 已提交
38
    ${BLUE}cicheck${NONE}: run CI tasks
39
    ${BLUE}assert_api_not_changed${NONE}: check api compability
40 41 42 43
    "
}

function init() {
44 45 46 47 48
    RED='\033[0;31m'
    BLUE='\033[0;34m'
    BOLD='\033[1m'
    NONE='\033[0m'

49
    PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
50 51 52
    if [ -z "${SCRIPT_NAME}" ]; then
        SCRIPT_NAME=$0
    fi
53 54 55 56 57 58 59 60 61 62 63 64 65 66
}

function cmake_gen() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build

    # build script will not fail if *.deb does not exist
    rm *.deb 2>/dev/null || true
    # delete previous built whl packages
    rm -rf python/dist 2>/dev/null || true

    # Support build for all python versions, currently
    # including cp27-cp27m and cp27-cp27mu.
    PYTHON_FLAGS=""
67
    SYSTEM=`uname -s`
J
Jiabin Yang 已提交
68
    if [ "$SYSTEM" == "Darwin" ]; then
V
velconia 已提交
69
        echo "Using python abi: $1"
J
Jiabin Yang 已提交
70
        if [[ "$1" == "cp27-cp27m" ]] || [[ "$1" == "" ]]; then
71 72 73
            if [ -d "/Library/Frameworks/Python.framework/Versions/2.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7
J
Jiabin Yang 已提交
74
                export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:${PATH}
75 76
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7
J
Jiabin Yang 已提交
77
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/2.7/lib/libpython2.7.dylib"
78
            pip install --user -r ${PADDLE_ROOT}/python/requirements.txt
79 80 81
            else
                exit 1
            fi
V
velconia 已提交
82
        elif [ "$1" == "cp35-cp35m" ]; then
V
velconia 已提交
83
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.5" ]; then
V
velconia 已提交
84 85 86 87 88 89
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib"
90
                pip3.5 uninstall -y protobuf
91
                pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt
V
velconia 已提交
92 93 94
            else
                exit 1
            fi
M
minqiyang 已提交
95 96 97 98 99 100 101 102
        elif [ "$1" == "cp36-cp36m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.6" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib"
103
                pip3.6 uninstall -y protobuf
104
                pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
105 106 107 108 109 110 111 112 113 114 115
            else
                exit 1
            fi
        elif [ "$1" == "cp37-cp37m" ]; then
            if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then
                export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
                export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/
            -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib"
116
                pip3.7 uninstall -y protobuf
117
                pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt
M
minqiyang 已提交
118 119 120
            else
                exit 1
            fi
121
        fi
M
minqiyang 已提交
122
    else
123
        if [ "$1" != "" ]; then
M
minqiyang 已提交
124
            echo "using python abi: $1"
125 126 127 128 129 130
            if [ "$1" == "cp27-cp27m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:}
                export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so"
131
                pip uninstall -y protobuf
132
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
133 134 135 136 137 138
            elif [ "$1" == "cp27-cp27mu" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:}
                export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
                PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
            -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so"
139
                pip uninstall -y protobuf
140
                pip install -r ${PADDLE_ROOT}/python/requirements.txt
141 142 143 144
            elif [ "$1" == "cp35-cp35m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3
M
minqiyang 已提交
145 146
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so"
147
                pip3.5 uninstall -y protobuf
148
                pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt
149 150 151 152 153 154
            elif [ "$1" == "cp36-cp36m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH}
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so"
155
                pip3.6 uninstall -y protobuf
156
                pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt
157 158 159
            elif [ "$1" == "cp37-cp37m" ]; then
                export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH}
                export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH}
160
                export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7
161 162
            -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m
            -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so"
163
                pip3.7 uninstall -y protobuf
164
                pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
165
           fi
166 167 168
        else
            pip uninstall -y protobuf
            pip install -r ${PADDLE_ROOT}/python/requirements.txt
169 170
        fi
    fi
M
minqiyang 已提交
171

J
JiabinYang 已提交
172
    if [ "$SYSTEM" == "Darwin" ]; then
173 174
        WITH_DISTRIBUTE=${WITH_DISTRIBUTE:-ON}
        WITH_AVX=${WITH_AVX:-ON}
J
Jiabin Yang 已提交
175
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-~/.cache/inference_demo}
J
JiabinYang 已提交
176
    else
J
Jiabin Yang 已提交
177
        INFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR:-/root/.cache/inference_demo}
178
    fi
M
minqiyang 已提交
179

G
gongweibao 已提交
180 181 182
    distibuted_flag=${WITH_DISTRIBUTE:-OFF}
    grpc_flag=${WITH_GRPC:-${distibuted_flag}}

183 184 185 186 187 188 189 190
    cat <<EOF
    ========================================
    Configuring cmake in /paddle/build ...
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
        ${PYTHON_FLAGS}
        -DWITH_DSO=ON
        -DWITH_GPU=${WITH_GPU:-OFF}
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF}
G
gongweibao 已提交
191
        -DWITH_DISTRIBUTE=${distibuted_flag}
192
        -DWITH_MKL=${WITH_MKL:-ON}
B
baojun-nervana 已提交
193
        -DWITH_NGRAPH=${WITH_NGRAPH:-OFF}
194 195 196 197 198 199 200 201 202 203 204
        -DWITH_AVX=${WITH_AVX:-OFF}
        -DWITH_GOLANG=${WITH_GOLANG:-OFF}
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All}
        -DWITH_PYTHON=${WITH_PYTHON:-ON}
        -DCUDNN_ROOT=/usr/
        -DWITH_TESTING=${WITH_TESTING:-ON}
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON}
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON}
205
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR}
206
        -DWITH_ANAKIN=${WITH_ANAKIN:-OFF}
207 208
        -DANAKIN_BUILD_FAT_BIN=${ANAKIN_BUILD_FAT_BIN:OFF}
        -DANAKIN_BUILD_CROSS_PLANTFORM=${ANAKIN_BUILD_CROSS_PLANTFORM:ON}
209
        -DPY_VERSION=${PY_VERSION:-2.7}
W
Wu Yi 已提交
210
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build}
G
gongweibao 已提交
211 212
        -DWITH_JEMALLOC=${WITH_JEMALLOC:-OFF} 
        -DWITH_GRPC=${grpc_flag}
213
    ========================================
214
EOF
215 216 217 218 219 220 221 222 223
    # Disable UNITTEST_USE_VIRTUALENV in docker because
    # docker environment is fully controlled by this script.
    # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
    cmake .. \
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
        ${PYTHON_FLAGS} \
        -DWITH_DSO=ON \
        -DWITH_GPU=${WITH_GPU:-OFF} \
        -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} \
G
gongweibao 已提交
224
        -DWITH_DISTRIBUTE=${distibuted_flag} \
225
        -DWITH_MKL=${WITH_MKL:-ON} \
B
baojun-nervana 已提交
226
        -DWITH_NGRAPH=${WITH_NGRAPH:-OFF} \
227 228 229 230 231 232 233 234 235 236
        -DWITH_AVX=${WITH_AVX:-OFF} \
        -DWITH_GOLANG=${WITH_GOLANG:-OFF} \
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \
        -DWITH_PYTHON=${WITH_PYTHON:-ON} \
        -DCUDNN_ROOT=/usr/ \
        -DWITH_TESTING=${WITH_TESTING:-ON} \
        -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake \
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
        -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \
        -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON} \
J
Jiabin Yang 已提交
237
        -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR} \
238
        -DWITH_ANAKIN=${WITH_ANAKIN:-OFF} \
239 240
        -DANAKIN_BUILD_FAT_BIN=${ANAKIN_BUILD_FAT_BIN:OFF}\
        -DANAKIN_BUILD_CROSS_PLANTFORM=${ANAKIN_BUILD_CROSS_PLANTFORM:ON}\
W
Wu Yi 已提交
241
        -DPY_VERSION=${PY_VERSION:-2.7} \
M
minqiyang 已提交
242
        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/paddle/build} \
G
gongweibao 已提交
243 244
        -DWITH_JEMALLOC=${WITH_JEMALLOC:-OFF} \
        -DWITH_GRPC=${grpc_flag}
J
JiabinYang 已提交
245

246 247 248 249 250 251 252 253 254 255 256 257
}

function abort(){
    echo "Your change doesn't follow PaddlePaddle's code style." 1>&2
    echo "Please use pre-commit to check what is wrong." 1>&2
    exit 1
}

function check_style() {
    trap 'abort' 0
    set -e

258 259 260
    if [ -x "$(command -v gimme)" ]; then
    	eval "$(GIMME_GO_VERSION=1.8.3 gimme)"
    fi
261 262 263 264

    # set up go environment for running gometalinter
    mkdir -p $GOPATH/src/github.com/PaddlePaddle/
    ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle
265 266 267
    mkdir -p ./build/go
    cp go/glide.* build/go
    cd build/go; glide install; cd -
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293

    export PATH=/usr/bin:$PATH
    pre-commit install
    clang-format --version

    if ! pre-commit run -a ; then
        git diff
        exit 1
    fi

    trap : 0
}

#=================================================
#              Build
#=================================================

function build() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
    make clean
294
    make -j `nproc`
D
dzhwinter 已提交
295
    make install -j `nproc`
296 297
}

298 299 300 301 302 303 304 305 306
function build_mac() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ============================================
    Building in /paddle/build ...
    ============================================
EOF
    make clean
J
JiabinYang 已提交
307 308
    make -j 8
    make install -j 8
309 310
}

311 312 313 314 315 316 317 318 319
function run_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
320 321 322 323 324
        if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
            ctest -V
        else
            ctest --output-on-failure
        fi
325 326 327
    fi
}

G
gongweibao 已提交
328 329 330 331 332 333 334 335 336 337 338 339
function run_brpc_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [[ ${WITH_TESTING:-ON} == "ON" \
        && ${WITH_DISTRIBUTE:-OFF} == "ON" \
        && ${WITH_GRPC:-OFF} == "OFF" ]] ; then
    cat <<EOF
    ========================================
    Running brpc unit tests ...
    ========================================
EOF
        set +x
340 341
        declare -a other_tests=("test_listen_and_serv_op" "system_allocator_test" \
        "rpc_server_test" "varhandle_test" "collective_server_test" "brpc_serde_test")
G
gongweibao 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
        all_tests=`ctest -N`

        for t in "${other_tests[@]}"
        do
            if [[ ${all_tests} != *$t* ]]; then
                continue
            fi

            if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
                ctest -V -R $t
            else
                ctest --output-on-failure -R $t
            fi
        done
        set -x

        if [[ ${TESTING_DEBUG_MODE:-OFF} == "ON" ]] ; then
            ctest -V -R test_dist_*
        else
            ctest --output-on-failure -R test_dist_*
        fi
    fi
}



368 369 370 371 372 373 374 375 376
function run_mac_test() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running unit tests ...
    ========================================
EOF
J
JiabinYang 已提交
377 378 379
        #remove proxy here to fix dist error on mac
        export http_proxy=
        export https_proxy=
380
        # TODO: jiabin need to refine this part when these tests fixed on mac
L
lujun 已提交
381
        ctest --output-on-failure -j $2
M
minqiyang 已提交
382
        # make install should also be test when unittest
383
        make install -j 8
384
        if [ "$1" == "cp27-cp27m" ]; then
385
            set -e
386
            pip install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
J
JiabinYang 已提交
387
            python ${PADDLE_ROOT}/paddle/scripts/installation_validate.py
388 389 390 391 392 393 394
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
        fi
M
minqiyang 已提交
395

T
Tao Luo 已提交
396
        paddle version
397 398 399 400 401 402 403 404 405 406

        if [ "$1" == "cp27-cp27m" ]; then
            pip uninstall -y paddlepaddle
        elif [ "$1" == "cp35-cp35m" ]; then
            pip3.5 uninstall -y paddlepaddle
        elif [ "$1" == "cp36-cp36m" ]; then
            pip3.6 uninstall -y paddlepaddle
        elif [ "$1" == "cp37-cp37m" ]; then
            pip3.7 uninstall -y paddlepaddle
        fi
407 408 409
    fi
}

Y
yuyang18 已提交
410 411 412 413 414 415
function assert_api_not_changed() {
    mkdir -p ${PADDLE_ROOT}/build/.check_api_workspace
    cd ${PADDLE_ROOT}/build/.check_api_workspace
    virtualenv .env
    source .env/bin/activate
    pip install ${PADDLE_ROOT}/build/python/dist/*whl
X
Xin Pan 已提交
416
    python ${PADDLE_ROOT}/tools/print_signatures.py paddle.fluid,paddle.reader > new.spec
417
    if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then
M
minqiyang 已提交
418 419 420
        # Use sed to make python2 and python3 sepc keeps the same
        sed -i 's/arg0: str/arg0: unicode/g' new.spec
        sed -i "s/\(.*Transpiler.*\).__init__ ArgSpec(args=\['self'].*/\1.__init__ /g" new.spec
M
minqiyang 已提交
421
    fi
X
fix  
Xin Pan 已提交
422 423 424
    # ComposeNotAligned has significant difference between py2 and py3
    sed -i '/.*ComposeNotAligned.*/d' new.spec

425
    python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API.spec new.spec
Y
yuyang18 已提交
426
    deactivate
427 428 429 430 431 432
}

function assert_api_spec_approvals() {
    if [ -z ${BRANCH} ]; then
        BRANCH="develop"
    fi
433

T
tianshuo78520a 已提交
434 435
    API_FILES=("cmake/external"
               "paddle/fluid/API.spec"
X
Xin Pan 已提交
436 437 438 439 440 441 442 443 444
               "paddle/fluid/framework/operator.h"
               "paddle/fluid/framework/tensor.h"
               "paddle/fluid/framework/lod_tensor.h"
               "paddle/fluid/framework/selected_rows.h"
               "paddle/fluid/framework/op_desc.h"
               "paddle/fluid/framework/block_desc.h"
               "paddle/fluid/framework/var_desc.h"
               "paddle/fluid/framework/scope.h"
               "paddle/fluid/framework/ir/node.h"
X
Xin Pan 已提交
445 446 447
               "paddle/fluid/framework/ir/graph.h"
               "paddle/fluid/framework/framework.proto"
               "paddle/fluid/operators/distributed/send_recv.proto.in")
X
fix  
Xin Pan 已提交
448
    for API_FILE in ${API_FILES[*]}; do
X
Xin Pan 已提交
449 450 451 452 453
      API_CHANGE=`git diff --name-only upstream/$BRANCH | grep "${API_FILE}" || true`
      echo "checking ${API_FILE} change, PR: ${GIT_PR_ID}, changes: ${API_CHANGE}"
      if [ ${API_CHANGE} ] && [ "${GIT_PR_ID}" != "" ]; then
          # NOTE: per_page=10000 should be ok for all cases, a PR review > 10000 is not human readable.
          APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \
X
Xin Pan 已提交
454
          python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 2887803`
X
Xin Pan 已提交
455 456
          echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
          if [ "${APPROVALS}" == "FALSE" ]; then
X
Xin Pan 已提交
457 458
              echo "You must have panyx0718 approval for the api change! ${API_FILE}"
              exit 1
X
Xin Pan 已提交
459 460 461
          fi
      fi
    done
462

X
fix  
Xin Pan 已提交
463
    HAS_CONST_CAST=`git diff -U0 upstream/$BRANCH |grep -o -m 1 "const_cast" || true`
464 465
    if [ ${HAS_CONST_CAST} ] && [ "${GIT_PR_ID}" != "" ]; then
        APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \
X
Xin Pan 已提交
466
        python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 2887803`
467 468
        echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
        if [ "${APPROVALS}" == "FALSE" ]; then
X
Xin Pan 已提交
469 470
            echo "You must have panyx0718 approval for the const_cast"
            exit 1
471 472 473
        fi
    fi

474 475 476 477 478 479 480 481 482 483 484 485
    pip install ${PADDLE_ROOT}/build/opt/paddle/share/wheels/*.whl
    CHECK_DOCK_MD5=`python ${PADDLE_ROOT}/tools/check_doc_approval.py`
    if [ "True" != ${CHECK_DOCK_MD5} ]; then
        APPROVALS=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000 | \
        python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 35982308`
        echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
        if [ "${APPROVALS}" == "FALSE" ]; then
            echo "You must have shanyi15 approval for the api doc change! "
            exit 1
        fi
        echo ${CHECK_DOCK_MD5} >/root/.cache/doc_md5.txt
    fi
Y
yuyang18 已提交
486 487 488
}


489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
function single_test() {
    TEST_NAME=$1
    if [ -z "${TEST_NAME}" ]; then
        echo -e "${RED}Usage:${NONE}"
        echo -e "${BOLD}${SCRIPT_NAME}${NONE} ${BLUE}single_test${NONE} [test_name]"
        exit 1
    fi
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    if [ ${WITH_TESTING:-ON} == "ON" ] ; then
    cat <<EOF
    ========================================
    Running ${TEST_NAME} ...
    ========================================
EOF
        ctest --output-on-failure -R ${TEST_NAME}
    fi
}

508 509 510
function bind_test() {
    # the number of process to run tests
    NUM_PROC=6
Q
Qiyang Min 已提交
511

512 513 514
    # calculate and set the memory usage for each process
    MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
    export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
Q
Qiyang Min 已提交
515

516 517
    # get the CUDA device count
    CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
Q
Qiyang Min 已提交
518

519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
    for (( i = 0; i < $NUM_PROC; i++ )); do
        cuda_list=()
        for (( j = 0; j < $CUDA_DEVICE_COUNT; j++ )); do
            s=$[i+j]
            n=$[s%CUDA_DEVICE_COUNT]
            if [ $j -eq 0 ]; then
                cuda_list=("$n")
            else
                cuda_list="$cuda_list,$n"
            fi
        done
        echo $cuda_list
        # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
        # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
        env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC --output-on-failure &
    done
    wait
}

538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
function gen_doc_lib() {
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
    cat <<EOF
    ========================================
    Building documentation library ...
    In /paddle/build
    ========================================
EOF
    cmake .. \
        -DCMAKE_BUILD_TYPE=Release \
        -DWITH_GPU=OFF \
        -DWITH_MKL=OFF \

    local LIB_TYPE=$1
    case $LIB_TYPE in
      full)
        # Build full Paddle Python module. Will timeout without caching 'copy_paddle_pybind' first
556
        make -j `nproc` framework_py_proto copy_paddle_pybind paddle_python
557 558 559 560 561 562 563 564 565 566 567 568 569
        ;;
      pybind)
        # Build paddle pybind library. Takes 49 minutes to build. Might timeout
        make -j `nproc` copy_paddle_pybind
        ;;
      proto)
        # Even smaller library.
        make -j `nproc` framework_py_proto
        ;;
      *)
        exit 0
        ;;
      esac
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
}

function gen_html() {
    cat <<EOF
    ========================================
    Converting C++ source code into HTML ...
    ========================================
EOF
    export WOBOQ_OUT=${PADDLE_ROOT}/build/woboq_out
    mkdir -p $WOBOQ_OUT
    cp -rv /woboq/data $WOBOQ_OUT/../data
    /woboq/generator/codebrowser_generator \
    	-b ${PADDLE_ROOT}/build \
    	-a \
    	-o $WOBOQ_OUT \
    	-p paddle:${PADDLE_ROOT}
    /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
}

function gen_dockerfile() {
    # Set BASE_IMAGE according to env variables
591 592
    CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1).$(echo $CUDA_VERSION | cut -d '.' -f 2)"
    CUDNN_MAJOR=$(echo $CUDNN_VERSION | cut -d '.' -f 1)
593
    if [[ ${WITH_GPU} == "ON" ]]; then
594
        BASE_IMAGE="nvidia/cuda:${CUDA_MAJOR}-cudnn${CUDNN_MAJOR}-runtime-ubuntu16.04"
595
    else
596
        BASE_IMAGE="ubuntu:16.04"
597 598 599 600
    fi

    DOCKERFILE_GPU_ENV=""
    DOCKERFILE_CUDNN_DSO=""
X
Xi Chen 已提交
601
    DOCKERFILE_CUBLAS_DSO=""
602 603
    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:\${LD_LIBRARY_PATH}"
X
Xi Chen 已提交
604
        DOCKERFILE_CUDNN_DSO="RUN ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.${CUDNN_MAJOR} /usr/lib/x86_64-linux-gnu/libcudnn.so"
X
Xi Chen 已提交
605
        DOCKERFILE_CUBLAS_DSO="RUN ln -sf /usr/local/cuda/targets/x86_64-linux/lib/libcublas.so.${CUDA_MAJOR} /usr/lib/x86_64-linux-gnu/libcublas.so"
606 607 608 609
    fi

    cat <<EOF
    ========================================
610
    Generate ${PADDLE_ROOT}/build/Dockerfile ...
611 612 613 614 615 616 617 618 619 620
    ========================================
EOF

    cat > ${PADDLE_ROOT}/build/Dockerfile <<EOF
    FROM ${BASE_IMAGE}
    MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
    ENV HOME /root
EOF

    if [[ ${WITH_GPU} == "ON"  ]]; then
621
        NCCL_DEPS="apt-get install -y --allow-downgrades libnccl2=2.2.13-1+cuda${CUDA_MAJOR} libnccl-dev=2.2.13-1+cuda${CUDA_MAJOR} || true"
622
    else
623
        NCCL_DEPS="true"
624 625
    fi

T
Tao Luo 已提交
626 627
    PADDLE_VERSION="paddle version"
    CMD='"paddle", "version"'
628

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
    if [ "$1" == "cp35-cp35m" ]; then
        cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
    ADD python/dist/*.whl /
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python3 python3-pip libgtk2.0-dev dmidecode python3-tk && \
        pip3 install opencv-python && pip3 install /*.whl; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f /*.whl && \
        ${PADDLE_VERSION} && \
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_CUBLAS_DSO}
    ${DOCKERFILE_GPU_ENV}
    ENV NCCL_LAUNCH_MODE PARALLEL
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
EOF
    elif [ "$1" == "cp36-cp36m" ]; then
        cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
    ADD python/dist/*.whl /
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \
        tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \
        ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \
        wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \
        tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
        make -j8 > /dev/null && make altinstall > /dev/null
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \
        pip3.6 install opencv-python && pip3.6 install /*.whl; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f /*.whl && \
        ${PADDLE_VERSION} && \
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_CUBLAS_DSO}
    ${DOCKERFILE_GPU_ENV}
    ENV NCCL_LAUNCH_MODE PARALLEL
EOF
    elif [ "$1" == "cp37-cp37m" ]; then
        cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
    ADD python/dist/*.whl /
    # run paddle version to install python packages first
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
        libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
        xz-utils tk-dev libffi-dev liblzma-dev
    RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \
        tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \
        CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
        make -j8 > /dev/null && make altinstall > /dev/null
    RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \
        pip3.7 install opencv-python && pip3.7 install /*.whl; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f /*.whl && \
        ${PADDLE_VERSION} && \
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_CUBLAS_DSO}
    ${DOCKERFILE_GPU_ENV}
    ENV NCCL_LAUNCH_MODE PARALLEL
693 694 695
EOF
    else
        cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
696 697
    ADD python/dist/*.whl /
    # run paddle version to install python packages first
698 699
    RUN apt-get update && ${NCCL_DEPS}
    RUN apt-get install -y wget python-pip python-opencv libgtk2.0-dev dmidecode python-tk && easy_install -U pip && \
700 701 702 703 704 705
        pip install /*.whl; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f /*.whl && \
        ${PADDLE_VERSION} && \
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
X
Xi Chen 已提交
706
    ${DOCKERFILE_CUBLAS_DSO}
707 708
    ${DOCKERFILE_GPU_ENV}
    ENV NCCL_LAUNCH_MODE PARALLEL
709
EOF
710 711
    fi

712
    cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
713 714 715 716 717
    # default command shows the paddle version and exit
    CMD [${CMD}]
EOF
}

718
function gen_fluid_lib() {
L
Lei Wang 已提交
719 720
    mkdir -p ${PADDLE_ROOT}/build
    cd ${PADDLE_ROOT}/build
T
Tao Luo 已提交
721
    cat <<EOF
722
    ========================================
723
    Generating fluid library for train and inference ...
724 725
    ========================================
EOF
T
Tao Luo 已提交
726 727 728
    cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON
    make -j `nproc` fluid_lib_dist
    make -j `nproc` inference_lib_dist
T
Tao Luo 已提交
729 730
}

731
function tar_fluid_lib() {
T
Tao Luo 已提交
732
    cat <<EOF
T
Tao Luo 已提交
733
    ========================================
734
    Taring fluid library for train and inference ...
T
Tao Luo 已提交
735 736
    ========================================
EOF
T
Tao Luo 已提交
737 738 739 740 741
    cd ${PADDLE_ROOT}/build
    cp -r fluid_install_dir fluid
    tar -czf fluid.tgz fluid
    cp -r fluid_inference_install_dir fluid_inference
    tar -czf fluid_inference.tgz fluid_inference
742 743
}

744
function test_fluid_lib() {
T
Tao Luo 已提交
745
    cat <<EOF
746
    ========================================
747
    Testing fluid library for inference ...
748 749
    ========================================
EOF
T
Tao Luo 已提交
750 751 752 753 754
    cd ${PADDLE_ROOT}/paddle/fluid/inference/api/demo_ci
    ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON} ${WITH_GPU:-OFF} ${INFERENCE_DEMO_INSTALL_DIR} \
             ${TENSORRT_INCLUDE_DIR:-/usr/local/TensorRT/include} \
             ${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib}
    ./clean.sh
755 756
}

757 758 759 760 761 762 763
function main() {
    local CMD=$1
    init
    case $CMD in
      build)
        cmake_gen ${PYTHON_ABI:-""}
        build
764
        gen_dockerfile ${PYTHON_ABI:-""}
765 766 767 768
        ;;
      test)
        run_test
        ;;
769 770 771
      single_test)
        single_test $2
        ;;
772 773 774
      bind_test)
        bind_test
        ;;
775 776 777
      gen_doc_lib)
        gen_doc_lib $2
        ;;
778 779 780 781
      html)
        gen_html
        ;;
      dockerfile)
782
        gen_dockerfile ${PYTHON_ABI:-""}
783 784 785
        ;;
      fluid_inference_lib)
        cmake_gen ${PYTHON_ABI:-""}
786 787 788
        gen_fluid_lib
        tar_fluid_lib
        test_fluid_lib
789 790 791 792
        ;;
      check_style)
        check_style
        ;;
L
Lei Wang 已提交
793 794 795
      cicheck)
        cmake_gen ${PYTHON_ABI:-""}
        build
796
        assert_api_not_changed ${PYTHON_ABI:-""}
L
Lei Wang 已提交
797
        run_test
798 799
        gen_fluid_lib
        test_fluid_lib
T
tianshuo78520a 已提交
800
        assert_api_spec_approvals
L
Lei Wang 已提交
801
        ;;
G
gongweibao 已提交
802 803 804 805 806
      cicheck_brpc)
        cmake_gen ${PYTHON_ABI:-""}
        build
        run_brpc_test
        ;;
807 808
      assert_api)
        assert_api_not_changed ${PYTHON_ABI:-""}
Y
Yu Yang 已提交
809
        assert_api_spec_approvals
810 811 812 813 814 815 816 817
        ;;
      test_inference)
        gen_fluid_lib
        test_fluid_lib
        ;;
      assert_api_approvals)
        assert_api_spec_approvals
        ;;
818 819 820
      maccheck)
        cmake_gen ${PYTHON_ABI:-""}
        build_mac
L
lujun 已提交
821
        run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
822
        ;;
J
JiabinYang 已提交
823 824 825 826
      macbuild)
        cmake_gen ${PYTHON_ABI:-""}
        build_mac
        ;;
M
minqiyang 已提交
827 828 829 830
      cicheck_py35)
        cmake_gen ${PYTHON_ABI:-""}
        build
        run_test
T
typhoonzero 已提交
831
        assert_api_not_changed ${PYTHON_ABI:-""}
M
minqiyang 已提交
832
        ;;
Y
Yu Yang 已提交
833 834 835 836 837 838 839 840 841
      cmake_gen)
        cmake_gen ${PYTHON_ABI:-""}
        ;;
      gen_fluid_lib)
        gen_fluid_lib
        ;;
      test_fluid_lib)
        test_fluid_lib
        ;;
842 843 844 845 846 847 848 849
      *)
        print_usage
        exit 0
        ;;
      esac
}

main $@