build.sh 7.6 KB
Newer Older
Z
zhangjinchao01 已提交
1 2
#!/bin/bash

T
typhoonzero 已提交
3 4 5 6 7 8 9 10
function cmake_gen() {
    mkdir -p /paddle/build
    cd /paddle/build

    # build script will not fail if *.deb does not exist
    rm *.deb 2>/dev/null || true
    # delete previous built whl packages
    rm -rf /paddle/paddle/dist 2>/dev/null || true
Y
Yi Wang 已提交
11

T
typhoonzero 已提交
12 13 14 15 16 17 18
    # Support build for all python versions, currently
    # including cp27-cp27m and cp27-cp27mu.
    PYTHON_FLAGS=""
    if [ "$1" != "" ]; then
        echo "using python abi: $1"
        if [ "$1" == "cp27-cp27m" ]; then
            export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:}
Y
Yancey 已提交
19
            export PATH=/opt/python/cp27-cp27m/bin/:${PATH}
T
typhoonzero 已提交
20 21 22 23 24
            PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
        -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
        -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so"
        elif [ "$1" == "cp27-cp27mu" ]; then
            export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:}
Y
Yancey 已提交
25
            export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
T
typhoonzero 已提交
26 27 28 29 30 31
            PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
        -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
        -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so"
        fi
    fi

T
typhoonzero 已提交
32 33 34
    cat <<EOF
    ========================================
    Configuring cmake in /paddle/build ...
Y
Yancey1989 已提交
35
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
T
typhoonzero 已提交
36
        ${PYTHON_FLAGS}
Y
Yang Yang(Tony) 已提交
37
        -DWITH_DSO=ON
T
typhoonzero 已提交
38 39
        -DWITH_DOC=OFF
        -DWITH_GPU=${WITH_GPU:-OFF}
T
typhoonzero 已提交
40
        -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF}
41
        -DWITH_MKL=${WITH_MKL:-ON}
T
typhoonzero 已提交
42
        -DWITH_AVX=${WITH_AVX:-OFF}
43
        -DWITH_GOLANG=${WITH_GOLANG:-OFF}
44
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All}
T
typhoonzero 已提交
45 46 47 48 49 50 51
        -DWITH_SWIG_PY=ON
        -DWITH_C_API=${WITH_C_API:-OFF}
        -DWITH_PYTHON=${WITH_PYTHON:-ON}
        -DWITH_SWIG_PY=${WITH_SWIG_PY:-ON}
        -DCUDNN_ROOT=/usr/
        -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-ON}
        -DWITH_TESTING=${WITH_TESTING:-ON}
52
        -DWITH_FAST_BUNDLE_TEST=ON
T
typhoonzero 已提交
53 54
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
    ========================================
Y
Yi Wang 已提交
55
EOF
T
typhoonzero 已提交
56 57 58 59
    # Disable UNITTEST_USE_VIRTUALENV in docker because
    # docker environment is fully controlled by this script.
    # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
    cmake .. \
Y
Yancey1989 已提交
60
        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
T
typhoonzero 已提交
61
        ${PYTHON_FLAGS} \
Y
Yang Yang(Tony) 已提交
62
        -DWITH_DSO=ON \
T
typhoonzero 已提交
63 64
        -DWITH_DOC=OFF \
        -DWITH_GPU=${WITH_GPU:-OFF} \
T
typhoonzero 已提交
65
        -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF} \
66
        -DWITH_MKL=${WITH_MKL:-ON} \
T
typhoonzero 已提交
67
        -DWITH_AVX=${WITH_AVX:-OFF} \
68
        -DWITH_GOLANG=${WITH_GOLANG:-OFF} \
69
        -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \
T
typhoonzero 已提交
70 71 72 73 74 75
        -DWITH_SWIG_PY=${WITH_SWIG_PY:-ON} \
        -DWITH_C_API=${WITH_C_API:-OFF} \
        -DWITH_PYTHON=${WITH_PYTHON:-ON} \
        -DCUDNN_ROOT=/usr/ \
        -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-ON} \
        -DWITH_TESTING=${WITH_TESTING:-ON} \
76
        -DWITH_FAST_BUNDLE_TEST=ON \
T
typhoonzero 已提交
77 78 79 80
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
}

function run_build() {
Y
Yi Wang 已提交
81
    cat <<EOF
T
typhoonzero 已提交
82 83 84
    ============================================
    Building in /paddle/build ...
    ============================================
Y
Yi Wang 已提交
85
EOF
L
Luo Tao 已提交
86
    make clean
T
typhoonzero 已提交
87 88 89 90 91
    make -j `nproc`
}

function run_test() {
    if [ ${WITH_TESTING:-ON} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
Y
Yi Wang 已提交
92
    cat <<EOF
T
typhoonzero 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
    ========================================
    Running unit tests ...
    ========================================
EOF
        ctest --output-on-failure
        # make install should also be test when unittest
        make install -j `nproc`
        pip install /usr/local/opt/paddle/share/wheels/*.whl
        paddle version
    fi
}


function gen_docs() {
    if [[ ${WITH_DOC:-OFF} == "ON" ]]; then
        cat <<EOF
    ========================================
    Building documentation ...
    In /paddle/build_doc
    ========================================
Y
Yi Wang 已提交
113
EOF
T
typhoonzero 已提交
114 115 116 117 118 119 120 121
        mkdir -p /paddle/build_doc
        pushd /paddle/build_doc
        cmake .. \
            -DWITH_DOC=ON \
            -DWITH_GPU=OFF \
            -DWITH_AVX=${WITH_AVX:-ON} \
            -DWITH_SWIG_PY=ON \
            -DWITH_STYLE_CHECK=OFF
122
        make -j `nproc` gen_proto_py framework_py_proto
L
Luo Tao 已提交
123
        make -j `nproc` copy_paddle_pybind
L
Luo Tao 已提交
124
        make -j `nproc` paddle_docs paddle_docs_cn paddle_api_docs
T
typhoonzero 已提交
125 126 127 128 129 130 131 132 133
        popd
    fi


    if [[ ${WOBOQ:-OFF} == 'ON' ]]; then
        cat <<EOF
    ========================================
    Converting C++ source code into HTML ...
    ========================================
Y
Yi Wang 已提交
134
EOF
T
typhoonzero 已提交
135 136 137 138 139 140 141 142 143 144 145
        export WOBOQ_OUT=/paddle/build/woboq_out
        mkdir -p $WOBOQ_OUT
        cp -rv /woboq/data $WOBOQ_OUT/../data
        /woboq/generator/codebrowser_generator \
            -b /paddle/build \
            -a \
            -o $WOBOQ_OUT \
            -p paddle:/paddle
        /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
    fi
}
Y
Yi Wang 已提交
146

T
typhoonzero 已提交
147 148

function gen_dockerfile() {
T
typhoonzero 已提交
149 150 151 152 153 154 155 156 157 158
    # Set BASE_IMAGE according to env variables
    if [[ ${WITH_GPU} == "ON" ]]; then
    BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04"
    else
    BASE_IMAGE="ubuntu:16.04"
    fi

    DOCKERFILE_GPU_ENV=""
    DOCKERFILE_CUDNN_DSO=""
    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
Y
Yancey 已提交
159
        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:\${LD_LIBRARY_PATH}"
T
typhoonzero 已提交
160 161
        DOCKERFILE_CUDNN_DSO="RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/lib/x86_64-linux-gnu/libcudnn.so"
    fi
T
typhoonzero 已提交
162 163 164 165 166

    cat <<EOF
    ========================================
    Generate /paddle/build/Dockerfile ...
    ========================================
R
reyoung 已提交
167 168
EOF

T
typhoonzero 已提交
169 170 171 172
    cat > /paddle/build/Dockerfile <<EOF
    FROM ${BASE_IMAGE}
    MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
    ENV HOME /root
R
reyoung 已提交
173
EOF
T
typhoonzero 已提交
174 175

    if [[ ${WITH_GPU} == "ON"  ]]; then
Q
QI JUN 已提交
176
        NCCL_DEPS="apt-get install -y libnccl2=2.1.2-1+cuda8.0 libnccl-dev=2.1.2-1+cuda8.0 &&"
T
typhoonzero 已提交
177
    else
Y
Yang Yang(Tony) 已提交
178
        NCCL_DEPS=""
T
typhoonzero 已提交
179 180 181 182 183 184 185
    fi

    cat >> /paddle/build/Dockerfile <<EOF
    ADD python/dist/*.whl /
    # run paddle version to install python packages first
    RUN apt-get update &&\
        ${NCCL_DEPS}\
Y
Yancey 已提交
186
        apt-get install -y wget python-pip dmidecode python-tk && pip install -U pip && \
T
typhoonzero 已提交
187 188 189 190 191 192 193
        pip install /*.whl; apt-get install -f -y && \
        apt-get clean -y && \
        rm -f /*.whl && \
        paddle version && \
        ldconfig
    ${DOCKERFILE_CUDNN_DSO}
    ${DOCKERFILE_GPU_ENV}
Y
Yang Yang(Tony) 已提交
194
    ENV NCCL_LAUNCH_MODE PARALLEL
T
typhoonzero 已提交
195 196 197 198 199 200
    ADD go/cmd/pserver/pserver /usr/bin/
    ADD go/cmd/master/master /usr/bin/
    # default command shows the paddle version and exit
    CMD ["paddle", "version"]
EOF
}
L
Luo Tao 已提交
201

Y
Yancey 已提交
202 203 204 205 206 207
function gen_capi_package() {
  if [[ ${WITH_C_API} == "ON" ]]; then
    install_prefix="/paddle/build/capi_output"
    rm -rf $install_prefix
    make DESTDIR="$install_prefix" install
    cd $install_prefix/usr/local
208
    ls | egrep -v "^Found.*item$" | xargs tar -cf /paddle/build/paddle.tgz
Y
Yancey 已提交
209 210 211
  fi
}

212 213 214 215 216 217 218 219 220 221 222
function gen_fluid_inference_lib() {
    if [ ${WITH_C_API:-OFF} == "OFF" ] ; then
    cat <<EOF
    ========================================
    Building fluid inference library ...
    ========================================
EOF
        make inference_lib_dist
    fi
}

T
typhoonzero 已提交
223
set -xe
T
typhoonzero 已提交
224

T
typhoonzero 已提交
225
cmake_gen ${PYTHON_ABI:-""}
T
typhoonzero 已提交
226 227 228 229
run_build
run_test
gen_docs
gen_dockerfile
Y
Yancey 已提交
230
gen_capi_package
231
gen_fluid_inference_lib
Y
Yancey 已提交
232 233 234 235 236 237 238

if [[ ${WITH_C_API:-OFF} == "ON" ]]; then
  printf "PaddlePaddle C-API libraries was generated on build/paddle.tgz\n" 
else
  printf "If you need to install PaddlePaddle in develop docker image,"
  printf "please make install or pip install build/python/dist/*.whl.\n"
fi