Merge pull request #5396 from typhoonzero/develop

Put files to platlib

paddle/scripts/docker/build.sh

@@ -2,31 +2,33 @@
 
 set -xe
 
-# Set BASE_IMAGE according to env variables
-if [[ ${WITH_GPU} == "ON" ]]; then
+
+function cmake_gen() {
+    # Set BASE_IMAGE according to env variables
+    if [[ ${WITH_GPU} == "ON" ]]; then
     BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04"
-else
+    else
     BASE_IMAGE="ubuntu:16.04"
-fi
+    fi
 
-DOCKERFILE_GPU_ENV=""
-DOCKERFILE_CUDNN_DSO=""
-if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
+    DOCKERFILE_GPU_ENV=""
+    DOCKERFILE_CUDNN_DSO=""
+    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
         DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}"
         DOCKERFILE_CUDNN_DSO="RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/lib/x86_64-linux-gnu/libcudnn.so"
-fi
+    fi
 
-mkdir -p /paddle/build
-cd /paddle/build
+    mkdir -p /paddle/build
+    cd /paddle/build
 
-# build script will not fail if *.deb does not exist
-rm *.deb 2>/dev/null || true
-# delete previous built whl packages
-rm -rf /paddle/paddle/dist 2>/dev/null || true
+    # build script will not fail if *.deb does not exist
+    rm *.deb 2>/dev/null || true
+    # delete previous built whl packages
+    rm -rf /paddle/paddle/dist 2>/dev/null || true
 
-cat <<EOF
-========================================
-Configuring cmake in /paddle/build ...
+    cat <<EOF
+    ========================================
+    Configuring cmake in /paddle/build ...
         -DCMAKE_BUILD_TYPE=Release
         -DWITH_DOC=OFF
         -DWITH_GPU=${WITH_GPU:-OFF}
@@ -42,14 +44,17 @@ Configuring cmake in /paddle/build ...
         -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-ON}
         -DWITH_TESTING=${WITH_TESTING:-ON}
         -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-========================================
+    ========================================
 EOF
 
-# Disable UNITTEST_USE_VIRTUALENV in docker because
-# docker environment is fully controlled by this script.
-# See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
-cmake .. \
+    # Disable UNITTEST_USE_VIRTUALENV in docker because
+    # docker environment is fully controlled by this script.
+    # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
+    cmake .. \
         -DCMAKE_BUILD_TYPE=Release \
+        -DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python \
+        -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include \
+        -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so \
         -DWITH_DOC=OFF \
         -DWITH_GPU=${WITH_GPU:-OFF} \
         -DWITH_MKLDNN=${WITH_MKLDNN:-ON} \
@@ -63,34 +68,40 @@ cmake .. \
         -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-ON} \
         -DWITH_TESTING=${WITH_TESTING:-ON} \
         -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+}
 
-cat <<EOF
-============================================
-Building in /paddle/build ...
-============================================
+function run_build() {
+    cat <<EOF
+    ============================================
+    Building in /paddle/build ...
+    ============================================
 EOF
-make -j `nproc`
+    make -j `nproc`
+}
 
-if [ ${WITH_TESTING:-ON} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
-cat <<EOF
-========================================
-Running unit tests ...
-========================================
+function run_test() {
+    if [ ${WITH_TESTING:-ON} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
+    cat <<EOF
+    ========================================
+    Running unit tests ...
+    ========================================
 EOF
         ctest --output-on-failure
         # make install should also be test when unittest
         make install -j `nproc`
         pip install /usr/local/opt/paddle/share/wheels/*.whl
         paddle version
-fi
+    fi
+}
 
 
-if [[ ${WITH_DOC:-OFF} == "ON" ]]; then
+function gen_docs() {
+    if [[ ${WITH_DOC:-OFF} == "ON" ]]; then
         cat <<EOF
-========================================
-Building documentation ...
+    ========================================
+    Building documentation ...
     In /paddle/build_doc
-========================================
+    ========================================
 EOF
         mkdir -p /paddle/build_doc
         pushd /paddle/build_doc
@@ -103,14 +114,14 @@ EOF
         make -j `nproc` gen_proto_py
         make -j `nproc` paddle_docs paddle_docs_cn
         popd
-fi
+    fi
 
 
-if [[ ${WOBOQ:-OFF} == 'ON' ]]; then
+    if [[ ${WOBOQ:-OFF} == 'ON' ]]; then
         cat <<EOF
-========================================
-Converting C++ source code into HTML ...
-========================================
+    ========================================
+    Converting C++ source code into HTML ...
+    ========================================
 EOF
         export WOBOQ_OUT=/paddle/build/woboq_out
         mkdir -p $WOBOQ_OUT
@@ -121,36 +132,34 @@ EOF
             -o $WOBOQ_OUT \
             -p paddle:/paddle
         /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
-fi
+    fi
+}
 
-cat <<EOF
-========================================
-Generate /paddle/build/Dockerfile ...
-========================================
-EOF
 
-cat > /paddle/build/Dockerfile <<EOF
-FROM ${BASE_IMAGE}
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-ENV HOME /root
+function gen_dockerfile() {
+
+    cat <<EOF
+    ========================================
+    Generate /paddle/build/Dockerfile ...
+    ========================================
 EOF
 
-if [[ -n ${APT_MIRROR} ]]; then
-cat >> /paddle/build/Dockerfile <<EOF
-RUN sed -i '${APT_MIRROR}' /etc/apt/sources.list
+    cat > /paddle/build/Dockerfile <<EOF
+    FROM ${BASE_IMAGE}
+    MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+    ENV HOME /root
 EOF
-fi
 
-if [[ ${WITH_GPU} == "ON"  ]]; then
+    if [[ ${WITH_GPU} == "ON"  ]]; then
         NCCL_DEPS="apt-get install -y libnccl-dev &&"
-else
+    else
         NCCL_DEPS="" 
-fi
+    fi
 
-cat >> /paddle/build/Dockerfile <<EOF
-ADD python/dist/*.whl /
-# run paddle version to install python packages first
-RUN apt-get update &&\
+    cat >> /paddle/build/Dockerfile <<EOF
+    ADD python/dist/*.whl /
+    # run paddle version to install python packages first
+    RUN apt-get update &&\
         ${NCCL_DEPS}\
         apt-get install -y wget python-pip && pip install -U pip && \
         pip install /*.whl; apt-get install -f -y && \
@@ -158,15 +167,22 @@ RUN apt-get update &&\
         rm -f /*.whl && \
         paddle version && \
         ldconfig
-${DOCKERFILE_CUDNN_DSO}
-${DOCKERFILE_GPU_ENV}
-ADD go/cmd/pserver/pserver /usr/bin/
-ADD go/cmd/master/master /usr/bin/
-ADD paddle/pybind/print_operators_doc /usr/bin/
-# default command shows the paddle version and exit
-CMD ["paddle", "version"]
+    ${DOCKERFILE_CUDNN_DSO}
+    ${DOCKERFILE_GPU_ENV}
+    ADD go/cmd/pserver/pserver /usr/bin/
+    ADD go/cmd/master/master /usr/bin/
+    # default command shows the paddle version and exit
+    CMD ["paddle", "version"]
 EOF
+}
 
 set +xe
+
+cmake_gen
+run_build
+run_test
+gen_docs
+gen_dockerfile
+
 printf "If you need to install PaddlePaddle in develop docker image,"
 printf "please make install or pip install build/python/dist/*.whl.\n"

python/CMakeLists.txt

@@ -44,6 +44,7 @@ add_custom_target(copy_paddle_pybind ALL DEPENDS ${PADDLE_SOURCE_DIR}/python/pad
 
 
 add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
+    COMMAND touch stub.cc
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
     COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
     COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python

python/setup.py.in

-from setuptools import setup, Distribution
+from setuptools import setup, Distribution, Extension
 class BinaryDistribution(Distribution):
     def has_ext_modules(foo):
         return True
@@ -41,6 +41,7 @@ setup(name='paddlepaddle',
       description='Parallel Distributed Deep Learning',
       install_requires=setup_requires,
       packages=packages,
+      ext_modules=[Extension('_foo', ['stub.cc'])],
       package_data={
         'paddle.v2.master': ['libpaddle_master.so'],
         'paddle.v2.framework': ['core.so'],
@@ -54,6 +55,5 @@ setup(name='paddlepaddle',
           'py_paddle': '${PADDLE_SOURCE_DIR}/paddle/py_paddle'
       },
       scripts=paddle_bins,
-      distclass=BinaryDistribution,
       data_files=[(paddle_rt_lib_dir, paddle_rt_libs)]
 )