Allow users to build a Docker image as development environment from Dockerfile{.gpu}

paddle/scripts/docker/Dockerfile

@@ -13,8 +13,8 @@ RUN apt-get update && \
     apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \
     apt-get clean -y
 
-RUN pip install --upgrade pip && \ 
-    pip install 'protobuf==3.1.0.post1' && \
+RUN pip install --upgrade pip && \
+    pip install -U protobuf && \
     pip install -U wheel pillow BeautifulSoup && \
     pip install -U docopt PyYAML sphinx && \
     pip install -U sphinx_rtd_theme recommonmark jupyter
@@ -23,10 +23,12 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
     cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \
     cd .. && rm -rf cmake-3.4.1
 
+ARG BUILD_AND_INSTALL
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_STYLE_CHECK
 
+ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
 ENV WITH_GPU=OFF
 ENV WITH_AVX=${WITH_AVX:-ON}
 ENV WITH_DOC=${WITH_DOC:-ON}
@@ -37,10 +39,6 @@ COPY . /paddle/
 RUN /paddle/paddle/scripts/docker/build.sh
 VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
 
-RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile
-RUN pip install /usr/local/opt/paddle/share/wheels/*.whl
-RUN paddle version  # print version after build
-
 # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' | chpasswd

paddle/scripts/docker/Dockerfile.dev

-FROM ubuntu:16.04
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG UBUNTU_MIRROR
-RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
-
-RUN apt-get update
-
-# Paddle development tools.
-RUN apt-get install -y coreutils git cmake g++ m4 python-pip swig
-
-# Paddle data tools.
-RUN apt-get install -y wget unzip tar xz-utils bzip2 gzip sed grep graphviz
-
-# Paddle documentation tools.
-RUN apt-get install -y doxygen
-
-# Paddle code auto-reformat tools.
-RUN apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev clang-format-3.8 clang-tidy-3.8
-
-# Paddle prerequisites.
-RUN apt-get install -y \
-    libprotobuf-dev protobuf-compiler \
-    libgoogle-glog-dev libgflags-dev libgtest-dev \
-    libatlas-dev libatlas3-base  \
-    python-protobuf python-numpy python-dev python-matplotlib \
-    libjpeg-dev zlib1g-dev
-
-# Development environment:
-RUN apt-get install -y openssh-server
-
-RUN apt-get clean -y
-
-# Build Google test
-RUN cd /usr/src/gtest && cmake . && make && cp *.a /usr/lib
-
-# Python dependencies.
-RUN pip install -U BeautifulSoup docopt PyYAML pillow \
-    sphinx sphinx_rtd_theme recommonmark jupyter
-
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_SWIG_PY
-ARG WITH_STYLE_CHECK
-
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-ON}
-ENV WITH_SWIG_PY=${WITH_SWIG_PY:-ON}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile
-
-# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
-RUN mkdir /var/run/sshd
-RUN echo 'root:root' | chpasswd
-RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
-EXPOSE 22
-
-CMD ["/usr/sbin/sshd", "-D"]

paddle/scripts/docker/Dockerfile.gpu

@@ -13,8 +13,8 @@ RUN apt-get update && \
     apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \
     apt-get clean -y
 
-RUN pip install --upgrade pip && \ 
-    pip install 'protobuf==3.1.0.post1' && \
+RUN pip install --upgrade pip && \
+    pip install -U protobuf && \
     pip install -U wheel pillow BeautifulSoup && \
     pip install -U docopt PyYAML sphinx && \
     pip install -U sphinx_rtd_theme recommonmark jupyter
@@ -23,10 +23,12 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
     cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \
     cd .. && rm -rf cmake-3.4.1
 
+ARG BUILD_AND_INSTALL
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_STYLE_CHECK
 
+ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
 ENV WITH_GPU=ON
 ENV WITH_AVX=${WITH_AVX:-ON}
 ENV WITH_DOC=${WITH_DOC:-ON}
@@ -37,10 +39,6 @@ COPY . /paddle/
 RUN /paddle/paddle/scripts/docker/build.sh
 VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
 
-RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile
-RUN pip install /usr/local/opt/paddle/share/wheels/*.whl
-RUN paddle version  # print version after build
-
 # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' | chpasswd

paddle/scripts/docker/build.sh

@@ -8,42 +8,50 @@ function abort(){
 trap 'abort' 0
 set -e
 
-if [ ${WITH_GPU} == 'ON' ]; then
-  ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
+# If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied
+# source tree to /paddle, and this scripts should build it into
+# /paddle/build.
+if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then
+    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
+	ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
+    fi
+
+    mkdir -p /paddle/build # -p means no error if exists
+    cd /paddle/build
+    cmake .. \
+	  -DWITH_DOC=ON \
+	  -DWITH_GPU=${WITH_GPU:-OFF} \
+	  -DWITH_AVX=${WITH_AVX:-OFF} \
+	  -DWITH_SWIG_PY=ON \
+	  -DCUDNN_ROOT=/usr/ \
+	  -DWITH_STYLE_CHECK=OFF \
+	  -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+    make -j `nproc`
+    make install
+
+    # Install woboq_codebrowser.
+    git clone https://github.com/woboq/woboq_codebrowser /woboq
+    cd /woboq
+    cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \
+	  -DCMAKE_BUILD_TYPE=Release \
+	  .
+    make
+
+    export WOBOQ_OUT=/usr/share/nginx/html/paddle
+    export BUILD_DIR=/paddle/build
+    mkdir -p $WOBOQ_OUT
+    cp -rv /woboq/data $WOBOQ_OUT/../data
+    /woboq/generator/codebrowser_generator \
+	-b /paddle/build \
+	-a \
+	-o $WOBOQ_OUT \
+	-p paddle:/paddle
+    /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
+    cd /woboq
+    make clean
+
+    pip install /usr/local/opt/paddle/share/wheels/*.whl
+    paddle version
 fi
 
-mkdir -p /paddle/build # -p means no error if exists
-cd /paddle/build
-cmake .. \
-      -DWITH_DOC=ON \
-      -DWITH_GPU=${WITH_GPU} \
-      -DWITH_AVX=${WITH_AVX} \
-      -DWITH_SWIG_PY=ON \
-      -DCUDNN_ROOT=/usr/ \
-      -DWITH_STYLE_CHECK=OFF \
-      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-make -j `nproc`
-make install
-
-# Install woboq_codebrowser.
-git clone https://github.com/woboq/woboq_codebrowser /woboq
-cd /woboq
-cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \
-      -DCMAKE_BUILD_TYPE=Release \
-      .
-make
-
-export WOBOQ_OUT=/usr/share/nginx/html/paddle
-export BUILD_DIR=/paddle/build
-mkdir -p $WOBOQ_OUT
-cp -rv /woboq/data $WOBOQ_OUT/../data
-/woboq/generator/codebrowser_generator \
-    -b /paddle/build \
-    -a \
-    -o $WOBOQ_OUT \
-    -p paddle:/paddle
-/woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
-cd /woboq
-make clean
-rm -rf /paddle/build
 trap : 0