new design: #1625

paddle/scripts/docker/Dockerfile.gpu → Dockerfile

+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
 FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
 MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
 
@@ -14,14 +16,12 @@ ARG WITH_STYLE_CHECK
 
 ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
 ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
-ENV WITH_GPU=ON
+ENV WITH_GPU=OFF
 ENV WITH_AVX=${WITH_AVX:-ON}
 ENV WITH_DOC=${WITH_DOC:-OFF}
 ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-ENV DOCKER_BUILD=TRUE
 
 ENV HOME /root
-
 # Add bash enhancements
 COPY ./paddle/scripts/docker/root/ /root/
 
@@ -50,9 +50,7 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
     cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
     cd .. && rm -rf cmake-3.4.1
 
-COPY . /paddle/
-RUN cd /paddle/ && git submodule update --init --recursive
-RUN /paddle/paddle/scripts/docker/build.sh
+RUN apt-get install -y swig
 
 VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
 
@@ -63,9 +61,5 @@ RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
 RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
 EXPOSE 22
 
-# Jupyter Notebook: Paddle book
-EXPOSE 8888
-
-COPY ./paddle/scripts/docker/entrypoint /opt/bin/
-
-CMD ["/opt/bin/entrypoint"]
+# development image default do build work
+CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]

paddle/scripts/docker/Dockerfile

-FROM ubuntu:14.04
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG UBUNTU_MIRROR
-RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
-
-# ENV variables
-ARG BUILD_WOBOQ
-ARG BUILD_AND_INSTALL
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
-ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-ENV DOCKER_BUILD=TRUE
-
-ENV HOME /root
-
-# Add bash enhancements
-COPY ./paddle/scripts/docker/root/ /root/
-
-RUN apt-get update && \
-    apt-get install -y git python-pip python-dev openssh-server bison && \
-    apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
-    apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
-    apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
-    apt-get install -y automake locales clang-format-3.8 && \
-    apt-get clean -y
-
-# git credential to skip password typing
-RUN git config --global credential.helper store
-
-# Fix locales to en_US.UTF-8
-RUN localedef -i en_US -f UTF-8 en_US.UTF-8
-
-RUN pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0' && \
-    pip install -U wheel pillow BeautifulSoup && \
-    pip install -U docopt PyYAML sphinx && \
-    pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \
-    pip install -U pre-commit 'requests==2.9.2' jupyter
-
-RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
-    cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
-    cd .. && rm -rf cmake-3.4.1
-
-COPY . /paddle/
-RUN cd /paddle/ && git submodule update --init --recursive
-RUN /paddle/paddle/scripts/docker/build.sh
-
-VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
-
-# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
-RUN mkdir /var/run/sshd
-RUN echo 'root:root' | chpasswd
-RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
-EXPOSE 22
-
-# Jupyter Notebook: Paddle book
-EXPOSE 8888
-
-COPY ./paddle/scripts/docker/entrypoint /opt/bin/
-
-CMD ["/opt/bin/entrypoint"]

paddle/scripts/docker/build.sh

@@ -7,7 +7,26 @@ function abort(){
 
 trap 'abort' 0
 set -e
-
+mkdir -p /paddle/dist/cpu
+mkdir -p /paddle/dist/gpu
+mkdir -p /paddle/dist/cpu-noavx
+mkdir -p /paddle/dist/gpu-noavx
+# Set BASE_IMAGE and DEB_PATH according to env variables
+if [ ${WITH_GPU} == "ON" ]; then
+  BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04"
+  if [ ${WITH_AVX} == "ON" ]; then
+    DEB_PATH="dist/gpu/"
+  else
+    DEB_PATH="dist/gpu-noavx/"
+  fi
+else
+  BASE_IMAGE="python:2.7.13-slim"
+  if [ ${WITH_AVX} == "ON" ]; then
+    DEB_PATH="dist/cpu/"
+  else
+    DEB_PATH="dist/cpu-noavx/"
+  fi
+fi
 # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied
 # source tree to /paddle, and this scripts should build it into
 # /paddle/build.
@@ -29,9 +48,13 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
 	  -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
     make -j `nproc`
     make install
+    # generate deb package for current build
+    # FIXME(typhoonzero): should we remove paddle/scripts/deb ?
+    cpack -D CPACK_GENERATOR='DEB' ..
+    mv /paddle/build/*.deb /paddle/${DEB_PATH}
 
     if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then
-        apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev 
+        apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev
         # Install woboq_codebrowser.
         git clone https://github.com/woboq/woboq_codebrowser /woboq
         cd /woboq
@@ -65,4 +88,46 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
     fi
 fi
 
+# generate production docker image Dockerfile
+if [ ${USE_MIRROR} ]; then
+  MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\"
+else
+  MIRROR_UPDATE="\\"
+fi
+
+cat > /paddle/build/Dockerfile <<EOF
+FROM ${BASE_IMAGE}
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV WITH_GPU=${WITH_GPU}
+ENV WITH_AVX=\${WITH_AVX:-ON}
+ENV WITH_DOC=\${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+
+# Use Fix locales to en_US.UTF-8
+
+RUN ${MIRROR_UPDATE}
+    apt-get update && \
+    apt-get install -y libgfortran3 && \
+    apt-get clean -y && \
+    pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0'
+RUN pip install numpy
+# Use different deb file when building different type of images
+ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/
+RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb
+
+ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
+# default command shows the paddle version and exit
+CMD ["paddle", "version"]
+EOF
+
 trap : 0

paddle/scripts/docker/buildall.sh

-#!/bin/bash
-
-BUILD_DIR=$PWD/build
-DEB_DIST_DIR=$PWD/dist
-VERSION=latest
-
-function build_in_docker() {
-  if [ ! -d $BUILD_DIR ]; then
-    mkdir -p $BUILD_DIR
-  fi
-  if [ ! -d $DEB_DIST_DIR ]; then
-    mkdir -p $DEB_DIST_DIR
-  fi
-  docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile
-  # FIXME: need to wait a signal not sleeping
-  BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600)
-  # NOTICE: build deb files for real paddle image
-  docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh"
-
-  docker stop $BUILDER && docker rm $BUILDER
-}
-
-function build_paddle_core() {
-  docker build . -t paddle:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile
-  docker build . -t paddle:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu
-  docker build . -t paddle:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx
-  docker build . -t paddle:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx
-}
-
-build_in_docker
-build_paddle_core

paddle/scripts/docker/paddle-core/Dockerfile

-FROM python:2.7.13-slim
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-# ENV variables
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-ENV HOME /root
-ENV LANG en_US.UTF-8
-
-# Use Fix locales to en_US.UTF-8
-
-RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -y libgfortran3 && \
-    apt-get clean -y && \
-    pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0'
-RUN pip install numpy
-# Use different deb file when building different type of images
-ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/
-RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb
-
-ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
-# default command shows the paddle version and exit
-CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.gpu

-FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-# ENV variables
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-ENV HOME /root
-ENV LANG en_US.UTF-8
-
-# Use Fix locales to en_US.UTF-8
-
-RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -y python python-pip libgfortran3 && \
-    apt-get clean -y && \
-    pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0'
-RUN pip install numpy
-# Use different deb file when building different type of images
-ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/
-RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb
-
-ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
-# default command shows the paddle version and exit
-CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx

-FROM python:2.7.13-slim
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-# ENV variables
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-ENV HOME /root
-ENV LANG en_US.UTF-8
-
-# Use Fix locales to en_US.UTF-8
-
-RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -y libgfortran3 && \
-    apt-get clean -y && \
-    pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0'
-RUN pip install numpy
-# Use different deb file when building different type of images
-ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/
-RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb
-
-
-ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
-# default command shows the paddle version and exit
-CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.noavx

-FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-# ENV variables
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-ENV HOME /root
-ENV LANG en_US.UTF-8
-
-# Use Fix locales to en_US.UTF-8
-
-RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -y python python-pip libgfortran3 && \
-    apt-get clean -y && \
-    pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0'
-RUN pip install numpy
-# Use different deb file when building different type of images
-ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/
-RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb
-
-ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
-# default command shows the paddle version and exit
-CMD ["paddle", "version"]

paddle/scripts/docker/paddle-dev/Dockerfile

-# A image for building paddle binaries
-# Use cuda devel base image for both cpu and gpu environment
-FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG UBUNTU_MIRROR
-RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
-
-# ENV variables
-ARG BUILD_WOBOQ
-ARG BUILD_AND_INSTALL
-ARG WITH_AVX
-ARG WITH_DOC
-ARG WITH_STYLE_CHECK
-
-ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
-ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
-ENV WITH_GPU=OFF
-ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-OFF}
-ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
-
-ENV HOME /root
-
-RUN apt-get update && \
-    apt-get install -y git python-pip python-dev openssh-server bison && \
-    apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
-    apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
-    apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
-    apt-get install -y automake locales clang-format-3.8 && \
-    apt-get clean -y
-
-# git credential to skip password typing
-RUN git config --global credential.helper store
-
-# Fix locales to en_US.UTF-8
-RUN localedef -i en_US -f UTF-8 en_US.UTF-8
-
-RUN pip install --upgrade pip && \
-    pip install -U 'protobuf==3.1.0' && \
-    pip install -U wheel pillow BeautifulSoup && \
-    pip install -U docopt PyYAML sphinx && \
-    pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \
-    pip install -U pre-commit 'requests==2.9.2' jupyter
-
-RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
-    cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
-    cd .. && rm -rf cmake-3.4.1
-
-RUN apt-get install -y swig
-# FIXME: wait a long time is OK
-CMD ["sleep", "3600"]