From 1ab419a1bdd5dd5013d923b156ac44c8db0d8955 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 17 Mar 2017 13:31:02 +0800 Subject: [PATCH] new design: #1625 --- .../docker/Dockerfile.gpu => Dockerfile | 18 ++--- paddle/scripts/docker/Dockerfile | 71 ------------------- paddle/scripts/docker/build.sh | 69 +++++++++++++++++- paddle/scripts/docker/buildall.sh | 31 -------- paddle/scripts/docker/paddle-core/Dockerfile | 32 --------- .../scripts/docker/paddle-core/Dockerfile.gpu | 32 --------- .../docker/paddle-core/Dockerfile.gpunoavx | 33 --------- .../docker/paddle-core/Dockerfile.noavx | 32 --------- paddle/scripts/docker/paddle-dev/Dockerfile | 53 -------------- 9 files changed, 73 insertions(+), 298 deletions(-) rename paddle/scripts/docker/Dockerfile.gpu => Dockerfile (88%) delete mode 100644 paddle/scripts/docker/Dockerfile delete mode 100644 paddle/scripts/docker/buildall.sh delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpu delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.noavx delete mode 100644 paddle/scripts/docker/paddle-dev/Dockerfile diff --git a/paddle/scripts/docker/Dockerfile.gpu b/Dockerfile similarity index 88% rename from paddle/scripts/docker/Dockerfile.gpu rename to Dockerfile index a687d490a..b835cc52c 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/Dockerfile @@ -1,3 +1,5 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors @@ -14,14 +16,12 @@ ARG WITH_STYLE_CHECK ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=ON +ENV WITH_GPU=OFF ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE ENV HOME /root - # Add bash enhancements COPY ./paddle/scripts/docker/root/ /root/ @@ -50,9 +50,7 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh +RUN apt-get install -y swig VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] @@ -63,9 +61,5 @@ RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config EXPOSE 22 -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] +# development image default do build work +CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/paddle/scripts/docker/Dockerfile b/paddle/scripts/docker/Dockerfile deleted file mode 100644 index 48af9e5b5..000000000 --- a/paddle/scripts/docker/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE - -ENV HOME /root - -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ - apt-get clean -y - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ - pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install -U pre-commit 'requests==2.9.2' jupyter - -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh - -VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -EXPOSE 22 - -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 668b6e6b8..9f91d8b57 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -7,7 +7,26 @@ function abort(){ trap 'abort' 0 set -e - +mkdir -p /paddle/dist/cpu +mkdir -p /paddle/dist/gpu +mkdir -p /paddle/dist/cpu-noavx +mkdir -p /paddle/dist/gpu-noavx +# Set BASE_IMAGE and DEB_PATH according to env variables +if [ ${WITH_GPU} == "ON" ]; then + BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/gpu/" + else + DEB_PATH="dist/gpu-noavx/" + fi +else + BASE_IMAGE="python:2.7.13-slim" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/cpu/" + else + DEB_PATH="dist/cpu-noavx/" + fi +fi # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied # source tree to /paddle, and this scripts should build it into # /paddle/build. @@ -29,9 +48,13 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` make install + # generate deb package for current build + # FIXME(typhoonzero): should we remove paddle/scripts/deb ? + cpack -D CPACK_GENERATOR='DEB' .. + mv /paddle/build/*.deb /paddle/${DEB_PATH} if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then - apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev + apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev # Install woboq_codebrowser. git clone https://github.com/woboq/woboq_codebrowser /woboq cd /woboq @@ -65,4 +88,46 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then fi fi +# generate production docker image Dockerfile +if [ ${USE_MIRROR} ]; then + MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\" +else + MIRROR_UPDATE="\\" +fi + +cat > /paddle/build/Dockerfile < + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=${WITH_GPU} +ENV WITH_AVX=\${WITH_AVX:-ON} +ENV WITH_DOC=\${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN ${MIRROR_UPDATE} + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] +EOF + trap : 0 diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh deleted file mode 100644 index 63e5301ad..000000000 --- a/paddle/scripts/docker/buildall.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -BUILD_DIR=$PWD/build -DEB_DIST_DIR=$PWD/dist -VERSION=latest - -function build_in_docker() { - if [ ! -d $BUILD_DIR ]; then - mkdir -p $BUILD_DIR - fi - if [ ! -d $DEB_DIST_DIR ]; then - mkdir -p $DEB_DIST_DIR - fi - docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile - # FIXME: need to wait a signal not sleeping - BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600) - # NOTICE: build deb files for real paddle image - docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh" - - docker stop $BUILDER && docker rm $BUILDER -} - -function build_paddle_core() { - docker build . -t paddle:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile - docker build . -t paddle:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu - docker build . -t paddle:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx - docker build . -t paddle:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx -} - -build_in_docker -build_paddle_core diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile deleted file mode 100644 index eade296a4..000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:2.7.13-slim -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpu b/paddle/scripts/docker/paddle-core/Dockerfile.gpu deleted file mode 100644 index 89386b037..000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.gpu +++ /dev/null @@ -1,32 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y python python-pip libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx deleted file mode 100644 index beaeb5c51..000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx +++ /dev/null @@ -1,33 +0,0 @@ -FROM python:2.7.13-slim -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb - - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.noavx b/paddle/scripts/docker/paddle-core/Dockerfile.noavx deleted file mode 100644 index 853dd7703..000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.noavx +++ /dev/null @@ -1,32 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y python python-pip libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-dev/Dockerfile b/paddle/scripts/docker/paddle-dev/Dockerfile deleted file mode 100644 index 11465a4ed..000000000 --- a/paddle/scripts/docker/paddle-dev/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root - -RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ - apt-get clean -y - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ - pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install -U pre-commit 'requests==2.9.2' jupyter - -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -RUN apt-get install -y swig -# FIXME: wait a long time is OK -CMD ["sleep", "3600"] -- GitLab