提交 21e878ff 编写于 作者: G gongweibao 提交者: GitHub

Merge pull request #1598 from typhoonzero/thinnerdocker

Thinner docker image
# A image for building paddle binaries
# Use cuda devel base image for both cpu and gpu environment
FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
ARG DEBIAN_FRONTEND=noninteractive
ARG UBUNTU_MIRROR
RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
# ENV variables
ARG BUILD_WOBOQ
......@@ -14,14 +16,12 @@ ARG WITH_STYLE_CHECK
ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
ENV WITH_GPU=ON
ENV WITH_GPU=OFF
ENV WITH_AVX=${WITH_AVX:-ON}
ENV WITH_DOC=${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV DOCKER_BUILD=TRUE
ENV HOME /root
# Add bash enhancements
COPY ./paddle/scripts/docker/root/ /root/
......@@ -50,9 +50,7 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
cd .. && rm -rf cmake-3.4.1
COPY . /paddle/
RUN cd /paddle/ && git submodule update --init --recursive
RUN /paddle/paddle/scripts/docker/build.sh
RUN apt-get install -y swig
VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
......@@ -63,9 +61,5 @@ RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
EXPOSE 22
# Jupyter Notebook: Paddle book
EXPOSE 8888
COPY ./paddle/scripts/docker/entrypoint /opt/bin/
CMD ["/opt/bin/entrypoint"]
# development image default do build work
CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]
#!/bin/bash
set -e
ARCH=$(uname -i)
apt-get update
apt-get install -y dh-make
cd ~
......@@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu
mkdir -p ~/dist/cpu-noavx
mkdir -p ~/dist/gpu-noavx
cd paddle
mkdir build
# clean build dir and third_party dir cache
rm -rf build third_party
mkdir -p build
cd build
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc`
# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything?
cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/cpu
rm -rf *
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/gpu
rm -rf *
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF
rm -f /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/cpu-noavx
rm -rf *
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/gpu-noavx
FROM ubuntu:14.04
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
ARG DEBIAN_FRONTEND=noninteractive
ARG UBUNTU_MIRROR
RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
# ENV variables
ARG BUILD_WOBOQ
ARG BUILD_AND_INSTALL
ARG WITH_AVX
ARG WITH_DOC
ARG WITH_STYLE_CHECK
ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
ENV WITH_GPU=OFF
ENV WITH_AVX=${WITH_AVX:-ON}
ENV WITH_DOC=${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV DOCKER_BUILD=TRUE
ENV HOME /root
# Add bash enhancements
COPY ./paddle/scripts/docker/root/ /root/
RUN apt-get update && \
apt-get install -y git python-pip python-dev openssh-server bison && \
apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
apt-get install -y automake locales clang-format-3.8 && \
apt-get clean -y
# git credential to skip password typing
RUN git config --global credential.helper store
# Fix locales to en_US.UTF-8
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
RUN pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0' && \
pip install -U wheel pillow BeautifulSoup && \
pip install -U docopt PyYAML sphinx && \
pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \
pip install -U pre-commit 'requests==2.9.2' jupyter
RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
cd .. && rm -rf cmake-3.4.1
COPY . /paddle/
RUN cd /paddle/ && git submodule update --init --recursive
RUN /paddle/paddle/scripts/docker/build.sh
VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"]
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
RUN mkdir /var/run/sshd
RUN echo 'root:root' | chpasswd
RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
EXPOSE 22
# Jupyter Notebook: Paddle book
EXPOSE 8888
COPY ./paddle/scripts/docker/entrypoint /opt/bin/
CMD ["/opt/bin/entrypoint"]
......@@ -7,7 +7,32 @@ function abort(){
trap 'abort' 0
set -e
mkdir -p /paddle/dist/cpu
mkdir -p /paddle/dist/gpu
mkdir -p /paddle/dist/cpu-noavx
mkdir -p /paddle/dist/gpu-noavx
# Set BASE_IMAGE and DEB_PATH according to env variables
if [ ${WITH_GPU} == "ON" ]; then
BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04"
# additional packages to install when building gpu images
GPU_DOCKER_PKG="python-pip"
if [ ${WITH_AVX} == "ON" ]; then
DEB_PATH="dist/gpu/"
DOCKER_SUFFIX="gpu"
else
DEB_PATH="dist/gpu-noavx/"
DOCKER_SUFFIX="gpu-noavx"
fi
else
BASE_IMAGE="python:2.7.13-slim"
if [ ${WITH_AVX} == "ON" ]; then
DEB_PATH="dist/cpu/"
DOCKER_SUFFIX="cpu"
else
DEB_PATH="dist/cpu-noavx/"
DOCKER_SUFFIX="noavx"
fi
fi
# If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied
# source tree to /paddle, and this scripts should build it into
# /paddle/build.
......@@ -17,8 +42,11 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
fi
mkdir -p /paddle/build # -p means no error if exists
cd /paddle/build
# clean local cmake and third_party cache
cd /paddle/build && rm -rf * && rm -rf ../third_party
if [ ${DELETE_BUILD_CACHE} == 'ON' ]; then
rm -rf * && rm -rf ../third_party
fi
cmake .. \
-DWITH_DOC=${WITH_DOC:-OFF} \
-DWITH_GPU=${WITH_GPU:-OFF} \
......@@ -29,6 +57,12 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc`
make install
# generate deb package for current build
# FIXME(typhoonzero): should we remove paddle/scripts/deb ?
# FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must
# install them in docker
cpack -D CPACK_GENERATOR='DEB' -D CPACK_DEBIAN_PACKAGE_DEPENDS="" ..
mv /paddle/build/*.deb /paddle/${DEB_PATH}
if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then
apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev
......@@ -65,4 +99,46 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
fi
fi
# generate production docker image Dockerfile
if [ ${USE_MIRROR} ]; then
MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\"
else
MIRROR_UPDATE="\\"
fi
cat > /paddle/build/Dockerfile.${DOCKER_SUFFIX} <<EOF
FROM ${BASE_IMAGE}
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
# ENV variables
ARG WITH_AVX
ARG WITH_DOC
ARG WITH_STYLE_CHECK
ENV WITH_GPU=${WITH_GPU}
ENV WITH_AVX=\${WITH_AVX:-ON}
ENV WITH_DOC=\${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF}
ENV HOME /root
ENV LANG en_US.UTF-8
# Use Fix locales to en_US.UTF-8
RUN ${MIRROR_UPDATE}
apt-get update && \
apt-get install -y libgfortran3 ${GPU_DOCKER_PKG} && \
apt-get clean -y && \
pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0' requests
RUN pip install numpy
# Use different deb file when building different type of images
ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
# default command shows the paddle version and exit
CMD ["paddle", "version"]
EOF
trap : 0
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册