diff --git a/paddle/scripts/docker/Dockerfile.gpu b/Dockerfile similarity index 85% rename from paddle/scripts/docker/Dockerfile.gpu rename to Dockerfile index a687d490a3af365f68cc6218ecbef7f64e4a1af1..8f3137df08875a640a6809e4cd8b3c4409007cff 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/Dockerfile @@ -1,9 +1,11 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors ARG DEBIAN_FRONTEND=noninteractive ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' +RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' # ENV variables ARG BUILD_WOBOQ @@ -14,14 +16,12 @@ ARG WITH_STYLE_CHECK ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=ON +ENV WITH_GPU=OFF ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE ENV HOME /root - # Add bash enhancements COPY ./paddle/scripts/docker/root/ /root/ @@ -50,9 +50,7 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh +RUN apt-get install -y swig VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] @@ -63,9 +61,5 @@ RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config EXPOSE 22 -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] +# development image default do build work +CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh index d13dea514841b110c304b8aa0e65ad16e42c75f3..51cf7f1b9ba7462e29dede474a2c72ce2e8d32a1 100755 --- a/paddle/scripts/deb/build_scripts/build.sh +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +ARCH=$(uname -i) apt-get update apt-get install -y dh-make cd ~ @@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu mkdir -p ~/dist/cpu-noavx mkdir -p ~/dist/gpu-noavx cd paddle -mkdir build + +# clean build dir and third_party dir cache +rm -rf build third_party +mkdir -p build cd build -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` +# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything? cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu rm -rf * -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF +rm -f /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu-noavx rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu-noavx diff --git a/paddle/scripts/docker/Dockerfile b/paddle/scripts/docker/Dockerfile deleted file mode 100644 index 48af9e5b5fe83f552b17cec5d843da74845497bc..0000000000000000000000000000000000000000 --- a/paddle/scripts/docker/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE - -ENV HOME /root - -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ - apt-get clean -y - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ - pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install -U pre-commit 'requests==2.9.2' jupyter - -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh - -VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -EXPOSE 22 - -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 668b6e6b84191c7042a4905f45879b7405be331c..c44874eede03a8b1060b15e175ad89622f925940 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -7,7 +7,32 @@ function abort(){ trap 'abort' 0 set -e - +mkdir -p /paddle/dist/cpu +mkdir -p /paddle/dist/gpu +mkdir -p /paddle/dist/cpu-noavx +mkdir -p /paddle/dist/gpu-noavx +# Set BASE_IMAGE and DEB_PATH according to env variables +if [ ${WITH_GPU} == "ON" ]; then + BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" + # additional packages to install when building gpu images + GPU_DOCKER_PKG="python-pip" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/gpu/" + DOCKER_SUFFIX="gpu" + else + DEB_PATH="dist/gpu-noavx/" + DOCKER_SUFFIX="gpu-noavx" + fi +else + BASE_IMAGE="python:2.7.13-slim" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/cpu/" + DOCKER_SUFFIX="cpu" + else + DEB_PATH="dist/cpu-noavx/" + DOCKER_SUFFIX="noavx" + fi +fi # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied # source tree to /paddle, and this scripts should build it into # /paddle/build. @@ -17,8 +42,11 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then fi mkdir -p /paddle/build # -p means no error if exists + cd /paddle/build # clean local cmake and third_party cache - cd /paddle/build && rm -rf * && rm -rf ../third_party + if [ ${DELETE_BUILD_CACHE} == 'ON' ]; then + rm -rf * && rm -rf ../third_party + fi cmake .. \ -DWITH_DOC=${WITH_DOC:-OFF} \ -DWITH_GPU=${WITH_GPU:-OFF} \ @@ -29,9 +57,15 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` make install + # generate deb package for current build + # FIXME(typhoonzero): should we remove paddle/scripts/deb ? + # FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must + # install them in docker + cpack -D CPACK_GENERATOR='DEB' -D CPACK_DEBIAN_PACKAGE_DEPENDS="" .. + mv /paddle/build/*.deb /paddle/${DEB_PATH} if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then - apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev + apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev # Install woboq_codebrowser. git clone https://github.com/woboq/woboq_codebrowser /woboq cd /woboq @@ -65,4 +99,46 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then fi fi +# generate production docker image Dockerfile +if [ ${USE_MIRROR} ]; then + MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\" +else + MIRROR_UPDATE="\\" +fi + +cat > /paddle/build/Dockerfile.${DOCKER_SUFFIX} < + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=${WITH_GPU} +ENV WITH_AVX=\${WITH_AVX:-ON} +ENV WITH_DOC=\${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN ${MIRROR_UPDATE} + apt-get update && \ + apt-get install -y libgfortran3 ${GPU_DOCKER_PKG} && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' requests +RUN pip install numpy +# Use different deb file when building different type of images +ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] +EOF + trap : 0