From 7485e5d750abeec3a724241d7b8d038adfb2344b Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Mon, 20 Mar 2017 16:52:10 -0700 Subject: [PATCH] refine docker build --- .dockerignore | 16 ++- Dockerfile | 7 +- paddle/scripts/docker/build.sh | 171 +++++++++++++-------------------- 3 files changed, 81 insertions(+), 113 deletions(-) mode change 120000 => 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 120000 index 3e4e48b0b..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..2b2e74053 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +*.DS_Store +build/ +*.user +.vscode +.idea +.project +.cproject +.pydevproject +Makefile +.test_env/ +third_party/ +*~ +bazel-* + +!build/*.deb diff --git a/Dockerfile b/Dockerfile index 536adb071..ccd43be66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,20 +3,17 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors -ARG DEBIAN_FRONTEND=noninteractive ARG UBUNTU_MIRROR RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' # ENV variables ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL ARG WITH_GPU ARG WITH_AVX ARG WITH_DOC ARG WITH_STYLE_CHECK ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} ENV WITH_GPU=${WITH_AVX:-OFF} ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} @@ -31,7 +28,7 @@ RUN apt-get update && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ + apt-get install -y automake locales clang-format-3.8 swig && \ apt-get clean -y # git credential to skip password typing @@ -51,8 +48,6 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 -RUN apt-get install -y swig - VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 3058cd903..285ebdb9c 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -1,105 +1,75 @@ #!/bin/bash -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 set -e -mkdir -p /paddle/dist/cpu -mkdir -p /paddle/dist/gpu -mkdir -p /paddle/dist/cpu-noavx -mkdir -p /paddle/dist/gpu-noavx -# Set BASE_IMAGE and DEB_PATH according to env variables + +# Set BASE_IMAGE according to env variables if [ ${WITH_GPU} == "ON" ]; then BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" # additional packages to install when building gpu images - GPU_DOCKER_PKG="python-pip" - if [ ${WITH_AVX} == "ON" ]; then - DEB_PATH="dist/gpu/" - DOCKER_SUFFIX="gpu" - else - DEB_PATH="dist/gpu-noavx/" - DOCKER_SUFFIX="gpu-noavx" - fi + GPU_DOCKER_PKG="python-pip python-dev" else BASE_IMAGE="python:2.7.13-slim" - if [ ${WITH_AVX} == "ON" ]; then - DEB_PATH="dist/cpu/" - DOCKER_SUFFIX="cpu" - else - DEB_PATH="dist/cpu-noavx/" - DOCKER_SUFFIX="noavx" - fi fi -# If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied -# source tree to /paddle, and this scripts should build it into -# /paddle/build. -if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then - if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so - fi - - mkdir -p /paddle/build # -p means no error if exists - cd /paddle/build - # clean local cmake and third_party cache - if [ ${DELETE_BUILD_CACHE} == 'ON' ]; then - rm -rf * && rm -rf ../third_party - fi - cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DWITH_DOC=${WITH_DOC:-OFF} \ - -DWITH_GPU=${WITH_GPU:-OFF} \ - -DWITH_AVX=${WITH_AVX:-OFF} \ - -DWITH_SWIG_PY=ON \ - -DCUDNN_ROOT=/usr/ \ - -DWITH_STYLE_CHECK=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - make -j `nproc` - make install - # generate deb package for current build - # FIXME(typhoonzero): should we remove paddle/scripts/deb ? - # FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must - # install them in docker - cpack -D CPACK_GENERATOR='DEB' -D CPACK_DEBIAN_PACKAGE_DEPENDS="" .. - mv /paddle/build/*.deb /paddle/${DEB_PATH} - - if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then - apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev - # Install woboq_codebrowser. - git clone https://github.com/woboq/woboq_codebrowser /woboq - cd /woboq - cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ - -DCMAKE_BUILD_TYPE=Release \ - . - make - - export WOBOQ_OUT=/usr/share/nginx/html/paddle - export BUILD_DIR=/paddle/build - mkdir -p $WOBOQ_OUT - cp -rv /woboq/data $WOBOQ_OUT/../data - /woboq/generator/codebrowser_generator \ + +DOCKERFILE_GPU_ENV="" +if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then + DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" +fi + +mkdir -p /paddle/build +cd /paddle/build + +# build script will not fail if *.deb does not exist +rm *.deb || true + +cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_DOC=${WITH_DOC:-OFF} \ + -DWITH_GPU=${WITH_GPU:-OFF} \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DCUDNN_ROOT=/usr/ \ + -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ + -DON_COVERALLS=${TEST:-OFF} \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +make -j `nproc` +if [[ ${TEST:-OFF} == "ON" ]]; then + make coveralls +fi +make install + +# generate deb package for current build +# FIXME(typhoonzero): should we remove paddle/scripts/deb ? +# FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must +# install them in docker +cpack -D CPACK_GENERATOR='DEB' -D CPACK_DEBIAN_PACKAGE_DEPENDS="" .. + +if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then + apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev + # Install woboq_codebrowser. + git clone https://github.com/woboq/woboq_codebrowser /woboq + cd /woboq + cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ + -DCMAKE_BUILD_TYPE=Release \ + . + make + + export WOBOQ_OUT=/usr/share/nginx/html/paddle + export BUILD_DIR=/paddle/build + mkdir -p $WOBOQ_OUT + cp -rv /woboq/data $WOBOQ_OUT/../data + /woboq/generator/codebrowser_generator \ -b /paddle/build \ -a \ -o $WOBOQ_OUT \ -p paddle:/paddle - /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT - cd /woboq - make clean - fi - - pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl - pip install /usr/local/opt/paddle/share/wheels/paddle*.whl - paddle version - - if [[ ${DOCKER_BUILD:-FALSE} == 'TRUE' ]]; then - # reduce docker image size - rm -rf /paddle/build - rm -rf /usr/local/opt/paddle/share/wheels/ - fi + /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT + cd /woboq + make clean fi +paddle version + # generate production docker image Dockerfile if [ ${USE_MIRROR} ]; then MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\" @@ -107,20 +77,10 @@ else MIRROR_UPDATE="\\" fi -cat > /paddle/build/Dockerfile.${DOCKER_SUFFIX} < /paddle/build/Dockerfile < -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=${WITH_GPU} -ENV WITH_AVX=\${WITH_AVX:-ON} -ENV WITH_DOC=\${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF} - ENV HOME /root ENV LANG en_US.UTF-8 @@ -128,18 +88,17 @@ ENV LANG en_US.UTF-8 RUN ${MIRROR_UPDATE} apt-get update && \ - apt-get install -y libgfortran3 ${GPU_DOCKER_PKG} && \ + apt-get install -y libgfortran3 libpython2.7 ${GPU_DOCKER_PKG} && \ apt-get clean -y && \ pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' requests -RUN pip install numpy + pip install -U 'protobuf==3.1.0' requests numpy # Use different deb file when building different type of images -ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb +ADD build/*.deb /usr/local/opt/paddle/deb/ -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# run paddle version to install python packages first +RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb && paddle version + +${DOCKERFILE_GPU_ENV} # default command shows the paddle version and exit CMD ["paddle", "version"] EOF - -trap : 0 -- GitLab