From 1b269dbe0628d88181bb0a079da4e02bb3735d2a Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Tue, 14 Mar 2017 15:09:49 +0800 Subject: [PATCH] use deb to build paddle-core image --- paddle/scripts/deb/build_scripts/build.sh | 18 +++++++--- paddle/scripts/docker/buildall.sh | 27 ++++++++------- paddle/scripts/docker/paddle-core/Dockerfile | 16 ++------- .../scripts/docker/paddle-core/Dockerfile.gpu | 32 ++++++++++++++++++ .../docker/paddle-core/Dockerfile.gpunoavx | 33 +++++++++++++++++++ .../docker/paddle-core/Dockerfile.noavx | 32 ++++++++++++++++++ paddle/scripts/docker/paddle-dev/Dockerfile | 7 ++-- 7 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpu create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.noavx diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh index d13dea51484..51cf7f1b9ba 100755 --- a/paddle/scripts/deb/build_scripts/build.sh +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +ARCH=$(uname -i) apt-get update apt-get install -y dh-make cd ~ @@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu mkdir -p ~/dist/cpu-noavx mkdir -p ~/dist/gpu-noavx cd paddle -mkdir build + +# clean build dir and third_party dir cache +rm -rf build third_party +mkdir -p build cd build -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` +# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything? cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu rm -rf * -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF +rm -f /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu-noavx rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu-noavx diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh index c10720ab62d..9c1a9857540 100644 --- a/paddle/scripts/docker/buildall.sh +++ b/paddle/scripts/docker/buildall.sh @@ -1,32 +1,31 @@ #!/bin/bash -BINARIES_DIR=paddle/scripts/docker/buildimage/binaries BUILD_DIR=$PWD/build +DEB_DIST_DIR=$PWD/dist +VERSION=latest function build_in_docker() { if [ ! -d $BUILD_DIR ]; then mkdir -p $BUILD_DIR fi + if [ ! -d $DEB_DIST_DIR ]; then + mkdir -p $DEB_DIST_DIR + fi docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile # FIXME: need to wait a signal not sleeping - BUILDER=$(docker run -d -v ${PWD}:/paddle paddle-build-env sleep 3600) - # TODO(typhoonzero): - docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" - mkdir -p $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR - # docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR - # - # docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR + BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600) + # NOTICE: build deb files for real paddle image + docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh" docker stop $BUILDER && docker rm $BUILDER } function build_paddle_core() { - docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile + docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile + docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu + docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx + docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx } build_in_docker -#build_paddle_core +build_paddle_core diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile index 628f5817761..eade296a401 100644 --- a/paddle/scripts/docker/paddle-core/Dockerfile +++ b/paddle/scripts/docker/paddle-core/Dockerfile @@ -23,20 +23,10 @@ RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/ pip install --upgrade pip && \ pip install -U 'protobuf==3.1.0' RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb -ADD build/paddle/trainer/paddle_trainer \ - build/paddle/pserver/paddle_pserver_main \ - build/paddle/trainer/paddle_merge_model \ - /usr/local/opt/paddle/bin/ -ADD build/paddle/scripts/usage.sh /usr/local/opt/paddle/bin/paddle_usage -ADD paddle/scripts/submit_local.sh.in /usr/bin/paddle - -ADD paddle/dist/py_paddle*linux*.whl build/python/dist/paddle*.whl \ - /usr/local/opt/paddle/share/wheels/ -RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ - pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \ - rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ - rm /usr/local/opt/paddle/share/wheels/paddle*.whl ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" # default command shows the paddle version and exit CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpu b/paddle/scripts/docker/paddle-core/Dockerfile.gpu new file mode 100644 index 00000000000..89386b0379a --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.gpu @@ -0,0 +1,32 @@ +FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y python python-pip libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx new file mode 100644 index 00000000000..beaeb5c51e0 --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx @@ -0,0 +1,33 @@ +FROM python:2.7.13-slim +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb + + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.noavx b/paddle/scripts/docker/paddle-core/Dockerfile.noavx new file mode 100644 index 00000000000..853dd7703ac --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.noavx @@ -0,0 +1,32 @@ +FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y python python-pip libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-dev/Dockerfile b/paddle/scripts/docker/paddle-dev/Dockerfile index 1aee44e21be..11465a4ed81 100644 --- a/paddle/scripts/docker/paddle-dev/Dockerfile +++ b/paddle/scripts/docker/paddle-dev/Dockerfile @@ -1,5 +1,6 @@ # A image for building paddle binaries -FROM ubuntu:14.04 +# Use cuda devel base image for both cpu and gpu environment +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors ARG DEBIAN_FRONTEND=noninteractive @@ -22,8 +23,7 @@ ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} ENV HOME /root -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y git python-pip python-dev openssh-server bison && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ @@ -48,5 +48,6 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 +RUN apt-get install -y swig # FIXME: wait a long time is OK CMD ["sleep", "3600"] -- GitLab