From 8c9716f7d4ea5c58414266cf89c9a5ed72af605a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=9B=8A?= Date: Thu, 26 Jan 2017 15:06:34 -0800 Subject: [PATCH] Allow users to build a Docker image as development environment from Dockerfile{.gpu} --- paddle/scripts/docker/Dockerfile | 10 ++-- paddle/scripts/docker/Dockerfile.dev | 62 --------------------- paddle/scripts/docker/Dockerfile.gpu | 10 ++-- paddle/scripts/docker/build.sh | 80 +++++++++++++++------------- 4 files changed, 52 insertions(+), 110 deletions(-) delete mode 100644 paddle/scripts/docker/Dockerfile.dev diff --git a/paddle/scripts/docker/Dockerfile b/paddle/scripts/docker/Dockerfile index 79c4efbed0b..d4845a72b61 100644 --- a/paddle/scripts/docker/Dockerfile +++ b/paddle/scripts/docker/Dockerfile @@ -13,8 +13,8 @@ RUN apt-get update && \ apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y -RUN pip install --upgrade pip && \ - pip install 'protobuf==3.1.0.post1' && \ +RUN pip install --upgrade pip && \ + pip install -U protobuf && \ pip install -U wheel pillow BeautifulSoup && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx_rtd_theme recommonmark jupyter @@ -23,10 +23,12 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \ cd .. && rm -rf cmake-3.4.1 +ARG BUILD_AND_INSTALL ARG WITH_AVX ARG WITH_DOC ARG WITH_STYLE_CHECK +ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} ENV WITH_GPU=OFF ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-ON} @@ -37,10 +39,6 @@ COPY . /paddle/ RUN /paddle/paddle/scripts/docker/build.sh VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] -RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -RUN pip install /usr/local/opt/paddle/share/wheels/*.whl -RUN paddle version # print version after build - # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd RUN echo 'root:root' | chpasswd diff --git a/paddle/scripts/docker/Dockerfile.dev b/paddle/scripts/docker/Dockerfile.dev deleted file mode 100644 index 2f245e2580f..00000000000 --- a/paddle/scripts/docker/Dockerfile.dev +++ /dev/null @@ -1,62 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -RUN apt-get update - -# Paddle development tools. -RUN apt-get install -y coreutils git cmake g++ m4 python-pip swig - -# Paddle data tools. -RUN apt-get install -y wget unzip tar xz-utils bzip2 gzip sed grep graphviz - -# Paddle documentation tools. -RUN apt-get install -y doxygen - -# Paddle code auto-reformat tools. -RUN apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev clang-format-3.8 clang-tidy-3.8 - -# Paddle prerequisites. -RUN apt-get install -y \ - libprotobuf-dev protobuf-compiler \ - libgoogle-glog-dev libgflags-dev libgtest-dev \ - libatlas-dev libatlas3-base \ - python-protobuf python-numpy python-dev python-matplotlib \ - libjpeg-dev zlib1g-dev - -# Development environment: -RUN apt-get install -y openssh-server - -RUN apt-get clean -y - -# Build Google test -RUN cd /usr/src/gtest && cmake . && make && cp *.a /usr/lib - -# Python dependencies. -RUN pip install -U BeautifulSoup docopt PyYAML pillow \ - sphinx sphinx_rtd_theme recommonmark jupyter - -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_SWIG_PY -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-ON} -ENV WITH_SWIG_PY=${WITH_SWIG_PY:-ON} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu index 6c1c2225d1a..da20b2635e1 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -13,8 +13,8 @@ RUN apt-get update && \ apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y -RUN pip install --upgrade pip && \ - pip install 'protobuf==3.1.0.post1' && \ +RUN pip install --upgrade pip && \ + pip install -U protobuf && \ pip install -U wheel pillow BeautifulSoup && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx_rtd_theme recommonmark jupyter @@ -23,10 +23,12 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \ cd .. && rm -rf cmake-3.4.1 +ARG BUILD_AND_INSTALL ARG WITH_AVX ARG WITH_DOC ARG WITH_STYLE_CHECK +ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} ENV WITH_GPU=ON ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-ON} @@ -37,10 +39,6 @@ COPY . /paddle/ RUN /paddle/paddle/scripts/docker/build.sh VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] -RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -RUN pip install /usr/local/opt/paddle/share/wheels/*.whl -RUN paddle version # print version after build - # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd RUN echo 'root:root' | chpasswd diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 7edba3dd09c..6197b41d6b5 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -8,42 +8,50 @@ function abort(){ trap 'abort' 0 set -e -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +# If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied +# source tree to /paddle, and this scripts should build it into +# /paddle/build. +if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then + if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then + ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so + fi + + mkdir -p /paddle/build # -p means no error if exists + cd /paddle/build + cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=${WITH_GPU:-OFF} \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DCUDNN_ROOT=/usr/ \ + -DWITH_STYLE_CHECK=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + make -j `nproc` + make install + + # Install woboq_codebrowser. + git clone https://github.com/woboq/woboq_codebrowser /woboq + cd /woboq + cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ + -DCMAKE_BUILD_TYPE=Release \ + . + make + + export WOBOQ_OUT=/usr/share/nginx/html/paddle + export BUILD_DIR=/paddle/build + mkdir -p $WOBOQ_OUT + cp -rv /woboq/data $WOBOQ_OUT/../data + /woboq/generator/codebrowser_generator \ + -b /paddle/build \ + -a \ + -o $WOBOQ_OUT \ + -p paddle:/paddle + /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT + cd /woboq + make clean + + pip install /usr/local/opt/paddle/share/wheels/*.whl + paddle version fi -mkdir -p /paddle/build # -p means no error if exists -cd /paddle/build -cmake .. \ - -DWITH_DOC=ON \ - -DWITH_GPU=${WITH_GPU} \ - -DWITH_AVX=${WITH_AVX} \ - -DWITH_SWIG_PY=ON \ - -DCUDNN_ROOT=/usr/ \ - -DWITH_STYLE_CHECK=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -make -j `nproc` -make install - -# Install woboq_codebrowser. -git clone https://github.com/woboq/woboq_codebrowser /woboq -cd /woboq -cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ - -DCMAKE_BUILD_TYPE=Release \ - . -make - -export WOBOQ_OUT=/usr/share/nginx/html/paddle -export BUILD_DIR=/paddle/build -mkdir -p $WOBOQ_OUT -cp -rv /woboq/data $WOBOQ_OUT/../data -/woboq/generator/codebrowser_generator \ - -b /paddle/build \ - -a \ - -o $WOBOQ_OUT \ - -p paddle:/paddle -/woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT -cd /woboq -make clean -rm -rf /paddle/build trap : 0 -- GitLab