From a6eec0228c9176095b6112e314763edd95a254a0 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 10 Mar 2017 17:09:19 +0800 Subject: [PATCH] thinner docker --- paddle/scripts/docker/README.md | 9 +++ paddle/scripts/docker/buildall.sh | 27 +++++++++ .../docker/buildimage/Dockerfile.build | 59 +++++++++++++++++++ paddle/scripts/docker/paddle-core/Dockerfile | 38 ++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 paddle/scripts/docker/README.md create mode 100644 paddle/scripts/docker/buildall.sh create mode 100644 paddle/scripts/docker/buildimage/Dockerfile.build create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md new file mode 100644 index 0000000000..167b6be8ee --- /dev/null +++ b/paddle/scripts/docker/README.md @@ -0,0 +1,9 @@ +# Build docker image + +We use a docker environment to build paddle binaries and put it into a runtime image `paddle-core` for uses of most cases + +***Notice***: do **not** run in this directory, run under the top level of this project like: + +``` +sh paddle/scripts/docker/buildall.sh +``` diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh new file mode 100644 index 0000000000..2d7de80965 --- /dev/null +++ b/paddle/scripts/docker/buildall.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +BINARIES_DIR=paddle/scripts/docker/buildimage/binaries + +function build_in_docker() { + docker build . -t paddle-build-env -f paddle/scripts/docker/buildimage/Dockerfile.build + BUILDER=$(docker run -d paddle-build-env) + docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" + mkdir -p $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR + docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR + + docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR + + docker stop $BUILDER && docker rm $BUILDER +} + +function build_paddle_core() { + docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile + +} + +build_in_docker +build_paddle_core diff --git a/paddle/scripts/docker/buildimage/Dockerfile.build b/paddle/scripts/docker/buildimage/Dockerfile.build new file mode 100644 index 0000000000..03e6ac792f --- /dev/null +++ b/paddle/scripts/docker/buildimage/Dockerfile.build @@ -0,0 +1,59 @@ +# A image for building paddle binaries +FROM ubuntu:14.04 +MAINTAINER PaddlePaddle Authors + +ARG DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR +RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' + +# ENV variables +ARG BUILD_WOBOQ +ARG BUILD_AND_INSTALL +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} +ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y git python-pip python-dev openssh-server bison && \ + apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ + apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ + apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ + apt-get install -y automake locales clang-format-3.8 && \ + apt-get clean -y + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' && \ + pip install -U wheel pillow BeautifulSoup && \ + pip install -U docopt PyYAML sphinx && \ + pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ + pip install -U pre-commit 'requests==2.9.2' jupyter + +RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ + cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ + cd .. && rm -rf cmake-3.4.1 + +COPY . /paddle/ +RUN cd /paddle/ && git submodule update --init --recursive +RUN /paddle/paddle/scripts/docker/build.sh + +VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] + + +# FIXME: wait a long time is OK +CMD ["sleep", "3600"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile new file mode 100644 index 0000000000..708e4fe801 --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile @@ -0,0 +1,38 @@ +FROM python:2.7.13-slim +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy + +ADD paddle/scripts/docker/buildimage/binaries/paddle paddle/scripts/docker/buildimage/binaries/paddle_trainer \ +paddle/scripts/docker/buildimage/binaries/paddle_pserver_main paddle/scripts/docker/buildimage/binaries/paddle_merge_model \ +paddle/scripts/docker/buildimage/binaries/paddle_usage /usr/local/opt/paddle/bin/ +ADD paddle/scripts/docker/buildimage/binaries/wheels/py_paddle*linux*.whl paddle/scripts/docker/buildimage/binaries/wheels/paddle*.whl \ + /usr/local/opt/paddle/share/wheels/ +RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ + pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \ + rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ + rm /usr/local/opt/paddle/share/wheels/paddle*.whl +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] -- GitLab