From 5d0f9d31c9b7c360e0794704ef1418f4e310dfd8 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Mon, 20 Mar 2017 20:06:02 +0800 Subject: [PATCH] update docker build scripts and readme --- paddle/scripts/docker/README.md | 33 ++++++++++++++++++--------------- paddle/scripts/docker/build.sh | 26 +++++++++++++------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index a14f22f82..edfffd024 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -78,20 +78,24 @@ The following commands check out the source code to the host and build the devel ```bash git clone https://github.com/PaddlePaddle/Paddle paddle cd paddle -docker build -t paddle:dev [--build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt] . +docker build -t paddle:dev . ``` -The `docker build` command assumes that `Dockerfile` is in the root source tree. Note that in this design, this `Dockerfile` is this only one in our repo. Add `--build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt` -if you want to use ubuntu mirrors to speed up the build. +The `docker build` command assumes that `Dockerfile` is in the root source tree. Note that in this design, this `Dockerfile` is this only one in our repo. +Users can specify a Ubuntu mirror server for faster downloading: + +```bash +docker build -t paddle:dev --build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt . +``` ### Build PaddlePaddle from Source Code Given the development image `paddle:dev`, the following command builds PaddlePaddle from the source tree on the development computer (host): ```bash -docker run -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" \ - -e "TEST=ON" -e "BUILD_AND_INSTALL=ON" [-e "DELETE_BUILD_CACHE=ON"] paddle:dev +docker run -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "TEST=OFF" \ + -e "BUILD_AND_INSTALL=ON" -e "DELETE_BUILD_CACHE=ON" paddle:dev ``` This command mounts the source directory on the host into `/paddle` in the container, so the default entry point of `paddle:dev`, `build.sh`, could build the source code with possible local changes. When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed. @@ -99,26 +103,25 @@ This command mounts the source directory on the host into `/paddle` in the conta `build.sh` builds the following: - PaddlePaddle binaries, -- `$PWD/build/paddle-.deb` for production installation, and +- `$PWD/dist//paddle-.deb` for production installation, and - `$PWD/build/Dockerfile`, which builds the production Docker image. -Environment varibles(use `ON` and `OFF` to put the switch on and off): -- `WITH_GPU`: build paddle with gpu driver and libraries. -- `WITH_AVX`: only lagacy hardwares without avx or sse or other [SIMD](https://en.wikipedia.org/wiki/SIMD) need to put it to "OFF" -- `TEST`: test after build -- `BUILD_AND_INSTALL`: put this to "OFF" if you don't really want to build, used to generate production Dockerfiles. -- `DELETE_BUILD_CACHE`: put this to "ON" when build paddle for multiple times, third_party will not download and build again. +Users can specify the following Docker build arguments with either "ON" or "OFF" value: +- `WITH_GPU`: ***Required***. Generates NVIDIA CUDA GPU code and relies on CUDA libraries. +- `WITH_AVX`: ***Required***. Set to "OFF" prevents from generating AVX instructions. If you don't know what is AVX, you might want to set "ON". +- `TEST`: ***Optional, default ON***. Build unit tests and run them after building. +- `BUILD_AND_INSTALL`: ***Optional, default ON***. Run `make` and `make install`. +- `DELETE_BUILD_CACHE`: ***Optional, default ON***. If set to "ON", the building script will delete, download, and re-build third party libraries. ### Build the Production Docker Image The following command builds the production image: ```bash -docker build -t paddle -f build/Dockerfile[.cpu|.gpu|.noavx|.gpu-noavx] . +docker build -t paddle -f build/Dockerfile . ``` -There are 4 different branch of the production image: cpu, gpu, noavx and gpu-noavx. -These images are made to minimal size -- they includes binary `paddle`, the shared library `libpaddle.so`, and Python runtime. +This production image is minimal -- it includes binary `paddle`, the shared library `libpaddle.so`, and Python runtime. ### Run PaddlePaddle Applications diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index c44874eed..300383dae 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -15,28 +15,24 @@ mkdir -p /paddle/dist/gpu-noavx if [ ${WITH_GPU} == "ON" ]; then BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" # additional packages to install when building gpu images - GPU_DOCKER_PKG="python-pip" + GPU_DOCKER_PKG="python-pip python-dev" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/gpu/" - DOCKER_SUFFIX="gpu" else DEB_PATH="dist/gpu-noavx/" - DOCKER_SUFFIX="gpu-noavx" fi else BASE_IMAGE="python:2.7.13-slim" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/cpu/" - DOCKER_SUFFIX="cpu" else DEB_PATH="dist/cpu-noavx/" - DOCKER_SUFFIX="noavx" fi fi # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied # source tree to /paddle, and this scripts should build it into # /paddle/build. -if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then +if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so fi @@ -44,7 +40,7 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then mkdir -p /paddle/build # -p means no error if exists cd /paddle/build # clean local cmake and third_party cache - if [ ${DELETE_BUILD_CACHE} == 'ON' ]; then + if [ ${DELETE_BUILD_CACHE:-ON} == 'ON' ]; then rm -rf * && rm -rf ../third_party fi cmake .. \ @@ -54,8 +50,12 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then -DWITH_SWIG_PY=ON \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=OFF \ + -DON_COVERALLS=${TEST:-ON} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` + if [ ${TEST:-ON} == "ON" ]; then + make coveralls + fi make install # generate deb package for current build # FIXME(typhoonzero): should we remove paddle/scripts/deb ? @@ -106,7 +106,7 @@ else MIRROR_UPDATE="\\" fi -cat > /paddle/build/Dockerfile.${DOCKER_SUFFIX} < /paddle/build/Dockerfile < @@ -116,7 +116,7 @@ ARG WITH_DOC ARG WITH_STYLE_CHECK ENV WITH_GPU=${WITH_GPU} -ENV WITH_AVX=\${WITH_AVX:-ON} +ENV WITH_AVX=${WITH_AVX} ENV WITH_DOC=\${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF} @@ -127,14 +127,14 @@ ENV LANG en_US.UTF-8 RUN ${MIRROR_UPDATE} apt-get update && \ - apt-get install -y libgfortran3 ${GPU_DOCKER_PKG} && \ + apt-get install -y libgfortran3 libpython2.7 ${GPU_DOCKER_PKG} && \ apt-get clean -y && \ pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' requests -RUN pip install numpy + pip install -U 'protobuf==3.1.0' requests numpy # Use different deb file when building different type of images ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb +# run paddle version to install python packages first +RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb && paddle version ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" # default command shows the paddle version and exit -- GitLab