From c95a04f215d5316a1be1e716c52ad0de23a6bd27 Mon Sep 17 00:00:00 2001 From: leonwanghui Date: Thu, 16 Apr 2020 16:10:46 +0800 Subject: [PATCH] Refactor docker image framework Signed-off-by: leonwanghui --- README.md | 10 +-- docker/mindspore-cpu/0.1.0-alpha/Dockerfile | 18 +---- docker/mindspore-cpu/devel/Dockerfile | 11 +-- docker/mindspore-cpu/runtime/Dockerfile | 64 +++++++++++++++++ docker/mindspore-gpu/0.1.0-alpha/Dockerfile | 19 +---- docker/mindspore-gpu/devel/Dockerfile | 31 ++------ docker/mindspore-gpu/runtime/Dockerfile | 80 +++++++++++++++++++++ 7 files changed, 160 insertions(+), 73 deletions(-) create mode 100644 docker/mindspore-cpu/runtime/Dockerfile create mode 100644 docker/mindspore-gpu/runtime/Dockerfile diff --git a/README.md b/README.md index 169f1985e..c9abd82da 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ MindSpore offers build options across multiple backends: | GPU CUDA 10.1 | Ubuntu-x86 | ✔️ | | CPU | Ubuntu-x86 | ✔️ | -For installation using `pip`, take `Ubuntu-x86` and `CPU` build version as an example: +For installation using `pip`, take `CPU` and `Ubuntu-x86` build version as an example: 1. Download whl from [MindSpore download page](https://www.mindspore.cn/versions/en), and install the package. @@ -97,9 +97,11 @@ currently the containerized build options are supported as follows: | :---------------- | :---------------------- | :-- | :---------- | | CPU | `mindspore/mindspore-cpu` | `0.1.0-alpha` | Production environment with pre-installed MindSpore `0.1.0-alpha` CPU release. | | | | `devel` | Development environment provided to build MindSpore (with `CPU` backend) from the source, refer to https://www.mindspore.cn/install/en for installation details. | +| | | `runtime` | Runtime environment provided to install MindSpore binary package with `CPU` backend. | | GPU | `mindspore/mindspore-gpu` | `0.1.0-alpha` | Production environment with pre-installed MindSpore `0.1.0-alpha` GPU release. | | | | `devel` | Development environment provided to build MindSpore (with `GPU CUDA10.1` backend) from the source, refer to https://www.mindspore.cn/install/en for installation details. | -| Ascend |
—
|
—
| Coming soon. | +| | | `runtime` | Runtime environment provided to install MindSpore binary package with `GPU` backend. | +| Ascend |
—
|
—
| Coming soon. | * CPU @@ -111,7 +113,7 @@ currently the containerized build options are supported as follows: * GPU - For `GPU` backend, please make sure the `nvidia-container-toolkit` has been installed in advance, here are some install guidelines for Ubuntu users: + For `GPU` backend, please make sure the `nvidia-container-toolkit` has been installed in advance, here are some install guidelines for `Ubuntu` users: ``` DISTRIBUTION=$(. /etc/os-release; echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add - @@ -153,7 +155,7 @@ currently the containerized build options are supported as follows: [ 2. 2. 2. 2.]]] ``` -If anyone wants to learn more about the build process of MindSpore docker images, +If you want to learn more about the building process of MindSpore docker images, please check out `docker` folder for the details. ## Quickstart diff --git a/docker/mindspore-cpu/0.1.0-alpha/Dockerfile b/docker/mindspore-cpu/0.1.0-alpha/Dockerfile index d24d23cf6..c9fb7c2b8 100644 --- a/docker/mindspore-cpu/0.1.0-alpha/Dockerfile +++ b/docker/mindspore-cpu/0.1.0-alpha/Dockerfile @@ -4,8 +4,7 @@ MAINTAINER leonwanghui # Set env ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 -ENV CMAKE_ROOT_PATH /usr/local/cmake-3.14.1 -ENV PATH ${PYTHON_ROOT_PATH}/bin:${CMAKE_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV PATH /usr/local/bin:$PATH # Install base tools RUN apt update \ @@ -64,20 +63,5 @@ RUN mkdir -pv /root/.pip \ && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf -# Install pip package -RUN pip install --no-cache-dir \ - numpy \ - wheel \ - nose \ - pytest \ - pytest-xdist - -# Install cmake (v3.14.1) -RUN cd /tmp \ - && wget https://github.com/Kitware/CMake/releases/download/v3.14.1/cmake-3.14.1-Linux-x86_64.sh \ - && mkdir -p ${CMAKE_ROOT_PATH} \ - && bash ./cmake-3.14.1-Linux-x86_64.sh --prefix=${CMAKE_ROOT_PATH} --exclude-subdir --skip-license \ - && rm -f /tmp/cmake-3.14.1-Linux-x86_64.sh - # Install MindSpore cpu whl package RUN pip install --no-cache-dir https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.1.0-alpha/MindSpore/cpu/ubuntu-x86/mindspore-0.1.0-cp37-cp37m-linux_x86_64.whl diff --git a/docker/mindspore-cpu/devel/Dockerfile b/docker/mindspore-cpu/devel/Dockerfile index a48326af9..77c422ef9 100644 --- a/docker/mindspore-cpu/devel/Dockerfile +++ b/docker/mindspore-cpu/devel/Dockerfile @@ -62,15 +62,8 @@ RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ RUN mkdir -pv /root/.pip \ && echo "[global]" > /root/.pip/pip.conf \ && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ - && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf - -# Install pip package -RUN pip install --no-cache-dir \ - numpy \ - wheel \ - nose \ - pytest \ - pytest-xdist + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf \ + && pip install --no-cache-dir wheel # Install cmake (v3.14.1) RUN cd /tmp \ diff --git a/docker/mindspore-cpu/runtime/Dockerfile b/docker/mindspore-cpu/runtime/Dockerfile new file mode 100644 index 000000000..ccf5f297a --- /dev/null +++ b/docker/mindspore-cpu/runtime/Dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV PATH /usr/local/bin:$PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf diff --git a/docker/mindspore-gpu/0.1.0-alpha/Dockerfile b/docker/mindspore-gpu/0.1.0-alpha/Dockerfile index 7b82b21a1..50ca2b9f0 100644 --- a/docker/mindspore-gpu/0.1.0-alpha/Dockerfile +++ b/docker/mindspore-gpu/0.1.0-alpha/Dockerfile @@ -4,9 +4,9 @@ MAINTAINER leonwanghui # Set env ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 -ENV CMAKE_ROOT_PATH /usr/local/cmake-3.14.1 ENV OMPI_ROOT_PATH /usr/local/openmpi-3.1.5 -ENV PATH ${PYTHON_ROOT_PATH}/bin:${CMAKE_ROOT_PATH}/bin:${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV PATH ${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH ${OMPI_ROOT_PATH}/lib:$LD_LIBRARY_PATH # Install base tools RUN apt update \ @@ -67,21 +67,6 @@ RUN mkdir -pv /root/.pip \ && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf -# Install pip package -RUN pip install --no-cache-dir \ - numpy \ - wheel \ - nose \ - pytest \ - pytest-xdist - -# Install cmake (v3.14.1) -RUN cd /tmp \ - && wget https://github.com/Kitware/CMake/releases/download/v3.14.1/cmake-3.14.1-Linux-x86_64.sh \ - && mkdir -p ${CMAKE_ROOT_PATH} \ - && bash ./cmake-3.14.1-Linux-x86_64.sh --prefix=${CMAKE_ROOT_PATH} --exclude-subdir --skip-license \ - && rm -f /tmp/cmake-3.14.1-Linux-x86_64.sh - # Install openmpi (v3.1.5) RUN cd /tmp \ && wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.5.tar.gz \ diff --git a/docker/mindspore-gpu/devel/Dockerfile b/docker/mindspore-gpu/devel/Dockerfile index 0b07605d9..113bed5b0 100644 --- a/docker/mindspore-gpu/devel/Dockerfile +++ b/docker/mindspore-gpu/devel/Dockerfile @@ -5,8 +5,8 @@ MAINTAINER leonwanghui # Set env ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 ENV CMAKE_ROOT_PATH /usr/local/cmake-3.14.1 -ENV OMPI_ROOT_PATH /usr/local/openmpi-3.1.5 -ENV PATH ${PYTHON_ROOT_PATH}/bin:${CMAKE_ROOT_PATH}/bin:${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV PATH ${CMAKE_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH # Install base tools RUN apt update \ @@ -35,9 +35,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt install -y \ autoconf \ libtool \ automake \ - flex \ - libnccl2=2.4.8-1+cuda10.1 \ - libnccl-dev=2.4.8-1+cuda10.1 + flex # Set bash RUN echo "dash dash/sh boolean false" | debconf-set-selections @@ -65,15 +63,8 @@ RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ RUN mkdir -pv /root/.pip \ && echo "[global]" > /root/.pip/pip.conf \ && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ - && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf - -# Install pip package -RUN pip install --no-cache-dir \ - numpy \ - wheel \ - nose \ - pytest \ - pytest-xdist + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf \ + && pip install --no-cache-dir wheel # Install cmake (v3.14.1) RUN cd /tmp \ @@ -81,15 +72,3 @@ RUN cd /tmp \ && mkdir -p ${CMAKE_ROOT_PATH} \ && bash ./cmake-3.14.1-Linux-x86_64.sh --prefix=${CMAKE_ROOT_PATH} --exclude-subdir --skip-license \ && rm -f /tmp/cmake-3.14.1-Linux-x86_64.sh - -# Install openmpi (v3.1.5) -RUN cd /tmp \ - && wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.5.tar.gz \ - && tar -xvf openmpi-3.1.5.tar.gz \ - && cd /tmp/openmpi-3.1.5 \ - && mkdir -p ${OMPI_ROOT_PATH} \ - && ./configure --prefix=${OMPI_ROOT_PATH} \ - && make -j4 \ - && make install -j4 \ - && rm -rf /tmp/openmpi-3.1.5 \ - && rm -f /tmp/openmpi-3.1.5.tar.gz diff --git a/docker/mindspore-gpu/runtime/Dockerfile b/docker/mindspore-gpu/runtime/Dockerfile new file mode 100644 index 000000000..9e8dabe59 --- /dev/null +++ b/docker/mindspore-gpu/runtime/Dockerfile @@ -0,0 +1,80 @@ +FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV OMPI_ROOT_PATH /usr/local/openmpi-3.1.5 +ENV PATH ${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH ${OMPI_ROOT_PATH}/lib:$LD_LIBRARY_PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex \ + libnccl2=2.4.8-1+cuda10.1 \ + libnccl-dev=2.4.8-1+cuda10.1 + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf + +# Install openmpi (v3.1.5) +RUN cd /tmp \ + && wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.5.tar.gz \ + && tar -xvf openmpi-3.1.5.tar.gz \ + && cd /tmp/openmpi-3.1.5 \ + && mkdir -p ${OMPI_ROOT_PATH} \ + && ./configure --prefix=${OMPI_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -rf /tmp/openmpi-3.1.5 \ + && rm -f /tmp/openmpi-3.1.5.tar.gz -- GitLab