From e3c73b981914c709e65e0b10fee1dc6d51501d07 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Mon, 27 Jul 2020 14:43:14 +0800 Subject: [PATCH] Add dockerfile (#24833) --- tools/dockerfile/Dockerfile.centos | 81 +++++++ tools/dockerfile/Dockerfile.ubuntu | 222 ++++++++++++++++++ tools/dockerfile/build_scripts/build.sh | 161 +++++++++++++ tools/dockerfile/build_scripts/build_utils.sh | 198 ++++++++++++++++ tools/dockerfile/build_scripts/install_gcc.sh | 31 +++ .../dockerfile/build_scripts/install_nccl2.sh | 29 +++ tools/dockerfile/build_scripts/install_trt.sh | 16 ++ .../build_scripts/manylinux1-check.py | 70 ++++++ .../build_scripts/python-tag-abi-tag.py | 21 ++ tools/dockerfile/build_scripts/ssl-check.py | 46 ++++ tools/dockerfile/centos6_manylinux.sh | 41 ++++ tools/dockerfile/ci_dockerfile.sh | 37 +++ tools/dockerfile/icode.sh | 109 +++++++++ tools/dockerfile/ubuntu16_dev.sh | 91 +++++++ 14 files changed, 1153 insertions(+) create mode 100644 tools/dockerfile/Dockerfile.centos create mode 100644 tools/dockerfile/Dockerfile.ubuntu create mode 100644 tools/dockerfile/build_scripts/build.sh create mode 100755 tools/dockerfile/build_scripts/build_utils.sh create mode 100644 tools/dockerfile/build_scripts/install_gcc.sh create mode 100644 tools/dockerfile/build_scripts/install_nccl2.sh create mode 100644 tools/dockerfile/build_scripts/install_trt.sh create mode 100644 tools/dockerfile/build_scripts/manylinux1-check.py create mode 100644 tools/dockerfile/build_scripts/python-tag-abi-tag.py create mode 100644 tools/dockerfile/build_scripts/ssl-check.py create mode 100755 tools/dockerfile/centos6_manylinux.sh create mode 100644 tools/dockerfile/ci_dockerfile.sh create mode 100755 tools/dockerfile/icode.sh create mode 100755 tools/dockerfile/ubuntu16_dev.sh diff --git a/tools/dockerfile/Dockerfile.centos b/tools/dockerfile/Dockerfile.centos new file mode 100644 index 00000000000..bb620e6822a --- /dev/null +++ b/tools/dockerfile/Dockerfile.centos @@ -0,0 +1,81 @@ +# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in +# order to satisfy the build of capnproto library (a nupic.core dependency), +# which requires some headers and symbols not present on CentOS-5 (e.g., +# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See +# https://github.com/sandstorm-io/capnproto/issues/350. +FROM nvidia/cuda: +MAINTAINER Numenta, based on the ManyLinux project + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel +COPY build_scripts /build_scripts +RUN bash build_scripts/build.sh +RUN bash build_scripts/install_nccl2.sh && \ + bash build_scripts/install_trt.sh +RUN rm -rf build_scripts + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem + +# for paddle +RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/gopath && \ + mkdir /root/gopath/bin && \ + mkdir /root/gopath/src + + +ENV GOROOT=/usr/local/go GOPATH=/root/gopath +ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH} + +# protobuf 3.6.1 +RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \ + tar xzf protobuf-cpp-3.6.1.tar.gz && \ + cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz + +RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt + +RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install setuptools -U + +RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install -r /root/requirements.txt && \ + go get github.com/Masterminds/glide && \ + rm -rf /root/requirements.txt + +RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install pre-commit 'ipython==5.3.0' opencv-python && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.15-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install pre-commit 'ipython==5.3.0' opencv-python && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install pre-commit 'ipython==5.3.0' opencv-python && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install pre-commit 'ipython==5.3.0' opencv-python && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install pre-commit 'ipython==5.3.0' opencv-python && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install pre-commit 'ipython==5.3.0' opencv-python + +RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ + cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +# gcc4.8 TRT +RUN mkdir -p /opt/compiler && cd /opt/compiler && \ + wget -q https://paddle-ci.gz.bcebos.com/gcc-4.8.2.tar.gz && \ + tar xf gcc-4.8.2.tar.gz && rm -f gcc-4.8.2.tar.gz + +CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu new file mode 100644 index 00000000000..f424d676f70 --- /dev/null +++ b/tools/dockerfile/Dockerfile.ubuntu @@ -0,0 +1,222 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM nvidia/cuda: +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY paddle/scripts/docker/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev \ + net-tools libtool module-init-tools && \ + apt-get clean -y + +# Downgrade gcc&&g++ + + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/gopath && \ + mkdir /root/gopath/bin && \ + mkdir /root/gopath/src +ENV GOROOT=/usr/local/go GOPATH=/root/gopath +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin +# install glide +RUN curl -s -q https://glide.sh/get | sh + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +# specify sphinx version as 1.5.6 and remove -U option for [pip install -U +# sphinx-rtd-theme] since -U option will cause sphinx being updated to newest +# version(1.7.1 for now), which causes building documentation failed. +RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ + pip3 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ + pip3 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ + pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ + pip3.6 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ + pip3.6 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ + pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ + pip3.7 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ + pip3.7 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ + pip3.8 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ + pip3.8 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ + pip3.8 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ + pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ + pip --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ + pip --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark + +RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip3 --no-cache-dir install opencv-python && \ + pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip3.6 --no-cache-dir install opencv-python && \ + pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip3.7 --no-cache-dir install opencv-python && \ + pip3.8 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.8 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip3.8 --no-cache-dir install opencv-python && \ + pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip --no-cache-dir install opencv-python + +#For docstring checker +RUN pip3 --no-cache-dir install pylint pytest astroid isort && \ + pip3.6 --no-cache-dir install pylint pytest astroid isort && \ + pip3.7 --no-cache-dir install pylint pytest astroid isort && \ + pip3.8 --no-cache-dir install pylint pytest astroid isort && \ + pip --no-cache-dir install pylint pytest astroid isort LinkChecker + +RUN pip3 --no-cache-dir install coverage && \ + pip3.6 --no-cache-dir install coverage && \ + pip3.7 --no-cache-dir install coverage && \ + pip3.8 --no-cache-dir install coverage && \ + pip --no-cache-dir install coverage + +COPY ./python/requirements.txt /root/ +RUN pip3 --no-cache-dir install -r /root/requirements.txt && \ + pip3.6 --no-cache-dir install -r /root/requirements.txt && \ + pip3.7 --no-cache-dir install -r /root/requirements.txt && \ + pip3.8 --no-cache-dir install -r /root/requirements.txt && \ + pip --no-cache-dir install -r /root/requirements.txt + +# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use +# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 +RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y && \ + pip3 --no-cache-dir install certifi urllib3[secure] && \ + pip3.6 --no-cache-dir install certifi urllib3[secure] && \ + pip3.7 --no-cache-dir install certifi urllib3[secure] && \ + pip3.8 --no-cache-dir install certifi urllib3[secure] && \ + pip --no-cache-dir install certifi urllib3[secure] + +# ar mishandles 4GB files +# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 +# remove them when apt-get support 2.27 and higher version +RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ + tar -xzf binutils_2.27.orig.tar.gz && \ + cd binutils-2.27 && \ + ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz + +RUN pip --no-cache-dir install -U netifaces==0.10.9 + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +EXPOSE 22 diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh new file mode 100644 index 00000000000..c42e9f25fe5 --- /dev/null +++ b/tools/dockerfile/build_scripts/build.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -ex + +# Python versions to be installed in /opt/$VERSION_NO +# NOTE Only need python 2.7.11 for nupic.core/nupic.bindings at this time, so +# remove others to expedite build and reduce docker image size. The original +# manylinux docker image project builds many python versions. +# NOTE We added back 3.5.1, since auditwheel requires python 3.3+ +CPYTHON_VERSIONS="3.8.0 3.7.0 3.6.0 3.5.1 2.7.15" + +# openssl version to build, with expected sha256 hash of .tar.gz +# archive +OPENSSL_ROOT=openssl-1.1.0i +OPENSSL_HASH=ebbfc844a8c8cc0ea5dc10b86c9ce97f401837f3fa08c17b2cdadc118253cf99 +EPEL_RPM_HASH=e5ed9ecf22d0c4279e92075a64c757ad2b38049bcf5c16c4f2b75d5f6860dc0d +DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc +PATCHELF_HASH=f2aa40a6148cb3b0ca807a1bf836b081793e55ec9e5540a5356d800132be7e0a +CURL_ROOT=curl-7.49.1 +CURL_HASH=eb63cec4bef692eab9db459033f409533e6d10e20942f4b060b32819e81885f1 +AUTOCONF_ROOT=autoconf-2.69 +AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 + +# Dependencies for compiling Python that we want to remove from +# the final image after compiling Python +PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel" + +# Libraries that are allowed as part of the manylinux1 profile +MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel" + +# Get build utilities +MY_DIR=$(dirname "${BASH_SOURCE[0]}") +source $MY_DIR/build_utils.sh + +# EPEL support +yum -y install wget curl +curl -sLO https://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm +check_sha256sum epel-release-6-8.noarch.rpm $EPEL_RPM_HASH + +# Dev toolset (for LLVM and other projects requiring C++11 support) +curl -sLO http://people.centos.org/tru/devtools-2/devtools-2.repo +check_sha256sum devtools-2.repo $DEVTOOLS_HASH +mv devtools-2.repo /etc/yum.repos.d/devtools-2.repo +rpm -Uvh --replacepkgs epel-release-6*.rpm +rm -f epel-release-6*.rpm + +# Development tools and libraries +yum -y install bzip2 make git patch unzip bison yasm diffutils \ + automake which file \ + kernel-devel-`uname -r` \ + devtoolset-2-binutils devtoolset-2-gcc \ + devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran \ + ${PYTHON_COMPILE_DEPS} + +# Install more recent version of cmake +# curl -O https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.sh +# /bin/sh cmake-3.8.1-Linux-x86_64.sh --prefix=/usr/local --skip-license +# rm cmake-3.8.1-Linux-x86_64.sh + +wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && tar xzf cmake-3.16.0.tar.gz && \ +cd cmake-3.16.0 && ./bootstrap && \ +make -j8 && make install && cd .. && rm cmake-3.16.0.tar.gz + +# Install newest autoconf +build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH +autoconf --version + +# Compile the latest Python releases. +# (In order to have a proper SSL module, Python is compiled +# against a recent openssl [see env vars above], which is linked +# statically. We delete openssl afterwards.) +build_openssl $OPENSSL_ROOT $OPENSSL_HASH +mkdir -p /opt/python +build_cpythons $CPYTHON_VERSIONS + +PY35_BIN=/opt/python/cp35-cp35m/bin +PY36_BIN=/opt/python/cp36-cp36m/bin +PY37_BIN=/opt/python/cp37-cp37m/bin +PY38_BIN=/opt/python/cp38-cp38m/bin +# NOTE Since our custom manylinux image builds pythons with shared +# libpython, we need to add libpython's dir to LD_LIBRARY_PATH before running +# python. +ORIGINAL_LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib:$(dirname ${PY36_BIN})/lib:$(dirname ${PY37_BIN})/lib:$(dirname ${PY38_BIN})/lib" + +# Our openssl doesn't know how to find the system CA trust store +# (https://github.com/pypa/manylinux/issues/53) +# And it's not clear how up-to-date that is anyway +# So let's just use the same one pip and everyone uses +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install certifi +ln -s $($PY35_BIN/python -c 'import certifi; print(certifi.where())') \ + /opt/_internal/certs.pem +# If you modify this line you also have to modify the versions in the +# Dockerfiles: +export SSL_CERT_FILE=/opt/_internal/certs.pem + +# Install newest curl +build_curl $CURL_ROOT $CURL_HASH +rm -rf /usr/local/include/curl /usr/local/lib/libcurl* /usr/local/lib/pkgconfig/libcurl.pc +hash -r +curl --version +curl-config --features + +# Now we can delete our built SSL +rm -rf /usr/local/ssl + +# Install patchelf (latest with unreleased bug fixes) +# FIXME(typhoonzero): restore this when the link is fixed. +# curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz +# check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH +# tar -xzf patchelf-0.9njs2.tar.gz +# (cd patchelf-0.9njs2 && ./configure && make && make install) +# rm -rf patchelf-0.9njs2.tar.gz patchelf-0.9njs2 +yum install -y patchelf + +# Install latest pypi release of auditwheel +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install auditwheel +ln -s $PY35_BIN/auditwheel /usr/local/bin/auditwheel + +# Clean up development headers and other unnecessary stuff for +# final image +yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \ + avahi freetype bitstream-vera-fonts \ + ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 || true +yum -y install ${MANYLINUX1_DEPS} && yum -y clean all > /dev/null 2>&1 || true +yum list installed +# we don't need libpython*.a, and they're many megabytes +find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f +# Strip what we can -- and ignore errors, because this just attempts to strip +# *everything*, including non-ELF files: +find /opt/_internal -type f -print0 \ + | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true +# We do not need the Python test suites, or indeed the precompiled .pyc and +# .pyo files. Partially cribbed from: +# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile +find /opt/_internal \ + \( -type d -a -name test -o -name tests \) \ + -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + -print0 | xargs -0 rm -f + +for PYTHON in /opt/python/*/bin/python; do + # Add matching directory of libpython shared library to library lookup path + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" + + # Smoke test to make sure that our Pythons work, and do indeed detect as + # being manylinux compatible: + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/manylinux1-check.py + # Make sure that SSL cert checking works + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/ssl-check.py +done + +# Restore LD_LIBRARY_PATH +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" + +# According to ar issues: https://lists.gnu.org/archive/html/bug-binutils/2016-05/msg00211.html +# we should install new version ar with 64-bit supported here +wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz +tar xzf binutils-2.27.tar.gz && cd binutils-2.27 +./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install diff --git a/tools/dockerfile/build_scripts/build_utils.sh b/tools/dockerfile/build_scripts/build_utils.sh new file mode 100755 index 00000000000..6f201a8579f --- /dev/null +++ b/tools/dockerfile/build_scripts/build_utils.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# Helper utilities for build + +PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python +# XXX: the official https server at www.openssl.org cannot be reached +# with the old versions of openssl and curl in Centos 5.11 hence the fallback +# to the ftp mirror: +# OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source +# Ditto the curl sources +CURL_DOWNLOAD_URL=http://curl.askapache.com/download + +GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py + +AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf + + +function check_var { + if [ -z "$1" ]; then + echo "required variable not defined" + exit 1 + fi +} + + +function lex_pyver { + # Echoes Python version string padded with zeros + # Thus: + # 3.2.1 -> 003002001 + # 3 -> 003000000 + echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' +} + + +function do_cpython_build { + local py_ver=$1 + check_var $py_ver + local ucs_setting=$2 + check_var $ucs_setting + tar -xzf Python-$py_ver.tgz + pushd Python-$py_ver + if [ "$ucs_setting" = "none" ]; then + unicode_flags="" + dir_suffix="" + else + local unicode_flags="--enable-unicode=$ucs_setting" + local dir_suffix="-$ucs_setting" + fi + local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}" + mkdir -p ${prefix}/lib + # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6 + + if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.6) ]; then + wget https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz + tar -zxf sqlite-autoconf-3250300.tar.gz + cd sqlite-autoconf-3250300 + ./configure --prefix=/usr/local + make -j8 && make install + cd ../ && rm sqlite-autoconf-3250300.tar.gz + fi + + # NOTE --enable-shared for generating libpython shared library needed for + # linking of some of the nupic.core test executables. + if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.7) ]; then + # NOTE python 3.7 should be installed via make altinstall rather than + # make install, and we should specify the location of ssl + CFLAGS="-Wformat" ./configure --prefix=${prefix} --with-openssl=/usr/local/ssl --enable-shared $unicode_flags > /dev/null + make -j8 > /dev/null + make altinstall > /dev/null + else + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} CFLAGS="-Wformat" ./configure --prefix=${prefix} --enable-shared $unicode_flags > /dev/null + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make -j8 > /dev/null + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make install > /dev/null + fi + popd + echo "ZZZ looking for libpython" + find / -name 'libpython*.so*' + rm -rf Python-$py_ver + # Some python's install as bin/python3. Make them available as + # bin/python. + if [ -e ${prefix}/bin/python3 ]; then + ln -s python3 ${prefix}/bin/python + fi + if [ -e ${prefix}/bin/python3.7 ]; then + ln -s python3.7 ${prefix}/bin/python + fi + if [ -e ${prefix}/bin/python3.8 ]; then + ln -s python3.8 ${prefix}/bin/python + fi + # NOTE Make libpython shared library visible to python calls below + LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python get-pip.py + LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/pip install wheel + cd / + ls ${MY_DIR} + local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${prefix} /opt/python/${abi_tag} +} + + +function build_cpython { + local py_ver=$1 + check_var $py_ver + check_var $PYTHON_DOWNLOAD_URL + wget -q $PYTHON_DOWNLOAD_URL/$py_ver/Python-$py_ver.tgz + if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then + # NOTE We only need wide unicode for nupic.bindings wheel + do_cpython_build $py_ver ucs2 + do_cpython_build $py_ver ucs4 + else + do_cpython_build $py_ver none + fi + rm -f Python-$py_ver.tgz +} + + +function build_cpythons { + for py_ver in $@; do + check_var $GET_PIP_URL + curl -sLO $GET_PIP_URL + build_cpython $py_ver + done + rm get-pip.py +} + + +function do_openssl_build { + ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null + make > /dev/null + make install > /dev/null +} + + +function check_sha256sum { + local fname=$1 + check_var ${fname} + local sha256=$2 + check_var ${sha256} + + echo "${sha256} ${fname}" > ${fname}.sha256 + sha256sum -c ${fname}.sha256 + rm ${fname}.sha256 +} + + +function build_openssl { + local openssl_fname=$1 + check_var ${openssl_fname} + local openssl_sha256=$2 + check_var ${openssl_sha256} + check_var ${OPENSSL_DOWNLOAD_URL} + curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz + check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} + tar -xzf ${openssl_fname}.tar.gz + (cd ${openssl_fname} && do_openssl_build) + rm -rf ${openssl_fname} ${openssl_fname}.tar.gz +} + + +function do_curl_build { + LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null + make > /dev/null + make install > /dev/null +} + + +function build_curl { + local curl_fname=$1 + check_var ${curl_fname} + local curl_sha256=$2 + check_var ${curl_sha256} + check_var ${CURL_DOWNLOAD_URL} + curl -sLO ${CURL_DOWNLOAD_URL}/${curl_fname}.tar.bz2 + check_sha256sum ${curl_fname}.tar.bz2 ${curl_sha256} + tar -jxf ${curl_fname}.tar.bz2 + (cd ${curl_fname} && do_curl_build) + rm -rf ${curl_fname} ${curl_fname}.tar.bz2 +} + + +function do_standard_install { + ./configure > /dev/null + make > /dev/null + make install > /dev/null +} + + +function build_autoconf { + local autoconf_fname=$1 + check_var ${autoconf_fname} + local autoconf_sha256=$2 + check_var ${autoconf_sha256} + check_var ${AUTOCONF_DOWNLOAD_URL} + curl -sLO ${AUTOCONF_DOWNLOAD_URL}/${autoconf_fname}.tar.gz + check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256} + tar -zxf ${autoconf_fname}.tar.gz + (cd ${autoconf_fname} && do_standard_install) + rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz +} diff --git a/tools/dockerfile/build_scripts/install_gcc.sh b/tools/dockerfile/build_scripts/install_gcc.sh new file mode 100644 index 00000000000..d7db7578133 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_gcc.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -ex + +if [ -f "/etc/redhat-release" ];then + lib_so_5=/usr/lib64/libstdc++.so.6 + lib_so_6=/usr/lib64/libstdc++.so.6 + lib_path=/usr/lib64 +else + lib_so_5=/usr/lib/x86_64-linux-gnu/libstdc++.so.5 + lib_so_6=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 + lib_path=/usr/lib/x86_64-linux-gnu +fi + +if [ "$1" == "gcc82" ]; then + wget https://paddle-ci.gz.bcebos.com/gcc-8.2.0.tar.xz + tar -xvf gcc-8.2.0.tar.xz && \ + cd gcc-8.2.0 && \ + unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ + ./contrib/download_prerequisites && \ + cd .. && mkdir temp_gcc82 && cd temp_gcc82 && \ + ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ + make -j8 && make install + cd .. && rm -rf temp_gcc82 + cp ${lib_so_6} ${lib_so_6}.bak && rm -f ${lib_so_6} && + ln -s /usr/local/gcc-8.2/lib64/libgfortran.so.5 ${lib_so_5} && \ + ln -s /usr/local/gcc-8.2/lib64/libstdc++.so.6 ${lib_so_6} && \ + cp /usr/local/gcc-8.2/lib64/libstdc++.so.6.0.25 ${lib_path} +fi diff --git a/tools/dockerfile/build_scripts/install_nccl2.sh b/tools/dockerfile/build_scripts/install_nccl2.sh new file mode 100644 index 00000000000..6307a52edd1 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_nccl2.sh @@ -0,0 +1,29 @@ +#!/bin/bash +VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| sed "s/release //") +if [ "$VERSION" == "10.0" ]; then + DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" +elif [ "$VERSION" == "10.1" ]; then + DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" +elif [ "$VERSION" == "9.0" ]; then + DEB="nccl-repo-ubuntu1604-2.3.7-ga-cuda9.0_1-1_amd64.deb" +else + DEB="nccl-repo-ubuntu1604-2.1.15-ga-cuda8.0_1-1_amd64.deb" +fi + +URL="http://nccl2-deb.gz.bcebos.com/$DEB" + +DIR="/nccl2" +mkdir -p $DIR +# we cached the nccl2 deb package in BOS, so we can download it with wget +# install nccl2: http://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html#down +wget -O $DIR/$DEB $URL + +cd $DIR && ar x $DEB && tar xf data.tar.xz +DEBS=$(find ./var/ -name "*.deb") +for sub_deb in $DEBS; do + echo $sub_deb + ar x $sub_deb && tar xf data.tar.xz +done +mv -f usr/include/nccl.h /usr/local/include/ +mv -f usr/lib/x86_64-linux-gnu/libnccl* /usr/local/lib/ +rm -rf $DIR diff --git a/tools/dockerfile/build_scripts/install_trt.sh b/tools/dockerfile/build_scripts/install_trt.sh new file mode 100644 index 00000000000..70297042bc6 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_trt.sh @@ -0,0 +1,16 @@ +#!/bin/bash +VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| sed "s/release //") + +if [[ "$VERSION" == "10.1" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.1-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda10.1-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda10.1-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda10.1-cudnn7/lib/* /usr/lib/ +elif [[ "$VERSION" == "10.0" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.0-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda10.0-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda10.0-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda10.0-cudnn7/lib/* /usr/lib/ +elif [[ "$VERSION" == "9.0" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda9.0-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda9.0-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda9.0-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda9.0-cudnn7/lib/* /usr/lib/ +fi diff --git a/tools/dockerfile/build_scripts/manylinux1-check.py b/tools/dockerfile/build_scripts/manylinux1-check.py new file mode 100644 index 00000000000..0d1a6df4eec --- /dev/null +++ b/tools/dockerfile/build_scripts/manylinux1-check.py @@ -0,0 +1,70 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Logic copied from PEP 513 + + +def is_manylinux1_compatible(): + # Only Linux, and only x86-64 / i686 + from distutils.util import get_platform + if get_platform() not in ["linux-x86_64", "linux-i686"]: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. + return have_compatible_glibc(2, 5) + + +def have_compatible_glibc(major, minimum_minor): + import ctypes + + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return False + + # Call gnu_get_libc_version, which returns a string like "2.5". + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + # Parse string and check against requested version. + version = [int(piece) for piece in version_str.split(".")] + assert len(version) == 2 + if major != version[0]: + return False + if minimum_minor > version[1]: + return False + return True + + +import sys +if is_manylinux1_compatible(): + print("%s is manylinux1 compatible" % (sys.executable, )) + sys.exit(0) +else: + print("%s is NOT manylinux1 compatible" % (sys.executable, )) + sys.exit(1) diff --git a/tools/dockerfile/build_scripts/python-tag-abi-tag.py b/tools/dockerfile/build_scripts/python-tag-abi-tag.py new file mode 100644 index 00000000000..0364ab3659e --- /dev/null +++ b/tools/dockerfile/build_scripts/python-tag-abi-tag.py @@ -0,0 +1,21 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Utility script to print the python tag + the abi tag for a Python +# See PEP 425 for exactly what these are, but an example would be: +# cp27-cp27mu + +from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag + +print("{0}{1}-{2}".format(get_abbr_impl(), get_impl_ver(), get_abi_tag())) diff --git a/tools/dockerfile/build_scripts/ssl-check.py b/tools/dockerfile/build_scripts/ssl-check.py new file mode 100644 index 00000000000..afef2812f3f --- /dev/null +++ b/tools/dockerfile/build_scripts/ssl-check.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cf. https://github.com/pypa/manylinux/issues/53 + +GOOD_SSL = "https://google.com" +BAD_SSL = "https://self-signed.badssl.com" + +import sys + +print("Testing SSL certificate checking for Python:", sys.version) + +if (sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4)): + print("This version never checks SSL certs; skipping tests") + sys.exit(0) + +if sys.version_info[0] >= 3: + from urllib.request import urlopen + EXC = OSError +else: + from urllib import urlopen + EXC = IOError + +print("Connecting to %s should work" % (GOOD_SSL, )) +urlopen(GOOD_SSL) +print("...it did, yay.") + +print("Connecting to %s should fail" % (BAD_SSL, )) +try: + urlopen(BAD_SSL) + # If we get here then we failed: + print("...it DIDN'T!!!!!11!!1one!") + sys.exit(1) +except EXC: + print("...it did, yay.") diff --git a/tools/dockerfile/centos6_manylinux.sh b/tools/dockerfile/centos6_manylinux.sh new file mode 100755 index 00000000000..ea9c8a7bf36 --- /dev/null +++ b/tools/dockerfile/centos6_manylinux.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -xe + +REPO="${REPO:-paddledocker}" + +function make_cuda9cudnn7(){ + sed 's//9.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp +} + + +function make_cuda10cudnn7() { + sed 's//10.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp +} + + +function make_cuda101cudnn7() { + sed 's//10.1-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#COPY build_scripts /build_scripts#COPY build_scripts /build_scripts \nRUN bash build_scripts/install_gcc.sh gcc82 \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH#g" Dockerfile.tmp +} + + +function main() { + local CMD=$1 + case $CMD in + cuda9cudnn7) + make_cuda9cudnn7 + ;; + cuda10cudnn7) + make_cuda10cudnn7 + ;; + cuda101cudnn7) + make_cuda101cudnn7 + ;; + *) + echo "Make dockerfile error, Without this paramet." + exit 1 + ;; + esac +} + +main $@ diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh new file mode 100644 index 00000000000..f088c5728a5 --- /dev/null +++ b/tools/dockerfile/ci_dockerfile.sh @@ -0,0 +1,37 @@ +#!/bin/bash +function make_ubuntu_dockerfile(){ + dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16" + sed 's//10.1-cudnn7-devel-ubuntu16.04/g' ./Dockerfile.ubuntu >${dockerfile_name} + sed -i 's#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g' ${dockerfile_name} + sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ + tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} + dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'` + sed -i 's##WORKDIR /usr/bin \ + COPY tools/dockerfile/build_scripts /build_scripts \ + RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \ + RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' ${dockerfile_name} + +} + + +function make_centos_dockerfile(){ + dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6" + sed 's//9.0-cudnn7-devel-centos6/g' Dockerfile.centos >${dockerfile_name} + sed -i 's#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g' ${dockerfile_name} + dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'` + sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ + tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} +} + + +function main() { + make_ubuntu_dockerfile + make_centos_dockerfile +} + +main $@ diff --git a/tools/dockerfile/icode.sh b/tools/dockerfile/icode.sh new file mode 100755 index 00000000000..da3ffb8c77d --- /dev/null +++ b/tools/dockerfile/icode.sh @@ -0,0 +1,109 @@ +#!/bin/bash + + +function install_gcc(){ + sed -i 's##RUN apt-get update \ + WORKDIR /usr/bin \ + RUN apt install -y gcc-4.8 g++-4.8 \&\& cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \&\& ln -s gcc-4.8 gcc \&\& ln -s g++-4.8 g++ #g' $1 +} + + +function install_gcc8(){ + sed -i 's##WORKDIR /usr/bin \ + COPY tools/dockerfile/build_scripts /build_scripts \ + RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \ + RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' $1 +} + + +function centos_gcc8(){ + sed -i "s#COPY build_scripts /build_scripts#COPY build_scripts /build_scripts \nRUN bash build_scripts/install_gcc.sh gcc82 \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH#g" $1 +} + + +function fix_https(){ + sed -i 's#https#http#g' /etc/apt/sources.list.d/nvidia-ml.list + sed -i 's#https#http#g' /etc/apt/sources.list.d/cuda.list +} + + +function all_change(){ + sed -i 's#ENV HOME /root#ENV HOME /root\nENV DEBIAN_FRONTEND=noninteractive#g' Dockerfile.ubuntu +} + +function centos() { + # centos6 + sed 's##8.0-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_cpu_runtime.dockerfile + sed 's##9.0-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_gpu_cuda9.0_cudnn7_single_gpu_runtime.dockerfile + sed 's##9.1-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_gpu_cuda9.1_cudnn7_single_gpu_runtime.dockerfile + sed 's##9.2-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_gpu_cuda9.2_cudnn7_single_gpu_runtime.dockerfile + sed 's##10.0-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_gpu_cuda10.0_cudnn7_single_gpu_runtime.dockerfile + sed 's##10.1-cudnn7-devel-centos6#g' Dockerfile.centos >test/centos_6_gpu_cuda10.1_cudnn7_single_gpu_runtime.dockerfile + centos_gcc8 "test/centos_6_gpu_cuda10.1_cudnn7_single_gpu_runtime.dockerfile" + + # centos7 + sed 's##8.0-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_cpu_runtime.dockerfile + sed 's##9.0-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_gpu_cuda9.0_cudnn7_single_gpu_runtime.dockerfile + sed 's##9.1-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_gpu_cuda9.1_cudnn7_single_gpu_runtime.dockerfile + sed 's##9.2-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_gpu_cuda9.2_cudnn7_single_gpu_runtime.dockerfile + sed 's##10.0-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_gpu_cuda10.0_cudnn7_single_gpu_runtime.dockerfile + sed 's##10.1-cudnn7-devel-centos7#g' Dockerfile.centos >test/centos_7_gpu_cuda10.1_cudnn7_single_gpu_runtime.dockerfile + centos_gcc8 "test/centos_7_gpu_cuda10.1_cudnn7_single_gpu_runtime.dockerfile" +} + + +function ubuntu() { + # ubuntu 14 + sed 's##8.0-cudnn7-devel-ubuntu14.04#g' Dockerfile.ubuntu >test/ubuntu_1404_cpu.dockerfile + install_gcc "test/ubuntu_1404_cpu.dockerfile" + sed 's##10.0-cudnn7-devel-ubuntu14.04#g' Dockerfile.ubuntu >test/ubuntu_1404_gpu_cuda10.0_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1404_gpu_cuda10.0_cudnn7_runtime.dockerfile" + sed 's##10.1-cudnn7-devel-ubuntu14.04#g' Dockerfile.ubuntu >test/ubuntu_1404_gpu_cuda10.1_cudnn7_runtime.dockerfile + install_gcc8 "test/ubuntu_1404_gpu_cuda10.1_cudnn7_runtime.dockerfile" + + # ubuntu 16 + sed 's##8.0-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_cpu.dockerfile + install_gcc "test/ubuntu_1604_cpu.dockerfile" + sed 's##9.0-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_gpu_cuda9.0_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1604_gpu_cuda9.0_cudnn7_runtime.dockerfile" + sed 's##9.1-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_gpu_cuda9.1_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1604_gpu_cuda9.1_cudnn7_runtime.dockerfile" + sed 's##9.2-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_gpu_cuda9.2_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1604_gpu_cuda9.2_cudnn7_runtime.dockerfile" + sed 's##10.0-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_gpu_cuda10.0_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1604_gpu_cuda10.0_cudnn7_runtime.dockerfile" + sed 's##10.1-cudnn7-devel-ubuntu16.04#g' Dockerfile.ubuntu >test/ubuntu_1604_gpu_cuda10.1_cudnn7_runtime.dockerfile + install_gcc8 "test/ubuntu_1604_gpu_cuda10.1_cudnn7_runtime.dockerfile" + + # ubuntu 18 + sed 's##8.0-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_cpu.dockerfile + install_gcc "test/ubuntu_1804_cpu.dockerfile" + sed 's##9.0-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_gpu_cuda9.0_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1804_gpu_cuda9.0_cudnn7_runtime.dockerfile" + sed 's##9.1-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_gpu_cuda9.1_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1804_gpu_cuda9.1_cudnn7_runtime.dockerfile" + sed 's##9.2-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_gpu_cuda9.2_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1804_gpu_cuda9.2_cudnn7_runtime.dockerfile" + sed 's##10.0-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_gpu_cuda10.0_cudnn7_runtime.dockerfile + install_gcc "test/ubuntu_1804_gpu_cuda10.0_cudnn7_runtime.dockerfile" + sed 's##10.1-cudnn7-devel-ubuntu18.04#g' Dockerfile.ubuntu >test/ubuntu_1804_gpu_cuda10.1_cudnn7_runtime.dockerfile + install_gcc8 "test/ubuntu_1804_gpu_cuda10.1_cudnn7_runtime.dockerfile" +} + + +function main() { + if [ ! -d "test" ];then + mkdir test + fi + all_change + centos + ubuntu +} + + +main diff --git a/tools/dockerfile/ubuntu16_dev.sh b/tools/dockerfile/ubuntu16_dev.sh new file mode 100755 index 00000000000..e7827b6598e --- /dev/null +++ b/tools/dockerfile/ubuntu16_dev.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +docker_name=$1 + +function ref_whl(){ + if [[ ${WITH_GPU} == "ON" ]]; then + ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR} + install_gpu="_gpu" + else + ref_gpu="cpu" + install_gpu="" + fi + + if [[ ${WITH_MKL} == "ON" ]]; then + ref_mkl=mkl + else + ref_mkl=openblas + fi + + if [[ ${gcc_version} == "8.2.0" ]];then + ref_gcc=_gcc8.2 + fi + + ref_web="https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}${ref_gcc}" + + if [[ ${PADDLE_BRANCH} == "0.0.0" && ${WITH_GPU} == "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + else + ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + fi + + if [[ ${PADDLE_BRANCH} != "0.0.0" && ${WITH_GPU} == "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl + else + ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + fi +} + + +function install_whl(){ + dockerfile_line=`wc -l Dockerfile.tmp|awk '{print $1}'` + sed -i "${dockerfile_line}i RUN wget ${ref_web}/${ref_paddle_whl} && pip install ${ref_paddle_whl} && rm -f ${ref_paddle_whl}" Dockerfile.tmp + sed -i "${dockerfile_line}i RUN wget ${ref_web}/${ref_paddle3_whl} && pip3.5 install ${ref_paddle3_whl} && rm -f ${ref_paddle3_whl}" Dockerfile.tmp + sed -i "${dockerfile_line}i RUN wget ${ref_web}/${ref_paddle36_whl} && pip3.6 install ${ref_paddle36_whl} && rm -f ${ref_paddle36_whl}" Dockerfile.tmp + sed -i "${dockerfile_line}i RUN wget ${ref_web}/${ref_paddle37_whl} && pip3.7 install ${ref_paddle37_whl} && rm -f ${ref_paddle37_whl}" Dockerfile.tmp +} + +function install_gcc(){ + if [ "${gcc_version}" == "8.2.0" ];then + sed -i 's##WORKDIR /usr/bin \ + COPY tools/dockerfile/build_scripts /build_scripts \ + RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \ + RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \ + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' Dockerfile.tmp + else + sed -i 's##RUN apt-get update \ + WORKDIR /usr/bin \ + RUN apt install -y gcc-4.8 g++-4.8 \&\& cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \&\& ln -s gcc-4.8 gcc \&\& ln -s g++-4.8 g++ #g' Dockerfile.tmp + fi +} + + + +function make_dockerfile(){ + sed "s//${docker_name}/g" tools/dockerfile/Dockerfile.ubuntu >Dockerfile.tmp +} + +function main(){ + make_dockerfile + install_gcc + ref_whl + install_whl +} + +main $@ -- GitLab