From f090066e853a1a9d0eca1e513d5e27f17c5bbc87 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Fri, 15 Jan 2021 10:28:48 +0800 Subject: [PATCH] Clean dockerfiles (#30401) * clean dockerfile * clean dockerfile --- Dockerfile | 219 ---------------- ...tos6_manylinux.sh => centos7_manylinux.sh} | 0 tools/manylinux1/Dockerfile.CI35 | 1 - tools/manylinux1/Dockerfile.CI35-GCC8 | 73 ------ tools/manylinux1/Dockerfile.Inference | 1 - .../Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 | 228 ---------------- ...Dockerfile.cuda10_cudnn7_gcc8_py35_centos6 | 82 ------ .../Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 | 245 ------------------ .../Dockerfile.cuda10_ubuntu18_cinn | 152 ----------- ...Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 | 71 ----- tools/manylinux1/Dockerfile.x64 | 63 ----- tools/manylinux1/README.md | 65 ----- tools/manylinux1/build_all.sh | 31 --- tools/manylinux1/build_scripts/build.sh | 163 ------------ tools/manylinux1/build_scripts/build_utils.sh | 195 -------------- .../manylinux1/build_scripts/install_nccl2.sh | 33 --- .../build_scripts/manylinux1-check.py | 70 ----- .../build_scripts/python-tag-abi-tag.py | 21 -- tools/manylinux1/build_scripts/ssl-check.py | 46 ---- 19 files changed, 1759 deletions(-) delete mode 100644 Dockerfile rename tools/dockerfile/{centos6_manylinux.sh => centos7_manylinux.sh} (100%) delete mode 120000 tools/manylinux1/Dockerfile.CI35 delete mode 100644 tools/manylinux1/Dockerfile.CI35-GCC8 delete mode 120000 tools/manylinux1/Dockerfile.Inference delete mode 100644 tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 delete mode 100644 tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_py35_centos6 delete mode 100644 tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 delete mode 100755 tools/manylinux1/Dockerfile.cuda10_ubuntu18_cinn delete mode 100644 tools/manylinux1/Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 delete mode 100644 tools/manylinux1/Dockerfile.x64 delete mode 100644 tools/manylinux1/README.md delete mode 100755 tools/manylinux1/build_all.sh delete mode 100644 tools/manylinux1/build_scripts/build.sh delete mode 100755 tools/manylinux1/build_scripts/build_utils.sh delete mode 100644 tools/manylinux1/build_scripts/install_nccl2.sh delete mode 100644 tools/manylinux1/build_scripts/manylinux1-check.py delete mode 100644 tools/manylinux1/build_scripts/python-tag-abi-tag.py delete mode 100644 tools/manylinux1/build_scripts/ssl-check.py diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index daab4340e35..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,219 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -# When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04 -MAINTAINER PaddlePaddle Authors - -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG WITH_GPU -ARG WITH_AVX - -ENV WITH_GPU=${WITH_GPU:-ON} -ENV WITH_AVX=${WITH_AVX:-ON} - -ENV HOME /root -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -# Prepare packages for Python -RUN apt-get update && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev - -# Downgrade gcc&&g++ -RUN apt-get update -WORKDIR /usr/bin -RUN apt install -y gcc-4.8 g++-4.8 -RUN cp gcc gcc.bak -RUN cp g++ g++.bak -RUN rm gcc -RUN rm g++ -RUN ln -s gcc-4.8 gcc -RUN ln -s g++-4.8 g++ - -# Install cmake3.16.0 -RUN mkdir -p /root/cmake_build && wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && \ - tar -zxvf cmake-3.16.0.tar.gz && rm cmake-3.16.0.tar.gz && \ - cd cmake-3.16.0 && ./bootstrap > /dev/null && \ - make -j8 > /dev/null && make install > /dev/null && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -ENV PATH=/usr/local/bin:$PATH - -RUN rm -r /root/cmake_build - -# Install Python3.6 -RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ - tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ - ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ - wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ - tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - -# Install Python3.7 -RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ - tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - -RUN rm -r /root/python_build - -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - libnccl2=2.1.2-1+cuda8.0 libnccl-dev=2.1.2-1+cuda8.0 \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool ccache && \ - apt-get clean -y - -# Install Python2.7.15 to replace original python -WORKDIR /home -ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz -WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/bin:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc -ENV PATH=/usr/local/python2.7.15/include:${PATH} -ENV PATH=/usr/local/python2.7.15/bin:${PATH} -ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} -ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python -WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip -WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install -WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz -RUN tar -zxvf pip-18.0.tar.gz -WORKDIR pip-18.0 -RUN python setup.py install - -WORKDIR /home -RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ - rm -r Python-$version setuptools-40.6.2 pip-18.0 - -# Install Go and glide -RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# Install TensorRT -# following TensorRT.tar.gz is not the default official one, we do two miny changes: -# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, -# and its size is only one-third of the official one. -# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. -# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. - -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT/include/* /usr/include/ && \ - cp -rf /usr/local/TensorRT/lib/* /usr/lib/ - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - - -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - -RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip --no-cache-dir install 'ipykernel==4.6.0' - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz && tar -xzf openmpi-1.4.5.tar.gz && \ - cd openmpi-1.4.5 && ./configure --prefix=/usr/local && make all -j8 && make install -j8 && \ - export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ - rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ - apt-get install libprotobuf-dev -y - -# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. -# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa -# So install a newer version here. -RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ - dpkg -i patchelf_0.10-2_amd64.deb - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc -EXPOSE 22 diff --git a/tools/dockerfile/centos6_manylinux.sh b/tools/dockerfile/centos7_manylinux.sh similarity index 100% rename from tools/dockerfile/centos6_manylinux.sh rename to tools/dockerfile/centos7_manylinux.sh diff --git a/tools/manylinux1/Dockerfile.CI35 b/tools/manylinux1/Dockerfile.CI35 deleted file mode 120000 index 6f5de91a12b..00000000000 --- a/tools/manylinux1/Dockerfile.CI35 +++ /dev/null @@ -1 +0,0 @@ -Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 \ No newline at end of file diff --git a/tools/manylinux1/Dockerfile.CI35-GCC8 b/tools/manylinux1/Dockerfile.CI35-GCC8 deleted file mode 100644 index e0c5d16bad6..00000000000 --- a/tools/manylinux1/Dockerfile.CI35-GCC8 +++ /dev/null @@ -1,73 +0,0 @@ -# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in -# order to satisfy the build of capnproto library (a nupic.core dependency), -# which requires some headers and symbols not present on CentOS-5 (e.g., -# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See -# https://github.com/sandstorm-io/capnproto/issues/350. -FROM nvidia/cuda:10.1-cudnn7-devel-centos6 -MAINTAINER Numenta, based on the ManyLinux project - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz gettext-devel expat-devel cpio perl curl-devel - -COPY tools/manylinux1/build_scripts /build_scripts -RUN bash build_scripts/build.sh && bash build_scripts/install_nccl2.sh && rm -rf build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -# gcc8.2 -RUN wget -q https://paddle-docker-tar.bj.bcebos.com/home/users/tianshuo/bce-python-sdk-0.8.27/gcc-8.2.0.tar.xz && \ - tar -xvf gcc-8.2.0.tar.xz && \ - cd gcc-8.2.0 && \ - sed -i 's#ftp://gcc.gnu.org/pub/gcc/infrastructure/#https://paddle-ci.gz.bcebos.com/#g' ./contrib/download_prerequisites && \ - unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ - ./contrib/download_prerequisites && \ - cd .. && mkdir temp_gcc82 && cd temp_gcc82 && \ - ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ - make -j8 && make install - -ENV PATH=/usr/local/gcc-8.2/bin:$PATH -RUN rm -rf /temp_gcc82 && rm -rf /gcc-8.2.0.tar.xz && rm -rf /gcc-8.2.0 - -# git 2.7.1 -RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \ - tar -xvf git-2.17.1.tar.gz && \ - cd git-2.17.1 && \ - ./configure --prefix=/usr/local && \ - make -j8 && make install - -# protobuf 3.6.1 -RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \ - tar xzf protobuf-cpp-3.6.1.tar.gz && \ - cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz - -RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install setuptools -U - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install -r /root/requirements.txt && \ - rm -rf /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install pre-commit 'ipython==5.3.0' - -RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ - cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz - -CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/tools/manylinux1/Dockerfile.Inference b/tools/manylinux1/Dockerfile.Inference deleted file mode 120000 index 0ba180b894b..00000000000 --- a/tools/manylinux1/Dockerfile.Inference +++ /dev/null @@ -1 +0,0 @@ -Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 \ No newline at end of file diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 deleted file mode 100644 index e996ec0e765..00000000000 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 +++ /dev/null @@ -1,228 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -# When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04 -MAINTAINER PaddlePaddle Authors - -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG WITH_GPU -ARG WITH_AVX - -ENV WITH_GPU=${WITH_GPU:-ON} -ENV WITH_AVX=${WITH_AVX:-ON} - -ENV HOME /root -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -# Prepare packages for Python -RUN apt-get update && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev - -# Downgrade gcc&&g++ -RUN apt-get update -WORKDIR /usr/bin -RUN apt install -y gcc-4.8 g++-4.8 -RUN cp gcc gcc.bak -RUN cp g++ g++.bak -RUN rm gcc -RUN rm g++ -RUN ln -s gcc-4.8 gcc -RUN ln -s g++-4.8 g++ - -# Install Python3.6 -RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ - tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ - ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ - wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ - tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - -# Install Python3.7 -RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ - tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null -RUN rm -r /root/python_build - -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - patchelf python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool && \ - apt-get clean -y - -# install cmake -WORKDIR /home -RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz -RUN tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz -RUN rm cmake-3.16.0-Linux-x86_64.tar.gz -ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH - -# Install Python2.7.15 to replace original python -WORKDIR /home -ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz -WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc -ENV PATH=/usr/local/python2.7.15/include:${PATH} -ENV PATH=/usr/local/python2.7.15/bin:${PATH} -ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} -ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python -WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip -WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install -WORKDIR /home - -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz -WORKDIR pip-18.0 -RUN python setup.py install && \ - python3.7 setup.py install && \ - python3.6 setup.py install && \ - python3 setup.py install - -WORKDIR /home -RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ - rm -r Python-$version setuptools-40.6.2 pip-18.0 - -# Install Go and glide -RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# Install TensorRT -# following TensorRT.tar.gz is not the default official one, we do two miny changes: -# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, -# and its size is only one-third of the official one. -# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. -# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. - -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-6.0.1.5.Ubuntu-16.04.x86_64-gnu.cuda-10.1.cudnn7.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-6.0.1.5.Ubuntu-16.04.x86_64-gnu.cuda-10.1.cudnn7.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT-6.0.1.5/include/* /usr/include/ && cp -rf /usr/local/TensorRT-6.0.1.5/lib/* /usr/lib/ - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - -# specify sphinx version as 1.5.6 and remove -U option for [pip install -U -# sphinx-rtd-theme] since -U option will cause sphinx being updated to newest -# version(1.7.1 for now), which causes building documentation failed. -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.6 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.7 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark - -RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://paddle-ci.gz.bcebos.com/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz && tar -xzf openmpi-1.4.5.tar.gz && \ - cd openmpi-1.4.5 && ./configure --prefix=/usr/local && make all -j8 && make install -j8 && \ - export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ - rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ - apt-get install libprotobuf-dev -y - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ - tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/ - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc -EXPOSE 22 diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_py35_centos6 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_py35_centos6 deleted file mode 100644 index 30f84141745..00000000000 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_py35_centos6 +++ /dev/null @@ -1,82 +0,0 @@ -# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in -# order to satisfy the build of capnproto library (a nupic.core dependency), -# which requires some headers and symbols not present on CentOS-5 (e.g., -# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See -# https://github.com/sandstorm-io/capnproto/issues/350. -FROM nvidia/cuda:10.1-cudnn7-devel-centos6 -MAINTAINER Numenta, based on the ManyLinux project - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel tk-devel tkinter libtool xz graphviz gettext-devel -RUN yum install -y curl-devel - -COPY tools/manylinux1/build_scripts ./build_scripts - -RUN bash build_scripts/build.sh && bash build_scripts/install_nccl2.sh && rm -rf build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -# gcc8.2 -RUN wget -q https://paddle-docker-tar.bj.bcebos.com/home/users/tianshuo/bce-python-sdk-0.8.27/gcc-8.2.0.tar.xz && \ - tar -xvf gcc-8.2.0.tar.xz && \ - cd gcc-8.2.0 && \ - sed -i 's#ftp://gcc.gnu.org/pub/gcc/infrastructure/#https://paddle-ci.gz.bcebos.com/#g' ./contrib/download_prerequisites && \ - unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ - ./contrib/download_prerequisites && \ - cd .. && mkdir temp_gcc82 && cd temp_gcc82 && \ - ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ - make -j8 && make install - -ENV PATH=/usr/local/gcc-8.2/bin:$PATH -RUN rm -rf /temp_gcc82 && rm -rf /gcc-8.2.0.tar.xz && rm -rf /gcc-8.2.0 - -# git 2.17.1 -RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \ - tar -xvf git-2.17.1.tar.gz && \ - cd git-2.17.1 && \ - ./configure --prefix=/usr/local && \ - make -j8 && make install - -# protobuf 3.6.1 -RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \ - tar xzf protobuf-cpp-3.6.1.tar.gz && \ - cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install setuptools -U - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install -r /root/requirements.txt && \ - rm -rf /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install pre-commit 'ipython==5.3.0' - -RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ - cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz - -CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 deleted file mode 100644 index 8a557a588d5..00000000000 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 +++ /dev/null @@ -1,245 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -# When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04 -MAINTAINER PaddlePaddle Authors - -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG WITH_GPU -ARG WITH_AVX - -ENV WITH_GPU=${WITH_GPU:-ON} -ENV WITH_AVX=${WITH_AVX:-ON} - -ENV HOME /root -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -ENV PATH=/usr/local/gcc-8.2/bin:$PATH -RUN rm -rf /temp_gcc82 && rm -rf /gcc-8.2.0.tar.xz && rm -rf /gcc-8.2.0 - -# Prepare packages for Python -# apt install openmpi: In order to run a single test of pslib coverage -RUN apt-get update && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev - -RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ - tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ - rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 - -# gcc8.2 -RUN wget -q https://paddle-docker-tar.bj.bcebos.com/home/users/tianshuo/bce-python-sdk-0.8.27/gcc-8.2.0.tar.xz && \ - tar -xvf gcc-8.2.0.tar.xz && \ - cd gcc-8.2.0 && \ - sed -i 's#ftp://gcc.gnu.org/pub/gcc/infrastructure/#https://paddle-ci.gz.bcebos.com/#g' ./contrib/download_prerequisites && \ - unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ - ./contrib/download_prerequisites && \ - cd .. && mkdir temp_gcc82 && cd temp_gcc82 && \ - ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ - make -j8 && make install - -ENV PATH=/usr/local/gcc-8.2/bin:$PATH -ENV LD_LIBRARY_PATH=/usr/local/gcc-8.2/lib64:$LD_LIBRARY_PATH -RUN rm -rf /temp_gcc82 && rm -rf /gcc-8.2.0.tar.xz && rm -rf /gcc-8.2.0 - -# Install Python3.6 -RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ - tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ - ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ - wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ - tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - -# Install Python3.7 -RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ - tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null -RUN rm -r /root/python_build - -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - patchelf python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool && \ - apt-get clean -y - -# install cmake -WORKDIR /home -RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz -RUN tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz -RUN rm cmake-3.16.0-Linux-x86_64.tar.gz -ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH - -# Install Python2.7.15 to replace original python -WORKDIR /home -ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz -WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc -ENV PATH=/usr/local/python2.7.15/include:${PATH} -ENV PATH=/usr/local/python2.7.15/bin:${PATH} -ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} -ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python -WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip -WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install -WORKDIR /home - -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz -WORKDIR pip-18.0 -RUN python setup.py install && \ - python3.7 setup.py install && \ - python3.6 setup.py install && \ - python3 setup.py install - -WORKDIR /home -RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ - rm -r Python-$version setuptools-40.6.2 pip-18.0 - -# Install Go and glide -RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# Install TensorRT -# following TensorRT.tar.gz is not the default official one, we do two miny changes: -# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, -# and its size is only one-third of the official one. -# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. -# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. - -RUN wget -q --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb && \ - dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb && \ - apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 --allow-change-held-packages - -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-6.0.1.5.Ubuntu-16.04.x86_64-gnu.cuda-10.1.cudnn7.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-6.0.1.5.Ubuntu-16.04.x86_64-gnu.cuda-10.1.cudnn7.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT-6.0.1.5/include/* /usr/include/ && cp -rf /usr/local/TensorRT-6.0.1.5/lib/* /usr/lib/ - -# Install patchelf-0.10 -RUN wget https://paddle-ci.gz.bcebos.com/patchelf-0.10.tar.gz && \ - tar -zxvf patchelf-0.10.tar.gz && cd patchelf-0.10 && \ - ./configure && make -j8 && make install - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - -# specify sphinx version as 1.5.6 and remove -U option for [pip install -U -# sphinx-rtd-theme] since -U option will cause sphinx being updated to newest -# version(1.7.1 for now), which causes building documentation failed. -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.6 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.7 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark - -RUN pip3 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3 --no-cache-dir install ipykernel==4.6.0 jupyter==1.0.0 && \ - pip3.6 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.6 --no-cache-dir install ipykernel==4.6.0 jupyter==1.0.0 && \ - pip3.7 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.7 --no-cache-dir install ipykernel==4.6.0 jupyter==1.0.0 && \ - pip --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip --no-cache-dir install ipykernel==4.6.0 - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://paddle-ci.gz.bcebos.com/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN apt-get install libprotobuf-dev -y - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ - tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/ - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc -EXPOSE 22 diff --git a/tools/manylinux1/Dockerfile.cuda10_ubuntu18_cinn b/tools/manylinux1/Dockerfile.cuda10_ubuntu18_cinn deleted file mode 100755 index 964f082b561..00000000000 --- a/tools/manylinux1/Dockerfile.cuda10_ubuntu18_cinn +++ /dev/null @@ -1,152 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -# When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_GPU -ARG WITH_AVX - -ENV WITH_GPU=${WITH_GPU:-ON} -ENV WITH_AVX=${WITH_AVX:-ON} -ENV DEBIAN_FRONTEND=noninteractive - -ENV HOME /root -# Add bash enhancements -COPY paddle/scripts/docker/root/ /root/ - -RUN apt-get update && \ - apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update && \ - apt-get install -y curl wget vim git unzip unrar tar xz-utils bzip2 gzip \ - coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev - - -# Downgrade gcc&&g++ -WORKDIR /usr/bin - RUN apt-get update --fix-missing - COPY tools/dockerfile/build_scripts /build_scripts - RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts - RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ - ENV PATH=/usr/local/gcc-8.2/bin:$PATH - -RUN apt-get update && \ - apt-get install -y python2.7 python2.7-dev \ - python3.5 python3.5-dev \ - python3.6 python3.6-dev \ - python3.7 python3.7-dev \ - python3.8 python3.8-dev && \ - curl https://bootstrap.pypa.io/ez_setup.py -o - | python2.7 && easy_install pip && \ - curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.5 && easy_install pip && \ - curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && easy_install pip && \ - curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.7 && easy_install pip && \ - curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.8 && easy_install pip && \ - rm /usr/bin/python && ln -s /usr/bin/python2.7 /usr/bin/python && \ - rm /usr/bin/python3 && ln -s /usr/bin/python3.5 /usr/bin/python3 && \ - rm /usr/local/bin/pip && ln -s /usr/local/bin/pip2.7 /usr/local/bin/pip && \ - rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip3 - - -# install cmake -WORKDIR /home -RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz -ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH - - -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ - tar -xzf binutils-2.33.1.tar.gz && \ - cd binutils-2.33.1 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz - - -# Install Go and glide -RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip3 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.6 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.6 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.7 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.8 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip --no-cache-dir install ipykernel==4.6.0 wheel - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort && \ - pip3.6 --no-cache-dir install pylint pytest astroid isort && \ - pip3.7 --no-cache-dir install pylint pytest astroid isort && \ - pip3.8 --no-cache-dir install pylint pytest astroid isort && \ - pip --no-cache-dir install pylint pytest astroid isort - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt && \ - pip3.6 --no-cache-dir install -r /root/requirements.txt && \ - pip3.7 --no-cache-dir install -r /root/requirements.txt && \ - pip3.8 --no-cache-dir install -r /root/requirements.txt && \ - pip --no-cache-dir install -r /root/requirements.txt - - -# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. -# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa -# So install a newer version here. -RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ - dpkg -i patchelf_0.10-2_amd64.deb - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -#CMD source ~/.bashrc - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -# For CINN environment -RUN apt update --fix-missing -RUN apt-get install autoconf autogen -RUN apt-get install libtool -RUN apt-get install zlib1g-dev -RUN apt install libginac-dev -y -RUN apt install clang cmake -y -RUN python3 -m pip install numpy -RUN python3 -m pip install pybind11 - - -# Install LLVM -RUN echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/source.list -RUN echo "deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/source.list -RUN echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" >> /etc/apt/source.list -RUN echo "deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" >> /etc/apt/source.list -RUN ln -s /usr/bin/llvm-config-6.0 /usr/bin/llvm-config -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - - -RUN apt update -RUN apt install libclang-dev llvm-10 llvm-10-dev libclang-10-dev -y - - -EXPOSE 22 diff --git a/tools/manylinux1/Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 b/tools/manylinux1/Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 deleted file mode 100644 index 82918fca37f..00000000000 --- a/tools/manylinux1/Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 +++ /dev/null @@ -1,71 +0,0 @@ -# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in -# order to satisfy the build of capnproto library (a nupic.core dependency), -# which requires some headers and symbols not present on CentOS-5 (e.g., -# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See -# https://github.com/sandstorm-io/capnproto/issues/350. -FROM nvidia/cuda:9.0-cudnn7-devel-centos6 -MAINTAINER Numenta, based on the ManyLinux project - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel tk-devel tkinter libtool xz graphviz gettext-devel -RUN yum install -y curl-devel - -COPY tools/manylinux1/build_scripts ./build_scripts - -RUN bash build_scripts/build.sh && bash build_scripts/install_nccl2.sh && rm -rf build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -# git 2.17.1 -RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \ - tar -xvf git-2.17.1.tar.gz && \ - cd git-2.17.1 && \ - ./configure --prefix=/usr/local && \ - make -j8 && make install - -# protobuf 3.6.1 -RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \ - tar xzf protobuf-cpp-3.6.1.tar.gz && \ - cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install setuptools -U - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install -r /root/requirements.txt && \ - rm -rf /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install pre-commit 'ipython==5.3.0' - -RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ - cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz - -RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ - tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/ - -CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/tools/manylinux1/Dockerfile.x64 b/tools/manylinux1/Dockerfile.x64 deleted file mode 100644 index 7ad1b3554ab..00000000000 --- a/tools/manylinux1/Dockerfile.x64 +++ /dev/null @@ -1,63 +0,0 @@ -# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in -# order to satisfy the build of capnproto library (a nupic.core dependency), -# which requires some headers and symbols not present on CentOS-5 (e.g., -# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See -# https://github.com/sandstorm-io/capnproto/issues/350. -FROM nvidia/cuda: -MAINTAINER Numenta, based on the ManyLinux project - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz -COPY build_scripts /build_scripts -RUN bash build_scripts/build.sh && bash build_scripts/install_nccl2.sh && rm -rf build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -# for paddle -RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src - - -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH} - -# protobuf 3.6.1 -RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \ - tar xzf protobuf-cpp-3.6.1.tar.gz && \ - cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz - -RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install setuptools -U - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install -r /root/requirements.txt && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install -r /root/requirements.txt && \ - go get github.com/Masterminds/glide && \ - rm -rf /root/requirements.txt - -RUN LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27mu/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH} /opt/python/cp27-cp27m/bin/pip install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.5.1/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.6.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.7.0/bin/pip3 install pre-commit 'ipython==5.3.0' - -RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ - cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz - -CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/tools/manylinux1/README.md b/tools/manylinux1/README.md deleted file mode 100644 index 0e590504017..00000000000 --- a/tools/manylinux1/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# buildtools - -We release PaddlePaddle and PaddlePaddle Fluid as shared libraries, -which, we hope could be released as wheel packages on PyPI, so we need -to make sure that the build follows the -[manulinux1](https://www.python.org/dev/peps/pep-0513/) standard. - -The manylinux standard suggests building Python modules on an old -system, because that a module would anyway depend on some shared -libraries, and Linux's shared library standard states that those built -with newer version compilers cannot work with those with older -versions. The suggested building environment is as old as CentOS 5. -However, PaddlePaddle relies on CUDA, and the earlies version of -[CentOS works with CUDA is 6](https://hub.docker.com/r/nvidia/cuda/). -So, here we provide a Docker image based on CentOS 6 and CUDA for -building PaddlePaddle and making the release supports "as-manylinux as -possible." or "sufficiently many Linux" according to [this -discussion](https://mail.python.org/pipermail/wheel-builders/2016-July/000175.html). - -The build output of our Docker image includes multiple wheel files -- -some contain the CPU-only binary, some others support CUDA; some are -compatible with the cp27m Python ABI, some others with cp27. - -To build these wheels, please run the following commands: - -```bash -git clone https://github.com/paddlepaddle/paddle -cd paddle/tools/manylinux1 -REPO=[yourrepo] ./build_all.sh -``` - -## Build PaddlePaddle for the different Python ABIs - -Choose one of the following Python ABI and set the correct environment variables. - -- cp27-cp27m - - ```bash - export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:} - export PATH=/opt/python/cp27-cp27m/bin/:${PATH} - export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python - -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7 - -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so" - ``` - -- cp27-cp27mu - - ```bash - export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:} - export PATH=/opt/python/cp27-cp27mu/bin/:${PATH} - export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python - -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7 - -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so" - ``` - -And then add the `PYTHON_FLAGS` as your cmake flags: - -```bash -cmake .. - ${PYTHON_FLAGS} \ - -DWITH_GPU=OFF \ - ... -``` - -You can find more details about cmake flags at [here](http://www.paddlepaddle.org/docs/develop/documentation/fluid/en/build_and_install/build_from_source_en.html#appendix-build-options) diff --git a/tools/manylinux1/build_all.sh b/tools/manylinux1/build_all.sh deleted file mode 100755 index d9801417675..00000000000 --- a/tools/manylinux1/build_all.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -xe - -REPO="${REPO:-typhoon1986}" - -# NOTE: version matches are determined! -sed 's//7.5-cudnn5-devel-centos6/g' Dockerfile.x64 | \ -sed 's//NVCC_GENCODE="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_52,code=sm_52"/g'> Dockerfile.tmp -docker build -t ${REPO}/paddle_manylinux_devel:cuda7.5_cudnn5 -f Dockerfile.tmp . -docker push ${REPO}/paddle_manylinux_devel:cuda7.5_cudnn5 - -sed 's//8.0-cudnn5-devel-centos6/g' Dockerfile.x64 | \ -sed 's//NVCC_GENCODE="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_60,code=compute_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_62,code=sm_62"/g'> Dockerfile.tmp -docker build -t ${REPO}/paddle_manylinux_devel:cuda8.0_cudnn5 -f Dockerfile.tmp . -docker push ${REPO}/paddle_manylinux_devel:cuda8.0_cudnn5 - -sed 's//8.0-cudnn7-devel-centos6/g' Dockerfile.x64 | \ -sed 's//NVCC_GENCODE="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_60,code=compute_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_62,code=sm_62"/g'> Dockerfile.tmp - -docker build -t ${REPO}/paddle_manylinux_devel:cuda8.0_cudnn7 -f Dockerfile.tmp . -docker push ${REPO}/paddle_manylinux_devel:cuda8.0_cudnn7 - -sed 's//9.0-cudnn7-devel-centos6/g' Dockerfile.x64 | \ -sed 's//NVCC_GENCODE="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_60,code=compute_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_62,code=sm_62 -gencode=arch=compute_70,code=sm_70"/g'> Dockerfile.tmp -docker build -t ${REPO}/paddle_manylinux_devel:cuda9.0_cudnn7 -f Dockerfile.tmp . -docker push ${REPO}/paddle_manylinux_devel:cuda9.0_cudnn7 - -sed 's//10.0-cudnn7-devel-centos6/g' Dockerfile.x64 | \ -sed 's//NVCC_GENCODE="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_60,code=compute_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_62,code=sm_62 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"/g'> Dockerfile.tmp -docker build -t ${REPO}/paddle_manylinux_devel:cuda10.0_cudnn7 -f Dockerfile.tmp . -docker push ${REPO}/paddle_manylinux_devel:cuda10.0_cudnn7 diff --git a/tools/manylinux1/build_scripts/build.sh b/tools/manylinux1/build_scripts/build.sh deleted file mode 100644 index 6c43ce4fad8..00000000000 --- a/tools/manylinux1/build_scripts/build.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/bash -# Top-level build script called from Dockerfile - -# Stop at any error, show all commands -set -ex - -# Python versions to be installed in /opt/$VERSION_NO -# NOTE Only need python 2.7.11 for nupic.core/nupic.bindings at this time, so -# remove others to expedite build and reduce docker image size. The original -# manylinux docker image project builds many python versions. -# NOTE We added back 3.5.1, since auditwheel requires python 3.3+ -CPYTHON_VERSIONS="3.7.0 3.6.0 3.5.1 2.7.11" - -# openssl version to build, with expected sha256 hash of .tar.gz -# archive -OPENSSL_ROOT=openssl-1.1.0i -OPENSSL_HASH=ebbfc844a8c8cc0ea5dc10b86c9ce97f401837f3fa08c17b2cdadc118253cf99 -EPEL_RPM_HASH=e5ed9ecf22d0c4279e92075a64c757ad2b38049bcf5c16c4f2b75d5f6860dc0d -DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc -PATCHELF_HASH=f2aa40a6148cb3b0ca807a1bf836b081793e55ec9e5540a5356d800132be7e0a -CURL_ROOT=curl-7.49.1 -CURL_HASH=eb63cec4bef692eab9db459033f409533e6d10e20942f4b060b32819e81885f1 -AUTOCONF_ROOT=autoconf-2.69 -AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 - -# Dependencies for compiling Python that we want to remove from -# the final image after compiling Python -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel" - -# Libraries that are allowed as part of the manylinux1 profile -MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel" - -# Get build utilities -MY_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MY_DIR/build_utils.sh - -# EPEL support -yum -y install wget curl -curl -sLO https://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm -check_sha256sum epel-release-6-8.noarch.rpm $EPEL_RPM_HASH - -# Dev toolset (for LLVM and other projects requiring C++11 support) -curl -sLO http://people.centos.org/tru/devtools-2/devtools-2.repo -check_sha256sum devtools-2.repo $DEVTOOLS_HASH -mv devtools-2.repo /etc/yum.repos.d/devtools-2.repo -rpm -Uvh --replacepkgs epel-release-6*.rpm -rm -f epel-release-6*.rpm - -# Development tools and libraries -yum -y install bzip2 make git patch unzip bison yasm diffutils \ - automake which file \ - kernel-devel-`uname -r` \ - devtoolset-2-binutils devtoolset-2-gcc \ - devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran \ - ${PYTHON_COMPILE_DEPS} - -# Install more recent version of cmake -# curl -O https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.sh -# /bin/sh cmake-3.8.1-Linux-x86_64.sh --prefix=/usr/local --skip-license -# rm cmake-3.8.1-Linux-x86_64.sh - -wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz -tar -zxvf cmake-3.16.0.tar.gz && rm cmake-3.16.0.tar.gz -cd cmake-3.16.0 && ./bootstrap -make -j `nproc` && make install -ln -s /usr/local/bin/cmake /usr/bin/cmake -PATH=/usr/local/bin:$PATH - -# Install newest autoconf -build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH -autoconf --version - -# Compile the latest Python releases. -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -build_openssl $OPENSSL_ROOT $OPENSSL_HASH -mkdir -p /opt/python -build_cpythons $CPYTHON_VERSIONS - -PY35_BIN=/opt/python/cp35-cp35m/bin -PY36_BIN=/opt/python/cp36-cp36m/bin -PY37_BIN=/opt/python/cp37-cp37m/bin -# NOTE Since our custom manylinux image builds pythons with shared -# libpython, we need to add libpython's dir to LD_LIBRARY_PATH before running -# python. -ORIGINAL_LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" -LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib:$(dirname ${PY36_BIN})/lib:$(dirname ${PY37_BIN})/lib" - -# Our openssl doesn't know how to find the system CA trust store -# (https://github.com/pypa/manylinux/issues/53) -# And it's not clear how up-to-date that is anyway -# So let's just use the same one pip and everyone uses -LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install certifi -ln -s $($PY35_BIN/python -c 'import certifi; print(certifi.where())') \ - /opt/_internal/certs.pem -# If you modify this line you also have to modify the versions in the -# Dockerfiles: -export SSL_CERT_FILE=/opt/_internal/certs.pem - -# Install newest curl -build_curl $CURL_ROOT $CURL_HASH -rm -rf /usr/local/include/curl /usr/local/lib/libcurl* /usr/local/lib/pkgconfig/libcurl.pc -hash -r -curl --version -curl-config --features - -# Now we can delete our built SSL -rm -rf /usr/local/ssl - -# Install patchelf (latest with unreleased bug fixes) -# FIXME(typhoonzero): restore this when the link is fixed. -# curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz -# check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH -# tar -xzf patchelf-0.9njs2.tar.gz -# (cd patchelf-0.9njs2 && ./configure && make && make install) -# rm -rf patchelf-0.9njs2.tar.gz patchelf-0.9njs2 -yum install -y patchelf - -# Install latest pypi release of auditwheel -LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install auditwheel -ln -s $PY35_BIN/auditwheel /usr/local/bin/auditwheel - -# Clean up development headers and other unnecessary stuff for -# final image -yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \ - avahi freetype bitstream-vera-fonts \ - ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 || true -yum -y install ${MANYLINUX1_DEPS} && yum -y clean all > /dev/null 2>&1 || true -yum list installed -# we don't need libpython*.a, and they're many megabytes -find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f -# Strip what we can -- and ignore errors, because this just attempts to strip -# *everything*, including non-ELF files: -find /opt/_internal -type f -print0 \ - | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -# We do not need the Python test suites, or indeed the precompiled .pyc and -# .pyo files. Partially cribbed from: -# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile -find /opt/_internal \ - \( -type d -a -name test -o -name tests \) \ - -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - -print0 | xargs -0 rm -f - -for PYTHON in /opt/python/*/bin/python; do - # Add matching directory of libpython shared library to library lookup path - LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" - - # Smoke test to make sure that our Pythons work, and do indeed detect as - # being manylinux compatible: - LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/manylinux1-check.py - # Make sure that SSL cert checking works - LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/ssl-check.py -done - -# Restore LD_LIBRARY_PATH -LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" - -# According to ar issues: https://lists.gnu.org/archive/html/bug-binutils/2016-05/msg00211.html -# we should install new version ar with 64-bit supported here -wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz -tar xzf binutils-2.27.tar.gz && cd binutils-2.27 -./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install diff --git a/tools/manylinux1/build_scripts/build_utils.sh b/tools/manylinux1/build_scripts/build_utils.sh deleted file mode 100755 index 690640a141a..00000000000 --- a/tools/manylinux1/build_scripts/build_utils.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/bash -# Helper utilities for build - -PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python -# XXX: the official https server at www.openssl.org cannot be reached -# with the old versions of openssl and curl in Centos 5.11 hence the fallback -# to the ftp mirror: -# OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source -OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source -# Ditto the curl sources -CURL_DOWNLOAD_URL=http://curl.askapache.com/download - -GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py - -AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf - - -function check_var { - if [ -z "$1" ]; then - echo "required variable not defined" - exit 1 - fi -} - - -function lex_pyver { - # Echoes Python version string padded with zeros - # Thus: - # 3.2.1 -> 003002001 - # 3 -> 003000000 - echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' -} - - -function do_cpython_build { - local py_ver=$1 - check_var $py_ver - local ucs_setting=$2 - check_var $ucs_setting - tar -xzf Python-$py_ver.tgz - pushd Python-$py_ver - if [ "$ucs_setting" = "none" ]; then - unicode_flags="" - dir_suffix="" - else - local unicode_flags="--enable-unicode=$ucs_setting" - local dir_suffix="-$ucs_setting" - fi - local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}" - mkdir -p ${prefix}/lib - # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6 - - if [ $(lex_pyver $py_ver) -eq $(lex_pyver 3.6) ]; then - wget https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz - tar -zxf sqlite-autoconf-3250300.tar.gz - cd sqlite-autoconf-3250300 - ./configure --prefix=/usr/local - make -j8 && make install - cd ../ && rm sqlite-autoconf-3250300.tar.gz - fi - - # NOTE --enable-shared for generating libpython shared library needed for - # linking of some of the nupic.core test executables. - if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.7) ]; then - # NOTE python 3.7 should be installed via make altinstall rather than - # make install, and we should specify the location of ssl - CFLAGS="-Wformat" ./configure --prefix=${prefix} --with-openssl=/usr/local/ssl --enable-shared $unicode_flags > /dev/null - make -j8 > /dev/null - make altinstall > /dev/null - else - LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} CFLAGS="-Wformat" ./configure --prefix=${prefix} --enable-shared $unicode_flags > /dev/null - LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make -j8 > /dev/null - LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make install > /dev/null - fi - popd - echo "ZZZ looking for libpython" - find / -name 'libpython*.so*' - rm -rf Python-$py_ver - # Some python's install as bin/python3. Make them available as - # bin/python. - if [ -e ${prefix}/bin/python3 ]; then - ln -s python3 ${prefix}/bin/python - fi - if [ -e ${prefix}/bin/python3.7 ]; then - ln -s python3.7 ${prefix}/bin/python - fi - # NOTE Make libpython shared library visible to python calls below - LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python get-pip.py - LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/pip install wheel - cd / - ls ${MY_DIR} - local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) - ln -s ${prefix} /opt/python/${abi_tag} -} - - -function build_cpython { - local py_ver=$1 - check_var $py_ver - check_var $PYTHON_DOWNLOAD_URL - wget -q $PYTHON_DOWNLOAD_URL/$py_ver/Python-$py_ver.tgz - if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then - # NOTE We only need wide unicode for nupic.bindings wheel - do_cpython_build $py_ver ucs2 - do_cpython_build $py_ver ucs4 - else - do_cpython_build $py_ver none - fi - rm -f Python-$py_ver.tgz -} - - -function build_cpythons { - for py_ver in $@; do - check_var $GET_PIP_URL - curl -sLO $GET_PIP_URL - build_cpython $py_ver - done - rm get-pip.py -} - - -function do_openssl_build { - ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null - make > /dev/null - make install > /dev/null -} - - -function check_sha256sum { - local fname=$1 - check_var ${fname} - local sha256=$2 - check_var ${sha256} - - echo "${sha256} ${fname}" > ${fname}.sha256 - sha256sum -c ${fname}.sha256 - rm ${fname}.sha256 -} - - -function build_openssl { - local openssl_fname=$1 - check_var ${openssl_fname} - local openssl_sha256=$2 - check_var ${openssl_sha256} - check_var ${OPENSSL_DOWNLOAD_URL} - curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz - check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} - tar -xzf ${openssl_fname}.tar.gz - (cd ${openssl_fname} && do_openssl_build) - rm -rf ${openssl_fname} ${openssl_fname}.tar.gz -} - - -function do_curl_build { - LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null - make > /dev/null - make install > /dev/null -} - - -function build_curl { - local curl_fname=$1 - check_var ${curl_fname} - local curl_sha256=$2 - check_var ${curl_sha256} - check_var ${CURL_DOWNLOAD_URL} - curl -sLO ${CURL_DOWNLOAD_URL}/${curl_fname}.tar.bz2 - check_sha256sum ${curl_fname}.tar.bz2 ${curl_sha256} - tar -jxf ${curl_fname}.tar.bz2 - (cd ${curl_fname} && do_curl_build) - rm -rf ${curl_fname} ${curl_fname}.tar.bz2 -} - - -function do_standard_install { - ./configure > /dev/null - make > /dev/null - make install > /dev/null -} - - -function build_autoconf { - local autoconf_fname=$1 - check_var ${autoconf_fname} - local autoconf_sha256=$2 - check_var ${autoconf_sha256} - check_var ${AUTOCONF_DOWNLOAD_URL} - curl -sLO ${AUTOCONF_DOWNLOAD_URL}/${autoconf_fname}.tar.gz - check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256} - tar -zxf ${autoconf_fname}.tar.gz - (cd ${autoconf_fname} && do_standard_install) - rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz -} diff --git a/tools/manylinux1/build_scripts/install_nccl2.sh b/tools/manylinux1/build_scripts/install_nccl2.sh deleted file mode 100644 index 0c9bf1409d9..00000000000 --- a/tools/manylinux1/build_scripts/install_nccl2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| sed "s/release //") -if [ "$VERSION" == "10.0" ]; then - DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" -elif [ "$VERSION" == "10.2" ]; then - DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" -elif [ "$VERSION" == "10.1" ]; then - DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" -elif [ "$VERSION" == "9.0" ]; then - DEB="nccl-repo-ubuntu1604-2.3.7-ga-cuda9.0_1-1_amd64.deb" -else - echo "nccl not found" - exit 2 -fi - -URL="http://nccl2-deb.cdn.bcebos.com/$DEB" - -DIR="/nccl2" -mkdir -p $DIR -# we cached the nccl2 deb package in BOS, so we can download it with wget -# install nccl2: http://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html#down -wget -q -O $DIR/$DEB $URL - -cd $DIR && ar x $DEB && tar xf data.tar.xz -DEBS=$(find ./var/ -name "*.deb") -for sub_deb in $DEBS; do - echo $sub_deb - ar x $sub_deb && tar xf data.tar.xz -done -mv -f usr/include/nccl.h /usr/local/include/ -mv -f usr/lib/x86_64-linux-gnu/libnccl* /usr/local/lib/ -rm /usr/include/nccl.h -rm -rf $DIR diff --git a/tools/manylinux1/build_scripts/manylinux1-check.py b/tools/manylinux1/build_scripts/manylinux1-check.py deleted file mode 100644 index 0d1a6df4eec..00000000000 --- a/tools/manylinux1/build_scripts/manylinux1-check.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Logic copied from PEP 513 - - -def is_manylinux1_compatible(): - # Only Linux, and only x86-64 / i686 - from distutils.util import get_platform - if get_platform() not in ["linux-x86_64", "linux-i686"]: - return False - - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux1_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass - - # Check glibc version. CentOS 5 uses glibc 2.5. - return have_compatible_glibc(2, 5) - - -def have_compatible_glibc(major, minimum_minor): - import ctypes - - process_namespace = ctypes.CDLL(None) - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return False - - # Call gnu_get_libc_version, which returns a string like "2.5". - gnu_get_libc_version.restype = ctypes.c_char_p - version_str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") - - # Parse string and check against requested version. - version = [int(piece) for piece in version_str.split(".")] - assert len(version) == 2 - if major != version[0]: - return False - if minimum_minor > version[1]: - return False - return True - - -import sys -if is_manylinux1_compatible(): - print("%s is manylinux1 compatible" % (sys.executable, )) - sys.exit(0) -else: - print("%s is NOT manylinux1 compatible" % (sys.executable, )) - sys.exit(1) diff --git a/tools/manylinux1/build_scripts/python-tag-abi-tag.py b/tools/manylinux1/build_scripts/python-tag-abi-tag.py deleted file mode 100644 index 0364ab3659e..00000000000 --- a/tools/manylinux1/build_scripts/python-tag-abi-tag.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Utility script to print the python tag + the abi tag for a Python -# See PEP 425 for exactly what these are, but an example would be: -# cp27-cp27mu - -from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag - -print("{0}{1}-{2}".format(get_abbr_impl(), get_impl_ver(), get_abi_tag())) diff --git a/tools/manylinux1/build_scripts/ssl-check.py b/tools/manylinux1/build_scripts/ssl-check.py deleted file mode 100644 index afef2812f3f..00000000000 --- a/tools/manylinux1/build_scripts/ssl-check.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cf. https://github.com/pypa/manylinux/issues/53 - -GOOD_SSL = "https://google.com" -BAD_SSL = "https://self-signed.badssl.com" - -import sys - -print("Testing SSL certificate checking for Python:", sys.version) - -if (sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4)): - print("This version never checks SSL certs; skipping tests") - sys.exit(0) - -if sys.version_info[0] >= 3: - from urllib.request import urlopen - EXC = OSError -else: - from urllib import urlopen - EXC = IOError - -print("Connecting to %s should work" % (GOOD_SSL, )) -urlopen(GOOD_SSL) -print("...it did, yay.") - -print("Connecting to %s should fail" % (BAD_SSL, )) -try: - urlopen(BAD_SSL) - # If we get here then we failed: - print("...it DIDN'T!!!!!11!!1one!") - sys.exit(1) -except EXC: - print("...it did, yay.") -- GitLab