From 9370aa6f5666eb3a97a9feeec46fe7a06748a6e8 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Wed, 23 Dec 2020 18:19:28 +0800 Subject: [PATCH] Update openssl (#29424) --- tools/dockerfile/Dockerfile.centos | 6 ++--- tools/dockerfile/Dockerfile.ubuntu18 | 7 ++++- tools/dockerfile/build_scripts/build.sh | 3 --- tools/dockerfile/build_scripts/build_utils.sh | 27 ++++++++++++++++--- .../dockerfile/build_scripts/install_nccl2.sh | 4 +-- tools/dockerfile/build_scripts/install_trt.sh | 8 +++--- tools/dockerfile/centos6_manylinux.sh | 16 ++++++++--- tools/dockerfile/ci_dockerfile.sh | 2 +- 8 files changed, 51 insertions(+), 22 deletions(-) diff --git a/tools/dockerfile/Dockerfile.centos b/tools/dockerfile/Dockerfile.centos index 7dc86e4b0b7..337874dac1b 100644 --- a/tools/dockerfile/Dockerfile.centos +++ b/tools/dockerfile/Dockerfile.centos @@ -10,7 +10,7 @@ ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} +ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel @@ -26,7 +26,7 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \ tar -xvf git-2.17.1.tar.gz && \ cd git-2.17.1 && \ - ./configure --prefix=/usr/local && \ + ./configure --with-openssl=/usr/local/ssl --prefix=/usr/local && \ make -j8 && make install # for paddle @@ -38,7 +38,7 @@ RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8 ENV GOROOT=/usr/local/go GOPATH=/root/gopath -ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH} +ENV PATH=/usr/local/ssl:${GOROOT}/bin:${GOPATH}/bin:${PATH} # protobuf 3.6.1 RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/protobuf-cpp-3.6.1.tar.gz && \ diff --git a/tools/dockerfile/Dockerfile.ubuntu18 b/tools/dockerfile/Dockerfile.ubuntu18 index 0ea09c3170e..62dd5734a8d 100644 --- a/tools/dockerfile/Dockerfile.ubuntu18 +++ b/tools/dockerfile/Dockerfile.ubuntu18 @@ -22,7 +22,7 @@ RUN apt-get update && \ apt-get update && \ apt-get install -y curl wget vim git unzip unrar tar xz-utils bzip2 gzip \ coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev \ - bison graphviz libjpeg-dev zlib1g-dev automake locales clang-format swig net-tools libtool module-init-tools + bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool module-init-tools # Downgrade gcc&&g++ WORKDIR /usr/bin @@ -127,4 +127,9 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ make -j8 && make install && \ ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache +# clang-form 3.8.0 +RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ + tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \ + cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz + EXPOSE 22 diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh index a5d886e0a86..aca95a58f6b 100644 --- a/tools/dockerfile/build_scripts/build.sh +++ b/tools/dockerfile/build_scripts/build.sh @@ -107,9 +107,6 @@ hash -r curl --version curl-config --features -# Now we can delete our built SSL -rm -rf /usr/local/ssl - # Install patchelf (latest with unreleased bug fixes) # FIXME(typhoonzero): restore this when the link is fixed. # curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz diff --git a/tools/dockerfile/build_scripts/build_utils.sh b/tools/dockerfile/build_scripts/build_utils.sh index 9f937cf9343..c8e0b6c3f27 100755 --- a/tools/dockerfile/build_scripts/build_utils.sh +++ b/tools/dockerfile/build_scripts/build_utils.sh @@ -1,4 +1,19 @@ #!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Helper utilities for build PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python @@ -51,7 +66,7 @@ function do_cpython_build { # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6 if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.6) ]; then - wget https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz + wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz tar -zxf sqlite-autoconf-3250300.tar.gz cd sqlite-autoconf-3250300 ./configure --prefix=/usr/local @@ -88,8 +103,8 @@ function do_cpython_build { ln -s python3.8 ${prefix}/bin/python fi # NOTE Make libpython shared library visible to python calls below - LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python get-pip.py - LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2 + LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py + LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2 cd / ls ${MY_DIR} local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) @@ -124,9 +139,10 @@ function build_cpythons { function do_openssl_build { - ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null + ./config -fPIC --prefix=/usr/local/ssl > /dev/null make > /dev/null make install > /dev/null + } @@ -160,6 +176,9 @@ function do_curl_build { LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null make > /dev/null make install > /dev/null + ln -s /usr/local/ssl/lib/libcrypto.so /usr/lib/libcrypto.so + ln -s /usr/local/ssl/lib/libssl.so /usr/lib/libssl.so + ln -s /usr/local/ssl/bin/openssl /usr/local/bin/openssl } diff --git a/tools/dockerfile/build_scripts/install_nccl2.sh b/tools/dockerfile/build_scripts/install_nccl2.sh index 9f2c30c477b..b06b3d44c6e 100644 --- a/tools/dockerfile/build_scripts/install_nccl2.sh +++ b/tools/dockerfile/build_scripts/install_nccl2.sh @@ -18,14 +18,14 @@ VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| if [ "$VERSION" == "10.0" ]; then DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" elif [ "$VERSION" == "10.2" ] || [ "$VERSION" == "10.1" ] || [ "$VERSION" == "11.0" ]; then - if [ -f "ls /etc/redhat-release " ];then + if [ -f "/etc/redhat-release" ];then rm -f /usr/local/lib/libnccl.so wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.7.8-1+cuda10.2.x86_64.rpm wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm rpm -ivh libnccl-2.7.8-1+cuda10.2.x86_64.rpm rpm -ivh libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm - rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f /usr/include/nccl.h + rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f libnccl-* exit 0 fi DEB="nccl-repo-ubuntu1604-2.7.8-ga-cuda10.2_1-1_amd64.deb" diff --git a/tools/dockerfile/build_scripts/install_trt.sh b/tools/dockerfile/build_scripts/install_trt.sh index 47d93c2dfca..e5ec70d2f37 100644 --- a/tools/dockerfile/build_scripts/install_trt.sh +++ b/tools/dockerfile/build_scripts/install_trt.sh @@ -27,10 +27,10 @@ elif [[ "$VERSION" == "11.0" ]];then cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/ rm TensorRT-7.1.3.4.Ubuntu-16.04.x86_64-gnu.cuda-11.0.cudnn8.0.tar.gz elif [[ "$VERSION" == "10.2" ]];then - wget -q https://paddle-ci.cdn.bcebos.com/TRT/TensorRT7-cuda10.2-cudnn7.tar.gz --no-check-certificate - tar -zxf TensorRT7-cuda10.2-cudnn7.tar.gz -C /usr/local - cp -rf /usr/local/TensorRT-7.0.0.11/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.0.0.11/lib/* /usr/lib/ - rm TensorRT7-cuda10.2-cudnn7.tar.gz + wget https://paddle-ci.gz.bcebos.com/TRT/TensorRT7-cuda10.2-cudnn8.tar.gz --no-check-certificate + tar -zxf TensorRT7-cuda10.2-cudnn8.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/ + rm TensorRT7-cuda10.2-cudnn8.tar.gz elif [[ "$VERSION" == "10.0" ]];then wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.0-cudnn7.tar.gz --no-check-certificate tar -zxf TensorRT6-cuda10.0-cudnn7.tar.gz -C /usr/local diff --git a/tools/dockerfile/centos6_manylinux.sh b/tools/dockerfile/centos6_manylinux.sh index a31376e22d2..490bff22826 100755 --- a/tools/dockerfile/centos6_manylinux.sh +++ b/tools/dockerfile/centos6_manylinux.sh @@ -19,22 +19,27 @@ set -xe REPO="${REPO:-paddledocker}" function make_cuda9cudnn7(){ - sed 's//9.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp + sed 's//9.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp } function make_cuda10cudnn7() { - sed 's//10.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp + sed 's//10.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp } function make_cuda101cudnn7() { - sed 's//10.1-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp + sed 's//10.1-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp } function make_cuda102cudnn7() { - sed 's//10.2-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp + sed 's//10.2-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp +} + +function make_cuda102cudnn8() { + sed 's//10.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp } @@ -58,6 +63,9 @@ function main() { cuda102cudnn7) make_cuda102cudnn7 ;; + cuda102cudnn8) + make_cuda102cudnn8 + ;; cuda11cudnn8) make_cuda11cudnn8 ;; diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh index 08d501c63c4..2fa3d5141e5 100644 --- a/tools/dockerfile/ci_dockerfile.sh +++ b/tools/dockerfile/ci_dockerfile.sh @@ -39,7 +39,7 @@ function make_ubuntu_dockerfile(){ function make_centos_dockerfile(){ dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6" sed "s//11.0-cudnn8-devel-centos7/g" Dockerfile.centos >${dockerfile_name} - sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g" ${dockerfile_name} + sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g" ${dockerfile_name} dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so \\ RUN ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ \\ -- GitLab