Update openssl (#29424)

tools/dockerfile/Dockerfile.centos

@@ -10,7 +10,7 @@ ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
+ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
 ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
 RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
@@ -26,7 +26,7 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
   tar -xvf git-2.17.1.tar.gz && \
   cd git-2.17.1 && \
-  ./configure --prefix=/usr/local && \
+  ./configure --with-openssl=/usr/local/ssl --prefix=/usr/local && \
   make -j8 && make install 
 
 # for paddle
@@ -38,7 +38,7 @@ RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8
 
 
 ENV GOROOT=/usr/local/go GOPATH=/root/gopath
-ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH}
+ENV PATH=/usr/local/ssl:${GOROOT}/bin:${GOPATH}/bin:${PATH}
 
 # protobuf 3.6.1
 RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/protobuf-cpp-3.6.1.tar.gz && \ 

tools/dockerfile/Dockerfile.ubuntu18

@@ -22,7 +22,7 @@ RUN apt-get update && \
   apt-get update && \
   apt-get install -y curl wget vim git unzip unrar tar xz-utils bzip2 gzip \ 
     coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev \
-    bison graphviz libjpeg-dev zlib1g-dev automake locales clang-format swig net-tools libtool module-init-tools
+    bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool module-init-tools
 
 # Downgrade gcc&&g++
 WORKDIR /usr/bin 
@@ -127,4 +127,9 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
     make -j8 && make install && \
     ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
 
+# clang-form 3.8.0
+RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ 
+    tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \
+    cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz 
+
 EXPOSE 22

tools/dockerfile/build_scripts/build.sh

@@ -107,9 +107,6 @@ hash -r
 curl --version
 curl-config --features
 
-# Now we can delete our built SSL
-rm -rf /usr/local/ssl
-
 # Install patchelf (latest with unreleased bug fixes)
 # FIXME(typhoonzero): restore this when the link is fixed.
 # curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz

tools/dockerfile/build_scripts/build_utils.sh

 #!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Helper utilities for build
 
 PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
@@ -51,7 +66,7 @@ function do_cpython_build {
     # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6
 
     if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.6) ]; then
-        wget https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz
+        wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz
         tar -zxf sqlite-autoconf-3250300.tar.gz
         cd sqlite-autoconf-3250300
         ./configure --prefix=/usr/local
@@ -88,8 +103,8 @@ function do_cpython_build {
         ln -s python3.8 ${prefix}/bin/python
     fi
     # NOTE Make libpython shared library visible to python calls below
-    LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python get-pip.py
-    LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2
     cd /
     ls ${MY_DIR}
     local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
@@ -124,9 +139,10 @@ function build_cpythons {
 
 
 function do_openssl_build {
-    ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null
+    ./config -fPIC --prefix=/usr/local/ssl > /dev/null
     make > /dev/null
     make install > /dev/null
+    
 }
 
 
@@ -160,6 +176,9 @@ function do_curl_build {
     LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null
     make > /dev/null
     make install > /dev/null
+    ln -s /usr/local/ssl/lib/libcrypto.so /usr/lib/libcrypto.so
+    ln -s /usr/local/ssl/lib/libssl.so /usr/lib/libssl.so
+    ln -s /usr/local/ssl/bin/openssl /usr/local/bin/openssl
 }
 
 

tools/dockerfile/build_scripts/install_nccl2.sh

@@ -18,14 +18,14 @@ VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"|
 if [ "$VERSION" == "10.0" ]; then
   DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb"
 elif [ "$VERSION" == "10.2" ] || [ "$VERSION" == "10.1" ] || [ "$VERSION" == "11.0" ]; then
-  if [ -f "ls /etc/redhat-release " ];then
+  if [ -f "/etc/redhat-release" ];then
     rm -f /usr/local/lib/libnccl.so 
     wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.7.8-1+cuda10.2.x86_64.rpm
     wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm
     wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm
     rpm -ivh libnccl-2.7.8-1+cuda10.2.x86_64.rpm
     rpm -ivh libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm
-    rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f /usr/include/nccl.h 
+    rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f libnccl-*
     exit 0
   fi
   DEB="nccl-repo-ubuntu1604-2.7.8-ga-cuda10.2_1-1_amd64.deb"

tools/dockerfile/build_scripts/install_trt.sh

@@ -27,10 +27,10 @@ elif [[ "$VERSION" == "11.0" ]];then
   cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/
   rm TensorRT-7.1.3.4.Ubuntu-16.04.x86_64-gnu.cuda-11.0.cudnn8.0.tar.gz
 elif [[ "$VERSION" == "10.2" ]];then
-  wget -q https://paddle-ci.cdn.bcebos.com/TRT/TensorRT7-cuda10.2-cudnn7.tar.gz --no-check-certificate 
-  tar -zxf TensorRT7-cuda10.2-cudnn7.tar.gz -C /usr/local
-  cp -rf /usr/local/TensorRT-7.0.0.11/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.0.0.11/lib/* /usr/lib/
-  rm TensorRT7-cuda10.2-cudnn7.tar.gz
+  wget https://paddle-ci.gz.bcebos.com/TRT/TensorRT7-cuda10.2-cudnn8.tar.gz --no-check-certificate
+  tar -zxf TensorRT7-cuda10.2-cudnn8.tar.gz -C /usr/local
+  cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/
+  rm TensorRT7-cuda10.2-cudnn8.tar.gz
 elif [[ "$VERSION" == "10.0" ]];then
   wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.0-cudnn7.tar.gz --no-check-certificate
   tar -zxf TensorRT6-cuda10.0-cudnn7.tar.gz -C /usr/local

tools/dockerfile/centos6_manylinux.sh

@@ -19,22 +19,27 @@ set -xe
 REPO="${REPO:-paddledocker}"
 
 function make_cuda9cudnn7(){
-  sed 's/<baseimg>/9.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp
+  sed 's/<baseimg>/9.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
 }
 
 
 function make_cuda10cudnn7() {
-  sed 's/<baseimg>/10.0-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp
+  sed 's/<baseimg>/10.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
 }
 
 
 function make_cuda101cudnn7() {
-  sed 's/<baseimg>/10.1-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp
+  sed 's/<baseimg>/10.1-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
   sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp 
 }
 
 function make_cuda102cudnn7() {
-  sed 's/<baseimg>/10.2-cudnn7-devel-centos6/g' Dockerfile.centos >Dockerfile.tmp
+  sed 's/<baseimg>/10.2-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+}
+
+function make_cuda102cudnn8() {
+  sed 's/<baseimg>/10.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
   sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
 }
 
@@ -58,6 +63,9 @@ function main() {
     cuda102cudnn7)
       make_cuda102cudnn7
       ;;
+    cuda102cudnn8)
+      make_cuda102cudnn8
+      ;;
     cuda11cudnn8)
       make_cuda11cudnn8
      ;;

tools/dockerfile/ci_dockerfile.sh

@@ -39,7 +39,7 @@ function make_ubuntu_dockerfile(){
 function make_centos_dockerfile(){
   dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6"
   sed "s/<baseimg>/11.0-cudnn8-devel-centos7/g" Dockerfile.centos >${dockerfile_name}
-  sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g" ${dockerfile_name} 
+  sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts  ./build_scripts#g" ${dockerfile_name} 
   dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
   sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so \\
     RUN ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ \\