From f5186c3c4bab17df887f674a5fcd490c1ddaf332 Mon Sep 17 00:00:00 2001 From: pangyoki Date: Wed, 7 Apr 2021 15:35:55 +0800 Subject: [PATCH] update name of develop whl package and upgrade gcc 4.8.2 to gcc 5.4 (#31240) * update develop whl package name * distingush cpu and gpu name * fix ref_gcc * change whl name * upgrade gcc 4.8 to 5.4 in ubuntu_dev * update gcc4.8 to 5.4 in centos * Upgrade pip from 18.0 to 20.0.1 * change 2.1.0_dev0 to 2.1.0.dev0 in gpu version --- tools/dockerfile/Dockerfile.ubuntu | 8 ++-- tools/dockerfile/build_scripts/install_gcc.sh | 14 +++++++ tools/dockerfile/centos7_manylinux.sh | 4 ++ tools/dockerfile/ubuntu16_dev.sh | 38 ++++++++++--------- tools/dockerfile/ubuntu18_dev.sh | 38 ++++++++++--------- 5 files changed, 62 insertions(+), 40 deletions(-) diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu index d68992717c5..f566e66a976 100644 --- a/tools/dockerfile/Dockerfile.ubuntu +++ b/tools/dockerfile/Dockerfile.ubuntu @@ -97,8 +97,8 @@ RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e44 WORKDIR /home/setuptools-40.6.2 RUN python setup.py build && python setup.py install WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz -WORKDIR pip-18.0 +RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz +WORKDIR pip-20.0.1 RUN python setup.py install && \ python3.8 setup.py install && \ python3.7 setup.py install && \ @@ -106,8 +106,8 @@ RUN python setup.py install && \ python3 setup.py install WORKDIR /home -RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ - rm -r Python-$version setuptools-40.6.2 pip-18.0 +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-20.0.1.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-20.0.1 # Install Go and glide RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ diff --git a/tools/dockerfile/build_scripts/install_gcc.sh b/tools/dockerfile/build_scripts/install_gcc.sh index e75021b2a9b..e744e9ddac6 100644 --- a/tools/dockerfile/build_scripts/install_gcc.sh +++ b/tools/dockerfile/build_scripts/install_gcc.sh @@ -43,4 +43,18 @@ if [ "$1" == "gcc82" ]; then ln -s /usr/local/gcc-8.2/lib64/libgfortran.so.5 ${lib_so_5} && \ ln -s /usr/local/gcc-8.2/lib64/libstdc++.so.6 ${lib_so_6} && \ cp /usr/local/gcc-8.2/lib64/libstdc++.so.6.0.25 ${lib_path} +elif [ "$1" == "gcc54" ]; then + wget -q http://ftp.tsukuba.wide.ad.jp/software/gcc/releases/gcc-5.4.0/gcc-5.4.0.tar.bz2 + tar -xvf gcc-5.4.0.tar.bz2 && \ + cd gcc-5.4.0 && \ + unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ + ./contrib/download_prerequisites && \ + cd .. && mkdir temp_gcc54 && cd temp_gcc54 && \ + ../gcc-5.4.0/configure --prefix=/usr/local/gcc-5.4 --enable-checking=release --enable-languages=c,c++ --disable-multilib && \ + make -j8 && make install + cd .. && rm -rf temp_gcc54 + cp ${lib_so_6} ${lib_so_6}.bak && rm -f ${lib_so_6} && + ln -s /usr/local/gcc-5.4/lib64/libgfortran.so.5 ${lib_so_5} && \ + ln -s /usr/local/gcc-5.4/lib64/libstdc++.so.6 ${lib_so_6} && \ + cp /usr/local/gcc-5.4/lib64/libstdc++.so.6.0.21 ${lib_path} fi diff --git a/tools/dockerfile/centos7_manylinux.sh b/tools/dockerfile/centos7_manylinux.sh index 490bff22826..5f8a48c8067 100755 --- a/tools/dockerfile/centos7_manylinux.sh +++ b/tools/dockerfile/centos7_manylinux.sh @@ -20,11 +20,15 @@ REPO="${REPO:-paddledocker}" function make_cuda9cudnn7(){ sed 's//9.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + } function make_cuda10cudnn7() { sed 's//10.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + } diff --git a/tools/dockerfile/ubuntu16_dev.sh b/tools/dockerfile/ubuntu16_dev.sh index b7d0d8e3e2a..23578b4143f 100755 --- a/tools/dockerfile/ubuntu16_dev.sh +++ b/tools/dockerfile/ubuntu16_dev.sh @@ -33,7 +33,7 @@ function ref_whl(){ fi if [[ ${WITH_GPU} != "ON" ]]; then - ref_gcc = "" + ref_gcc="" elif [[ ${gcc_version} == "8.2.0" ]];then ref_gcc=_gcc8.2 fi @@ -44,29 +44,31 @@ function ref_whl(){ ref_version=.post100 elif [[ ${ref_CUDA_MAJOR} == "10.1" ]];then ref_version=.post101 - elif [[ ${ref_CUDA_MAJOR} == "10.2" ]];then + elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} == "develop" ]];then + ref_version=.post102 + elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} != "develop" ]];then ref_version="" elif [[ ${ref_CUDA_MAJOR} == "9" ]];then ref_version=.post90 fi + + ref_dev=2.1.0.dev0 ref_web="https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}${ref_gcc}" - if [[ ${PADDLE_VERSION} == "0.0.0" && ${WITH_GPU} == "ON" ]]; then - ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp27-cp27mu-linux_x86_64.whl - ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp35-cp35m-linux_x86_64.whl - ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp36-cp36m-linux_x86_64.whl - ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp37-cp37m-linux_x86_64.whl - ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp38-cp38-linux_x86_64.whl - else - ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp27-cp27mu-linux_x86_64.whl - ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp35-cp35m-linux_x86_64.whl - ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp36-cp36m-linux_x86_64.whl - ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp37-cp37m-linux_x86_64.whl - ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp38-cp38-linux_x86_64.whl - fi - - if [[ ${PADDLE_VERSION} != "0.0.0" && ${WITH_GPU} == "ON" ]]; then + if [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} == "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp38-cp38-linux_x86_64.whl + elif [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} != "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${ref_dev}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${ref_dev}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${ref_dev}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${ref_dev}-cp38-cp38-linux_x86_64.whl + elif [[ ${PADDLE_VERSION} != "develop" && ${WITH_GPU} == "ON" ]]; then ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp27-cp27mu-linux_x86_64.whl ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp35-cp35m-linux_x86_64.whl ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp36-cp36m-linux_x86_64.whl @@ -106,7 +108,7 @@ function install_gcc(){ else sed -i 's##RUN apt-get update \ WORKDIR /usr/bin \ - RUN apt install -y gcc-4.8 g++-4.8 \&\& cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \&\& ln -s gcc-4.8 gcc \&\& ln -s g++-4.8 g++ #g' Dockerfile.tmp + RUN apt install -y gcc g++ #g' Dockerfile.tmp fi } diff --git a/tools/dockerfile/ubuntu18_dev.sh b/tools/dockerfile/ubuntu18_dev.sh index 19572f639bc..6c6a14529ca 100755 --- a/tools/dockerfile/ubuntu18_dev.sh +++ b/tools/dockerfile/ubuntu18_dev.sh @@ -33,7 +33,7 @@ function ref_whl(){ fi if [[ ${WITH_GPU} != "ON" ]]; then - ref_gcc = "" + ref_gcc="" elif [[ ${gcc_version} == "8.2.0" ]];then ref_gcc=_gcc8.2 fi @@ -44,29 +44,31 @@ function ref_whl(){ ref_version=.post100 elif [[ ${ref_CUDA_MAJOR} == "10.1" ]];then ref_version=.post101 - elif [[ ${ref_CUDA_MAJOR} == "10.2" ]];then + elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} == "develop" ]];then + ref_version=.post102 + elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} != "develop" ]];then ref_version="" elif [[ ${ref_CUDA_MAJOR} == "9" ]];then ref_version=.post90 fi + + ref_dev=2.1.0.dev0 ref_web="https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}${ref_gcc}" - if [[ ${PADDLE_VERSION} == "0.0.0" && ${WITH_GPU} == "ON" ]]; then - ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp27-cp27mu-linux_x86_64.whl - ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp35-cp35m-linux_x86_64.whl - ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp36-cp36m-linux_x86_64.whl - ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp37-cp37m-linux_x86_64.whl - ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp38-cp38-linux_x86_64.whl - else - ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp27-cp27mu-linux_x86_64.whl - ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp35-cp35m-linux_x86_64.whl - ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp36-cp36m-linux_x86_64.whl - ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp37-cp37m-linux_x86_64.whl - ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp38-cp38-linux_x86_64.whl - fi - - if [[ ${PADDLE_VERSION} != "0.0.0" && ${WITH_GPU} == "ON" ]]; then + if [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} == "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp38-cp38-linux_x86_64.whl + elif [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} != "ON" ]]; then + ref_paddle_whl=paddlepaddle${install_gpu}-${ref_dev}-cp27-cp27mu-linux_x86_64.whl + ref_paddle3_whl=paddlepaddle${install_gpu}-${ref_dev}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${ref_dev}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${ref_dev}-cp38-cp38-linux_x86_64.whl + elif [[ ${PADDLE_VERSION} != "develop" && ${WITH_GPU} == "ON" ]]; then ref_paddle_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp27-cp27mu-linux_x86_64.whl ref_paddle3_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp35-cp35m-linux_x86_64.whl ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp36-cp36m-linux_x86_64.whl @@ -107,7 +109,7 @@ function install_gcc(){ else sed -i 's##RUN apt-get update \ WORKDIR /usr/bin \ - RUN apt install -y gcc-4.8 g++-4.8 \&\& cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \&\& ln -s gcc-4.8 gcc \&\& ln -s g++-4.8 g++ #g' Dockerfile.tmp + RUN apt install -y gcc g++ #g' Dockerfile.tmp fi } -- GitLab