diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 84e3aaca86cf6b5670fbc36692d51962ef86e0a7..ea1c55af46ea0d47c1a606564b9a44a5c0f1a7eb 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -198,7 +198,8 @@ function cmake_base() { gloo_flag=${distibuted_flag} if [ "$CMD" != "assert_file_approvals" ];then - python -m pip install distro + which python + python -V python ${PADDLE_ROOT}/tools/summary_env.py bash ${PADDLE_ROOT}/tools/get_cpu_info.sh fi diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh index de533e787de5b97031b36edf011d1bb28fbc5a9a..e5a6c240fe5f95c7380c9da33ca0e2c16fd4ac7d 100644 --- a/tools/dockerfile/ci_dockerfile.sh +++ b/tools/dockerfile/ci_dockerfile.sh @@ -26,7 +26,7 @@ function make_ubuntu_dockerfile(){ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub " ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ COPY tools/dockerfile/build_scripts /build_scripts \\ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\ @@ -38,7 +38,7 @@ function make_ubuntu_dockerfile(){ ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} sed -i "s#bash /build_scripts/install_nccl2.sh#wget -q --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\ RUN dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\ - RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 pigz --allow-change-held-packages #g" ${dockerfile_name} + RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 zstd pigz --allow-change-held-packages #g" ${dockerfile_name} } function make_ubuntu_trt7_dockerfile(){ @@ -62,7 +62,7 @@ function make_ubuntu_trt7_dockerfile(){ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub " ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro \&\& pip3.8 install PyGithub distro" ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ COPY tools/dockerfile/build_scripts /build_scripts \\ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\ @@ -73,7 +73,7 @@ function make_ubuntu_trt7_dockerfile(){ RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\ ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} sed -i "s#bash /build_scripts/install_nccl2.sh#wget -q --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\ - RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 pigz --allow-change-held-packages #g" ${dockerfile_name} + RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 zstd pigz --allow-change-held-packages #g" ${dockerfile_name} } @@ -82,7 +82,10 @@ function make_centos_dockerfile(){ sed "s//11.0-cudnn8-devel-centos7/g" Dockerfile.centos >${dockerfile_name} sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g" ${dockerfile_name} dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') - sed -i "${dockerfile_line}i RUN yum install -y pigz graphviz" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN yum install -y pigz graphviz zstd" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip3.7 install distro" ${dockerfile_name} + sed -i "${dockerfile_line}i ENV LD_LIBRARY_PATH /opt/_internal/cpython-3.7.0/lib:/usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 " ${dockerfile_name} + sed -i "${dockerfile_line}i ENV PATH /opt/_internal/cpython-3.7.0/bin:/usr/local/ssl:/usr/local/gcc-8.2/bin:/usr/local/go/bin:/root/gopath/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/java/jdk1.8.0_192/bin " ${dockerfile_name} sed -i "${dockerfile_line}i RUN rm -f /usr/bin/cc && ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc" ${dockerfile_name} sed -i "${dockerfile_line}i RUN rm -f /usr/bin/g++ && ln -s /usr/local/gcc-5.4/bin/g++ /usr/bin/g++" ${dockerfile_name} sed -i "${dockerfile_line}i RUN rm -f /usr/bin/c++ && ln -s /usr/local/gcc-5.4/bin/c++ /usr/bin/c++" ${dockerfile_name} @@ -105,12 +108,12 @@ function make_cinn_dockerfile(){ dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ + sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ tar -xvf git-2.17.1.tar.gz \&\& \ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub " ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel distro \&\& pip3.7 install PyGithub \&\& pip3.8 install distro" ${dockerfile_name} } @@ -121,12 +124,12 @@ function make_ce_framework_dockcerfile(){ sed -i "7i RUN chmod 777 /tmp" ${dockerfile_name} sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y pigz libcurl4-openssl-dev gettext \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ + sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y zstd pigz libcurl4-openssl-dev gettext \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ tar -xvf git-2.17.1.tar.gz \&\& \ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub " ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ COPY tools/dockerfile/build_scripts /build_scripts \\ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\