Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2299
Star 20931
Fork 5422
P
Paddle
未验证 提交 96ae48b7 编写于 作者: T tianshuo78520a 提交者： GitHub
浏览文件
操作

Change CI dockerfile (#28236)

上级 fdc06f21
隐藏空白更改
内联 并排
Showing with 58 addition and 40 deletion
tools/dockerfile/Dockerfile.ubuntu
浏览文件 @ 96ae48b7
...@@ -207,6 +207,8 @@ RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/ ...@@ -207,6 +207,8 @@ RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/
cd binutils-2.27 && \ cd binutils-2.27 && \
./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz
RUN apt-get install libprotobuf-dev -y
# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
# So install a newer version here. # So install a newer version here.
...@@ -224,4 +226,5 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ ...@@ -224,4 +226,5 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
make -j8 && make install && \ make -j8 && make install && \
ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
EXPOSE 22 EXPOSE 22
tools/dockerfile/ci_dockerfile.sh
浏览文件 @ 96ae48b7
#!/bin/bash #!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function make_ubuntu_dockerfile(){ function make_ubuntu_dockerfile(){
dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16" dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16"
sed 's/<baseimg>/10.1-cudnn7-devel-ubuntu16.04/g' ./Dockerfile.ubuntu >${dockerfile_name} sed "s/<baseimg>/10.1-cudnn7-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name}
sed -i 's#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g' ${dockerfile_name} sed -i "s#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g" ${dockerfile_name}
dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'` dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i 's#<install_gcc>#WORKDIR /usr/bin \ sed -i "s#<install_gcc>#WORKDIR /usr/bin \\
COPY tools/dockerfile/build_scripts /build_scripts \ COPY tools/dockerfile/build_scripts /build_scripts \\
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\
RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \ RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \ RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \ RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \ RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \ RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\
ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' ${dockerfile_name} ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
sed -i 's#bash /build_scripts/install_nccl2.sh#wget --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb && \ sed -i "s#bash /build_scripts/install_nccl2.sh#wget --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb && \ RUN dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 --allow-change-held-packages #g' ${dockerfile_name} RUN apt-get install -y libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 --allow-change-held-packages #g" ${dockerfile_name}
} }
function make_centos_dockerfile(){ function make_centos_dockerfile(){
dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6" dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6"
sed 's/<baseimg>/10.2-cudnn7-devel-centos6/g' Dockerfile.centos >${dockerfile_name} sed "s/<baseimg>/10.2-cudnn7-devel-centos6/g" Dockerfile.centos >${dockerfile_name}
sed -i 's#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g' ${dockerfile_name} sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g" ${dockerfile_name}
dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'` dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so && \ sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so \\
ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ && \ RUN ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ \\
rm -rf /usr/include/NvInfer*" ${dockerfile_name} RUN rm -rf /usr/include/NvInfer*" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \ sed -i $"${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \\
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} RUN tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i 's#RUN bash build_scripts/install_nccl2.sh##g' ${dockerfile_name} sed -i "s#RUN bash build_scripts/install_nccl2.sh##g" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.7.8-1+cuda10.2.x86_64.rpm && \ sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.7.8-1+cuda10.2.x86_64.rpm \\
wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm && \ RUN wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm \\
wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && \ RUN wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm \\
rpm -ivh libnccl-2.7.8-1+cuda10.2.x86_64.rpm && \ RUN rpm -ivh libnccl-2.7.8-1+cuda10.2.x86_64.rpm \\
rpm -ivh libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm && \ RUN rpm -ivh libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm \\
rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f /usr/local/include/nccl.h " ${dockerfile_name} RUN rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f /usr/local/include/nccl.h " ${dockerfile_name}
sed -i 's#<install_gcc>#WORKDIR /usr/bin \ sed -i "s#<install_gcc>#WORKDIR /usr/bin \\
COPY tools/dockerfile/build_scripts /build_scripts \ COPY tools/dockerfile/build_scripts /build_scripts \\
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\
RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \ RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \ RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \ RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \ RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \ RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\
ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' ${dockerfile_name} ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
} }
...@@ -55,4 +70,4 @@ function main() { ...@@ -55,4 +70,4 @@ function main() {
make_centos_dockerfile make_centos_dockerfile
} }
main $@ main "$@"
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或