diff --git a/python/examples/README_CN.md b/python/examples/README_CN.md index 1da323709e9cd16437881cd47cf4c8f292fbcda3..60b7cd77aadbca0002be1618a53af8caba215afa 100644 --- a/python/examples/README_CN.md +++ b/python/examples/README_CN.md @@ -6,4 +6,4 @@ - imagenet ResNet50/ResNet101 - detection faster_rcnn/yolov3/pp-yolo/ttf-net -i + diff --git a/python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README_CN.md b/python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README_CN.md index 96f3a305da6dea25c8585c84e4c727c4ae6e34ee..1fcced94133e5c66f91f7e7972549cb7f86bfef3 100644 --- a/python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README_CN.md +++ b/python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README_CN.md @@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/ ### 启动服务 ``` tar xf faster_rcnn_r50_fpn_1x_coco.tar -python -m paddle_serving_server_gpu.serve --model pddet_serving_model --port 9494 --gpu_ids 0 +python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0 ``` 该模型支持TensorRT,如果想要更快的预测速度,可以开启`--use_trt`选项。 diff --git a/python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README_CN.md b/python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README_CN.md index 283c21768240e13df087bc749fd65eed5aa4b2c1..f403246b6024eb3514991aba88881344cccb8dd8 100644 --- a/python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README_CN.md +++ b/python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README_CN.md @@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/ ### 启动服务 ``` tar xf ppyolo_r50vd_dcn_1x_coco.tar -python -m paddle_serving_server_gpu.serve --model pddet_serving_model --port 9494 --gpu_ids 0 +python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0 ``` 该模型支持TensorRT,如果想要更快的预测速度,可以开启`--use_trt`选项。 diff --git a/python/examples/detection/ttfnet_darknet53_1x_coco/README_CN.md b/python/examples/detection/ttfnet_darknet53_1x_coco/README_CN.md index 838dcecc122f3f0d0547f0623d2d1ddf6b122fc8..976c94a4d783802a72e5d2b72bccb6e2bcfe1cba 100644 --- a/python/examples/detection/ttfnet_darknet53_1x_coco/README_CN.md +++ b/python/examples/detection/ttfnet_darknet53_1x_coco/README_CN.md @@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/ ### 启动服务 ``` tar xf ttfnet_darknet53_1x_coco.tar -python -m paddle_serving_server_gpu.serve --model pddet_serving_model --port 9494 --gpu_ids 0 +python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0 ``` 该模型支持TensorRT,如果想要更快的预测速度,可以开启`--use_trt`选项。 diff --git a/python/examples/detection/yolov3_darknet53_270e_coco/README_CN.md b/python/examples/detection/yolov3_darknet53_270e_coco/README_CN.md index 6d02612501db67c0207e409db6f4cceef2bda6c4..0dd69e25cfe04f743adb7c6f3b5c50215fa4598a 100644 --- a/python/examples/detection/yolov3_darknet53_270e_coco/README_CN.md +++ b/python/examples/detection/yolov3_darknet53_270e_coco/README_CN.md @@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/ ### 启动服务 ``` tar xf yolov3_darknet53_270e_coco.tar -python -m paddle_serving_server_gpu.serve --model pddet_serving_model --port 9494 --gpu_ids 0 +python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0 ``` 该模型支持TensorRT,如果想要更快的预测速度,可以开启`--use_trt`选项。 diff --git a/tools/Dockerfile b/tools/Dockerfile index bf4254495e5a1163455887008540945d5898182e..ab83229af56c65cb9505b524eb08fe5ce2931a5b 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -1,16 +1,187 @@ -FROM centos:7.3.1611 - -RUN yum -y install wget && \ - yum -y install epel-release && yum -y install patchelf && \ - yum -y install gcc gcc-c++ make python-devel && \ - yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install python3 python3-devel && \ - yum clean all - -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python get-pip.py && rm get-pip.py && \ - localedef -c -i en_US -f UTF-8 en_US.UTF-8 && \ - echo "export LANG=en_US.utf8" >> /root/.bashrc && \ - echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:9.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.0-cudnn7 b/tools/Dockerfile.cuda10.0-cudnn7 index c26eaeb986cbd7d66ee51bac1444cf293800d839..74a2846ea80c795808e1de4a0894e77f93bf7312 100644 --- a/tools/Dockerfile.cuda10.0-cudnn7 +++ b/tools/Dockerfile.cuda10.0-cudnn7 @@ -1,24 +1,181 @@ -FROM nvidia/cuda:10.0-cudnn7-devel-centos7 as builder - -FROM nvidia/cuda:10.0-cudnn7-runtime-centos7 -RUN yum -y install wget && \ - yum -y install epel-release && yum -y install patchelf && \ - yum -y install gcc gcc-c++ make python-devel && \ - yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install python3 python3-devel && \ - yum clean all - -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python get-pip.py && rm get-pip.py - -RUN ln -s /usr/local/cuda-10.0/lib64/libcublas.so.10.0 /usr/local/cuda-10.0/lib64/libcublas.so && \ - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> /root/.bashrc && \ - ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudnn.so.7 /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudnn.so && \ - echo 'export LD_LIBRARY_PATH=/usr/local/cuda-10.0/targets/x86_64-linux/lib:$LD_LIBRARY_PATH' >> /root/.bashrc && \ - echo "export LANG=en_US.utf8" >> /root/.bashrc && \ - echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc && \ - mkdir -p /usr/local/cuda/extras - -COPY --from=builder /usr/local/cuda/extras/CUPTI /usr/local/cuda/extras/CUPTI +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:10.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +RUN apt-get update + WORKDIR /usr/bin + RUN apt install -y gcc-4.8 g++-4.8 && cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ && ln -s gcc-4.8 gcc && ln -s g++-4.8 g++ + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.0-cudnn7.devel b/tools/Dockerfile.cuda10.0-cudnn7.devel index 3215ee7dee0e64a0d5583a7d2024b413d526b000..74a2846ea80c795808e1de4a0894e77f93bf7312 100644 --- a/tools/Dockerfile.cuda10.0-cudnn7.devel +++ b/tools/Dockerfile.cuda10.0-cudnn7.devel @@ -1,45 +1,181 @@ -FROM nvidia/cuda:10.0-cudnn7-devel-centos7 - -RUN yum -y install wget >/dev/null \ - && yum -y install gcc gcc-c++ make glibc-static which \ - && yum -y install git openssl-devel curl-devel bzip2-devel python-devel \ - && yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false \ - && yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false \ - && yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false - -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.2/protobuf-all-3.11.2.tar.gz && \ - tar zxf protobuf-all-3.11.2.tar.gz && \ - cd protobuf-3.11.2 && \ - ./configure && make -j4 && make install && \ - make clean && \ - cd .. && rm -rf protobuf-* - -RUN wget https://cmake.org/files/v3.2/cmake-3.2.0-Linux-x86_64.tar.gz >/dev/null \ - && tar xzf cmake-3.2.0-Linux-x86_64.tar.gz \ - && mv cmake-3.2.0-Linux-x86_64 /usr/local/cmake3.2.0 \ - && echo 'export PATH=/usr/local/cmake3.2.0/bin:$PATH' >> /root/.bashrc \ - && rm cmake-3.2.0-Linux-x86_64.tar.gz - -RUN wget https://dl.google.com/go/go1.14.linux-amd64.tar.gz >/dev/null \ - && tar xzf go1.14.linux-amd64.tar.gz \ - && mv go /usr/local/go \ - && echo 'export GOROOT=/usr/local/go' >> /root/.bashrc \ - && echo 'export PATH=/usr/local/go/bin:$PATH' >> /root/.bashrc \ - && rm go1.14.linux-amd64.tar.gz - -RUN yum -y install python-devel sqlite-devel \ - && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py >/dev/null \ - && python get-pip.py >/dev/null \ - && rm get-pip.py \ - && pip install requests - -RUN yum install -y python3 python3-devel \ - && yum -y install epel-release && yum -y install patchelf libXext libSM libXrender\ - && yum clean all \ - && pip3 install requests - -RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8 \ - && echo "export LANG=en_US.utf8" >> /root/.bashrc \ - && echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:10.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors +# ENV variables +ARG WITH_GPU +ARG WITH_AVX +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +RUN apt-get update + WORKDIR /usr/bin + RUN apt install -y gcc-4.8 g++-4.8 && cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ && ln -s gcc-4.8 gcc && ln -s g++-4.8 g++ + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.1-cudnn7 b/tools/Dockerfile.cuda10.1-cudnn7 new file mode 100644 index 0000000000000000000000000000000000000000..a87c40f93ba67e9e2791e96b10e7449e70fa62b3 --- /dev/null +++ b/tools/Dockerfile.cuda10.1-cudnn7 @@ -0,0 +1,187 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:10.1-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.1-cudnn7.devel b/tools/Dockerfile.cuda10.1-cudnn7.devel index daab4340e35702b1728a62f3b7cf5d131a20097d..a87c40f93ba67e9e2791e96b10e7449e70fa62b3 100644 --- a/tools/Dockerfile.cuda10.1-cudnn7.devel +++ b/tools/Dockerfile.cuda10.1-cudnn7.devel @@ -1,12 +1,9 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment # When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04 +FROM hub.baidubce.com/ctr/cuda:10.1-cudnn7-devel-ubuntu16.04 MAINTAINER PaddlePaddle Authors -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - # ENV variables ARG WITH_GPU ARG WITH_AVX @@ -16,7 +13,7 @@ ENV WITH_AVX=${WITH_AVX:-ON} ENV HOME /root # Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ +COPY tools/dockerfile/scripts/root/ /root/ # Prepare packages for Python RUN apt-get update && \ @@ -24,27 +21,39 @@ RUN apt-get update && \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + # Downgrade gcc&&g++ -RUN apt-get update -WORKDIR /usr/bin -RUN apt install -y gcc-4.8 g++-4.8 -RUN cp gcc gcc.bak -RUN cp g++ g++.bak -RUN rm gcc -RUN rm g++ -RUN ln -s gcc-4.8 gcc -RUN ln -s g++-4.8 g++ - -# Install cmake3.16.0 -RUN mkdir -p /root/cmake_build && wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && \ - tar -zxvf cmake-3.16.0.tar.gz && rm cmake-3.16.0.tar.gz && \ - cd cmake-3.16.0 && ./bootstrap > /dev/null && \ - make -j8 > /dev/null && make install > /dev/null && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -ENV PATH=/usr/local/bin:$PATH - -RUN rm -r /root/cmake_build +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH # Install Python3.6 RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ @@ -53,79 +62,76 @@ RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-a wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig # Install Python3.7 RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig -RUN rm -r /root/python_build +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - libnccl2=2.1.2-1+cuda8.0 libnccl-dev=2.1.2-1+cuda8.0 \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool ccache && \ - apt-get clean -y +RUN rm -r /root/python_build # Install Python2.7.15 to replace original python WORKDIR /home ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/bin:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc ENV PATH=/usr/local/python2.7.15/include:${PATH} ENV PATH=/usr/local/python2.7.15/bin:${PATH} ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install +RUN python setup.py build && python setup.py install WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz -RUN tar -zxvf pip-18.0.tar.gz +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz WORKDIR pip-18.0 -RUN python setup.py install +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install WORKDIR /home RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ rm -r Python-$version setuptools-40.6.2 pip-18.0 # Install Go and glide -RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go # should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} # Install TensorRT # following TensorRT.tar.gz is not the default official one, we do two miny changes: @@ -134,10 +140,10 @@ RUN curl -s -q https://glide.sh/get | sh # 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. # See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT/include/* /usr/include/ && \ - cp -rf /usr/local/TensorRT/lib/* /usr/lib/ +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts # git credential to skip password typing RUN git config --global credential.helper store @@ -145,75 +151,37 @@ RUN git config --global credential.helper store # Fix locales to en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8 -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - - -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - -RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip --no-cache-dir install 'ipykernel==4.6.0' - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz && tar -xzf openmpi-1.4.5.tar.gz && \ - cd openmpi-1.4.5 && ./configure --prefix=/usr/local && make all -j8 && make install -j8 && \ - export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ - rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ - apt-get install libprotobuf-dev -y +RUN apt-get install libprotobuf-dev -y # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa # So install a newer version here. -RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ dpkg -i patchelf_0.10-2_amd64.deb # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.2-cudnn8 b/tools/Dockerfile.cuda10.2-cudnn8 new file mode 100644 index 0000000000000000000000000000000000000000..ad4b4993b5fb0b36ce61a0aca609f2e68be9f2e9 --- /dev/null +++ b/tools/Dockerfile.cuda10.2-cudnn8 @@ -0,0 +1,141 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:10.2-cudnn8-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + python-pip python-dev python-opencv \ + wget tar xz-utils bzip2 \ + curl sed grep \ + python-matplotlib \ + vim && \ + apt-get clean -y + +# Downgrade gcc&&g++ +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN apt-get install -y make automake + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +ENV PATH=/usr/local/include/python3.6m/:${PATH} +ENV PATH=/usr/local/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.6 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +RUN python3.6 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda10.2-cudnn8.devel b/tools/Dockerfile.cuda10.2-cudnn8.devel index daab4340e35702b1728a62f3b7cf5d131a20097d..ad4b4993b5fb0b36ce61a0aca609f2e68be9f2e9 100644 --- a/tools/Dockerfile.cuda10.2-cudnn8.devel +++ b/tools/Dockerfile.cuda10.2-cudnn8.devel @@ -1,12 +1,9 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment # When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04 +FROM hub.baidubce.com/ctr/cuda:10.2-cudnn8-devel-ubuntu16.04 MAINTAINER PaddlePaddle Authors -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - # ENV variables ARG WITH_GPU ARG WITH_AVX @@ -16,35 +13,35 @@ ENV WITH_AVX=${WITH_AVX:-ON} ENV HOME /root # Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ +COPY tools/dockerfile/scripts/root/ /root/ # Prepare packages for Python RUN apt-get update && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev + apt-get install -y make build-essential + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + python-pip python-dev python-opencv \ + wget tar xz-utils bzip2 \ + curl sed grep \ + python-matplotlib \ + vim && \ + apt-get clean -y # Downgrade gcc&&g++ -RUN apt-get update -WORKDIR /usr/bin -RUN apt install -y gcc-4.8 g++-4.8 -RUN cp gcc gcc.bak -RUN cp g++ g++.bak -RUN rm gcc -RUN rm g++ -RUN ln -s gcc-4.8 gcc -RUN ln -s g++-4.8 g++ - -# Install cmake3.16.0 -RUN mkdir -p /root/cmake_build && wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && \ - tar -zxvf cmake-3.16.0.tar.gz && rm cmake-3.16.0.tar.gz && \ - cd cmake-3.16.0 && ./bootstrap > /dev/null && \ - make -j8 > /dev/null && make install > /dev/null && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -ENV PATH=/usr/local/bin:$PATH - -RUN rm -r /root/cmake_build +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN apt-get install -y make automake # Install Python3.6 RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ @@ -53,79 +50,56 @@ RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-a wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig -# Install Python3.7 -RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ - tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null +ENV PATH=/usr/local/include/python3.6m/:${PATH} +ENV PATH=/usr/local/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 RUN rm -r /root/python_build -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - libnccl2=2.1.2-1+cuda8.0 libnccl-dev=2.1.2-1+cuda8.0 \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool ccache && \ - apt-get clean -y - # Install Python2.7.15 to replace original python WORKDIR /home ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/bin:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc ENV PATH=/usr/local/python2.7.15/include:${PATH} ENV PATH=/usr/local/python2.7.15/bin:${PATH} ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install +RUN python setup.py build && python setup.py install WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz -RUN tar -zxvf pip-18.0.tar.gz +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz WORKDIR pip-18.0 -RUN python setup.py install +RUN python setup.py install && \ + python3.6 setup.py install WORKDIR /home RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ rm -r Python-$version setuptools-40.6.2 pip-18.0 # Install Go and glide -RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go # should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} # Install TensorRT # following TensorRT.tar.gz is not the default official one, we do two miny changes: @@ -134,10 +108,10 @@ RUN curl -s -q https://glide.sh/get | sh # 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. # See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT/include/* /usr/include/ && \ - cp -rf /usr/local/TensorRT/lib/* /usr/lib/ +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts # git credential to skip password typing RUN git config --global credential.helper store @@ -145,75 +119,23 @@ RUN git config --global credential.helper store # Fix locales to en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8 -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - - -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - -RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip --no-cache-dir install 'ipykernel==4.6.0' - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz && tar -xzf openmpi-1.4.5.tar.gz && \ - cd openmpi-1.4.5 && ./configure --prefix=/usr/local && make all -j8 && make install -j8 && \ - export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ - rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ - apt-get install libprotobuf-dev -y - # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa # So install a newer version here. -RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ - dpkg -i patchelf_0.10-2_amd64.deb # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config CMD source ~/.bashrc + +RUN python3.6 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + EXPOSE 22 diff --git a/tools/Dockerfile.cuda11-cudnn8 b/tools/Dockerfile.cuda11-cudnn8 new file mode 100644 index 0000000000000000000000000000000000000000..07f1ba0f5d387e889843d1f1dc4e108b81e12930 --- /dev/null +++ b/tools/Dockerfile.cuda11-cudnn8 @@ -0,0 +1,189 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:11.0-cudnn8-devel-ubuntu18.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} +ENV DEBIAN_FRONTEND=noninteractive +ENV LD_LIBRARY_PATH=/usr/local/cuda-11.0/targets/x86_64-linux/lib:$LD_LIBRARY_PATH + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +RUN apt-get update && \ + apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip \ + coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx \ + bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool module-init-tools libcurl4-openssl-dev libffi-dev + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so.1.0.0 /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 /usr/lib/libcrypto.so.10 + +# Downgrade gcc&&g++ +WORKDIR /usr/bin +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts +RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ +RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc +RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ +RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc +RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ +ENV PATH=/usr/local/gcc-8.2/bin:$PATH + + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# remove them when apt-get support 2.27 and higher version +RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ + tar -xzf binutils-2.33.1.tar.gz && \ + cd binutils-2.33.1 && \ + ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz + + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH + + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +#CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN ln -s /usr/share/pyshared/lsb_release.py /usr/bin/lsb_release.py +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda11-cudnn8.devel b/tools/Dockerfile.cuda11-cudnn8.devel index daab4340e35702b1728a62f3b7cf5d131a20097d..07f1ba0f5d387e889843d1f1dc4e108b81e12930 100644 --- a/tools/Dockerfile.cuda11-cudnn8.devel +++ b/tools/Dockerfile.cuda11-cudnn8.devel @@ -1,50 +1,54 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment # When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04 +FROM hub.baidubce.com/ctr/cuda:11.0-cudnn8-devel-ubuntu18.04 MAINTAINER PaddlePaddle Authors -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - # ENV variables ARG WITH_GPU ARG WITH_AVX ENV WITH_GPU=${WITH_GPU:-ON} ENV WITH_AVX=${WITH_AVX:-ON} +ENV DEBIAN_FRONTEND=noninteractive +ENV LD_LIBRARY_PATH=/usr/local/cuda-11.0/targets/x86_64-linux/lib:$LD_LIBRARY_PATH ENV HOME /root # Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ +COPY tools/dockerfile/scripts/root/ /root/ -# Prepare packages for Python RUN apt-get update && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev + apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip \ + coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx \ + bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool module-init-tools libcurl4-openssl-dev libffi-dev + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so.1.0.0 /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 /usr/lib/libcrypto.so.10 # Downgrade gcc&&g++ -RUN apt-get update -WORKDIR /usr/bin -RUN apt install -y gcc-4.8 g++-4.8 -RUN cp gcc gcc.bak -RUN cp g++ g++.bak -RUN rm gcc -RUN rm g++ -RUN ln -s gcc-4.8 gcc -RUN ln -s g++-4.8 g++ - -# Install cmake3.16.0 -RUN mkdir -p /root/cmake_build && wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && \ - tar -zxvf cmake-3.16.0.tar.gz && rm cmake-3.16.0.tar.gz && \ - cd cmake-3.16.0 && ./bootstrap > /dev/null && \ - make -j8 > /dev/null && make install > /dev/null && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -ENV PATH=/usr/local/bin:$PATH - -RUN rm -r /root/cmake_build +WORKDIR /usr/bin +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts +RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ +RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc +RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ +RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc +RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ +ENV PATH=/usr/local/gcc-8.2/bin:$PATH + + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH # Install Python3.6 RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ @@ -53,79 +57,83 @@ RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-a wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig # Install Python3.7 RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig -RUN rm -r /root/python_build +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - python3 python3-dev python3-pip \ - git python-pip python-dev python-opencv openssh-server bison \ - libnccl2=2.1.2-1+cuda8.0 libnccl-dev=2.1.2-1+cuda8.0 \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib gcc-4.8 g++-4.8 \ - automake locales clang-format swig \ - liblapack-dev liblapacke-dev \ - clang-3.8 llvm-3.8 libclang-3.8-dev \ - net-tools libtool ccache && \ - apt-get clean -y +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build # Install Python2.7.15 to replace original python WORKDIR /home ENV version=2.7.15 -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz -RUN tar -xvf Python-$version.tgz +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz WORKDIR /home/Python-$version -RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 -RUN make && make install - -RUN echo "export PATH=/usr/local/bin:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc -RUN echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc -RUN echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc -RUN echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc ENV PATH=/usr/local/python2.7.15/include:${PATH} ENV PATH=/usr/local/python2.7.15/bin:${PATH} ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH -RUN mv /usr/bin/python /usr/bin/python.bak -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python -RUN ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip -RUN apt-get -y install unzip -RUN unzip setuptools-40.6.2.zip +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip WORKDIR /home/setuptools-40.6.2 -RUN python setup.py build -RUN python setup.py install +RUN python setup.py build && python setup.py install WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz -RUN tar -zxvf pip-18.0.tar.gz +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz WORKDIR pip-18.0 -RUN python setup.py install +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install WORKDIR /home RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ rm -r Python-$version setuptools-40.6.2 pip-18.0 +# remove them when apt-get support 2.27 and higher version +RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ + tar -xzf binutils-2.33.1.tar.gz && \ + cd binutils-2.33.1 && \ + ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz + + # Install Go and glide -RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH + # Install TensorRT # following TensorRT.tar.gz is not the default official one, we do two miny changes: @@ -134,10 +142,10 @@ RUN curl -s -q https://glide.sh/get | sh # 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. # See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. -RUN wget -q https://paddlepaddledeps.bj.bcebos.com/TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz --no-check-certificate && \ - tar -zxf TensorRT-4.0.1.6-ubuntu14.04.x86_64-gnu.cuda.8.0.cudnn7.0.tar.gz -C /usr/local && \ - cp -rf /usr/local/TensorRT/include/* /usr/include/ && \ - cp -rf /usr/local/TensorRT/lib/* /usr/lib/ +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts # git credential to skip password typing RUN git config --global credential.helper store @@ -145,75 +153,37 @@ RUN git config --global credential.helper store # Fix locales to en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8 -# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter -# version util jupyter fixes this issue. - - -RUN pip3 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - -RUN pip3 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ - pip --no-cache-dir install 'ipykernel==4.6.0' - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort -RUN pip3.7 --no-cache-dir install pylint pytest astroid isort -RUN pip --no-cache-dir install pylint pytest astroid isort LinkChecker - -RUN pip3 --no-cache-dir install coverage -RUN pip3.6 --no-cache-dir install coverage -RUN pip3.7 --no-cache-dir install coverage -RUN pip --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt -RUN pip3.6 --no-cache-dir install -r /root/requirements.txt -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt -RUN pip --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y -RUN pip3 --no-cache-dir install certifi urllib3[secure] -RUN pip3.6 --no-cache-dir install certifi urllib3[secure] -RUN pip3.7 --no-cache-dir install certifi urllib3[secure] -RUN pip --no-cache-dir install certifi urllib3[secure] - - - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz && tar -xzf openmpi-1.4.5.tar.gz && \ - cd openmpi-1.4.5 && ./configure --prefix=/usr/local && make all -j8 && make install -j8 && \ - export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ - rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ - apt-get install libprotobuf-dev -y +RUN apt-get install libprotobuf-dev -y # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa # So install a newer version here. -RUN wget -q http://mirrors.kernel.org/ubuntu/pool/universe/p/patchelf/patchelf_0.10-2_amd64.deb && \ +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ dpkg -i patchelf_0.10-2_amd64.deb # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc +#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +#CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN ln -s /usr/share/pyshared/lsb_release.py /usr/bin/lsb_release.py +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + EXPOSE 22 diff --git a/tools/Dockerfile.cuda9.0-cudnn7 b/tools/Dockerfile.cuda9.0-cudnn7 index ead1cfe77d7148789d84bc01fb05cedda5ff1fe6..eb715683be9f6da1d8a00eaa085a81688c11a3b3 100644 --- a/tools/Dockerfile.cuda9.0-cudnn7 +++ b/tools/Dockerfile.cuda9.0-cudnn7 @@ -1,24 +1,181 @@ -FROM nvidia/cuda:9.0-cudnn7-devel-centos7 as builder - -FROM nvidia/cuda:9.0-cudnn7-runtime-centos7 -RUN yum -y install wget && \ - yum -y install epel-release && yum -y install patchelf && \ - yum -y install gcc gcc-c++ make python-devel && \ - yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false && \ - yum -y install python3 python3-devel && \ - yum clean all - -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python get-pip.py && rm get-pip.py - -RUN ln -s /usr/local/cuda-9.0/lib64/libcublas.so.9.0 /usr/local/cuda-9.0/lib64/libcublas.so && \ - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> /root/.bashrc && \ - ln -s /usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudnn.so.7 /usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudnn.so && \ - echo 'export LD_LIBRARY_PATH=/usr/local/cuda-9.0/targets/x86_64-linux/lib:$LD_LIBRARY_PATH' >> /root/.bashrc && \ - echo "export LANG=en_US.utf8" >> /root/.bashrc && \ - echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc && \ - mkdir -p /usr/local/cuda/extras - -COPY --from=builder /usr/local/cuda/extras/CUPTI /usr/local/cuda/extras/CUPTI +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:9.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +RUN apt-get update + WORKDIR /usr/bin + RUN apt install -y gcc-4.8 g++-4.8 && cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ && ln -s gcc-4.8 gcc && ln -s g++-4.8 g++ + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.cuda9.0-cudnn7.devel b/tools/Dockerfile.cuda9.0-cudnn7.devel index 42b2d7eb5c0766c8a97130ec93ea5945607cf32b..eb715683be9f6da1d8a00eaa085a81688c11a3b3 100644 --- a/tools/Dockerfile.cuda9.0-cudnn7.devel +++ b/tools/Dockerfile.cuda9.0-cudnn7.devel @@ -1,42 +1,181 @@ -FROM nvidia/cuda:9.0-cudnn7-devel-centos7 -RUN yum -y install wget >/dev/null \ - && yum -y install gcc gcc-c++ make glibc-static which \ - && yum -y install git openssl-devel curl-devel bzip2-devel python-devel \ - && yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false \ - && yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false \ - && yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false - -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.2/protobuf-all-3.11.2.tar.gz && \ - tar zxf protobuf-all-3.11.2.tar.gz && \ - cd protobuf-3.11.2 && \ - ./configure && make -j4 && make install && \ - make clean && \ - cd .. && rm -rf protobuf-* - -RUN wget https://cmake.org/files/v3.2/cmake-3.2.0-Linux-x86_64.tar.gz >/dev/null \ - && tar xzf cmake-3.2.0-Linux-x86_64.tar.gz \ - && mv cmake-3.2.0-Linux-x86_64 /usr/local/cmake3.2.0 \ - && echo 'export PATH=/usr/local/cmake3.2.0/bin:$PATH' >> /root/.bashrc \ - && rm cmake-3.2.0-Linux-x86_64.tar.gz - -RUN wget https://dl.google.com/go/go1.14.linux-amd64.tar.gz >/dev/null \ - && tar xzf go1.14.linux-amd64.tar.gz \ - && mv go /usr/local/go \ - && echo 'export GOROOT=/usr/local/go' >> /root/.bashrc \ - && echo 'export PATH=/usr/local/go/bin:$PATH' >> /root/.bashrc \ - && rm go1.14.linux-amd64.tar.gz - -RUN yum -y install python-devel sqlite-devel \ - && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py >/dev/null \ - && python get-pip.py >/dev/null \ - && rm get-pip.py \ - && pip install requests - -RUN yum install -y python3 python3-devel \ - && yum -y install epel-release && yum -y install patchelf libXext libSM libXrender\ - && yum clean all \ - && pip3 install requests - -RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8 \ - && echo "export LANG=en_US.utf8" >> /root/.bashrc \ - && echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:9.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +RUN apt-get update + WORKDIR /usr/bin + RUN apt install -y gcc-4.8 g++-4.8 && cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ && ln -s gcc-4.8 gcc && ln -s g++-4.8 g++ + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/Dockerfile.devel b/tools/Dockerfile.devel index a0f1d03983466b3ca70ce2d4673ad8874e323a08..ab83229af56c65cb9505b524eb08fe5ce2931a5b 100644 --- a/tools/Dockerfile.devel +++ b/tools/Dockerfile.devel @@ -1,32 +1,187 @@ -FROM centos:7.3.1611 -RUN yum -y install wget \ - && yum -y install gcc gcc-c++ make glibc-static which \ - && yum -y install git openssl-devel curl-devel bzip2-devel python-devel - -RUN wget https://cmake.org/files/v3.2/cmake-3.2.0-Linux-x86_64.tar.gz >/dev/null \ - && tar xzf cmake-3.2.0-Linux-x86_64.tar.gz \ - && mv cmake-3.2.0-Linux-x86_64 /usr/local/cmake3.2.0 \ - && echo 'export PATH=/usr/local/cmake3.2.0/bin:$PATH' >> /root/.bashrc \ - && rm cmake-3.2.0-Linux-x86_64.tar.gz - -RUN wget https://dl.google.com/go/go1.14.linux-amd64.tar.gz >/dev/null \ - && tar xzf go1.14.linux-amd64.tar.gz \ - && mv go /usr/local/go \ - && echo 'export GOROOT=/usr/local/go' >> /root/.bashrc \ - && echo 'export PATH=/usr/local/go/bin:$PATH' >> /root/.bashrc \ - && rm go1.14.linux-amd64.tar.gz - -RUN yum -y install python-devel sqlite-devel \ - && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py >/dev/null \ - && python get-pip.py >/dev/null \ - && rm get-pip.py \ - && pip install requests - -RUN yum install -y python3 python3-devel \ - && yum -y install epel-release && yum -y install patchelf libXext libSM libXrender\ - && yum clean all \ - && pip3 install requests - -RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8 \ - && echo "export LANG=en_US.utf8" >> /root/.bashrc \ - && echo "export LANGUAGE=en_US.utf8" >> /root/.bashrc +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +# When you modify it, please be aware of cudnn-runtime version +FROM hub.baidubce.com/ctr/cuda:9.0-cudnn7-devel-ubuntu16.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_GPU +ARG WITH_AVX + +ENV WITH_GPU=${WITH_GPU:-ON} +ENV WITH_AVX=${WITH_AVX:-ON} + +ENV HOME /root +# Add bash enhancements +COPY tools/dockerfile/scripts/root/ /root/ + +# Prepare packages for Python +RUN apt-get update && \ + apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev + +RUN apt-get update && \ + apt-get install -y --allow-downgrades --allow-change-held-packages \ + patchelf git python-pip python-dev python-opencv openssh-server bison \ + wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-matplotlib \ + automake locales clang-format swig \ + liblapack-dev liblapacke-dev libcurl4-openssl-dev \ + net-tools libtool module-init-tools vim && \ + apt-get clean -y + +RUN ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so.10 && \ + ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so /usr/lib/libcrypto.so.10 + +RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ + rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 + +# Downgrade gcc&&g++ +WORKDIR /usr/bin + COPY tools/dockerfile/build_scripts /build_scripts + RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts + RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ + RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc + RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ + ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install cmake +WORKDIR /home +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz +ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH + +# Install Python3.6 +RUN mkdir -p /root/python_build/ && wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz && \ + tar -zxf sqlite-autoconf-3250300.tar.gz && cd sqlite-autoconf-3250300 && \ + ./configure -prefix=/usr/local && make -j8 && make install && cd ../ && rm sqlite-autoconf-3250300.tar.gz && \ + wget -q https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \ + tar -xzf Python-3.6.0.tgz && cd Python-3.6.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.7 +RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ + tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.8 +RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ + tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig + +# Install Python3.5 +RUN wget -q https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz && \ + tar -xzf Python-3.5.1.tgz && cd Python-3.5.1 && \ + CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.5.1 --enable-shared > /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && ldconfig +ENV PATH=/usr/local/include/python3.6m/:/usr/local/python3.5.1/include:${PATH} +ENV PATH=/usr/local/bin:/usr/local/python3.5.1/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/python3.5.1/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python3.5.1/include/python3.5:/usr/local/include/python3.6m/:$CPLUS_INCLUDE_PATH +RUN ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/local/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/python3.5 /usr/bin/python3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/local/bin/pip3.5 && ln -sf /usr/local/python3.5.1/bin/pip3.5 /usr/bin/pip3.5 && ln -sf /usr/local/bin/python3.6 /usr/local/bin/python3 && ln -sf /usr/local/bin/python3.6 /usr/bin/python3 && ln -sf /usr/local/bin/pip3.6 /usr/local/bin/pip3 && ln -sf /usr/local/bin/pip3.6 /usr/bin/pip3 + +RUN rm -r /root/python_build + +# Install Python2.7.15 to replace original python +WORKDIR /home +ENV version=2.7.15 +RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && tar -xvf Python-$version.tgz +WORKDIR /home/Python-$version +RUN ./configure --enable-unicode=ucs4 --enable-shared CFLAGS=-fPIC --prefix=/usr/local/python2.7.15 && make && make install + +RUN echo "export PATH=/usr/local/python2.7.15/include:${PATH}" >> ~/.bashrc && echo "export PATH=/usr/local/python2.7.15/bin:${PATH}" >> ~/.bashrc && echo "export LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH}" >> ~/.bashrc && echo "export CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH" >> ~/.bashrc +ENV PATH=/usr/local/python2.7.15/include:${PATH} +ENV PATH=/usr/local/python2.7.15/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/python2.7.15/lib:${LD_LIBRARY_PATH} +ENV CPLUS_INCLUDE_PATH=/usr/local/python2.7.15/include/python2.7:$CPLUS_INCLUDE_PATH +RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/local/bin/python && ln -s /usr/local/python2.7.15/bin/python2.7 /usr/bin/python + +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip && apt-get -y install unzip && unzip setuptools-40.6.2.zip +WORKDIR /home/setuptools-40.6.2 +RUN python setup.py build && python setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/69/81/52b68d0a4de760a2f1979b0931ba7889202f302072cc7a0d614211bc7579/pip-18.0.tar.gz && tar -zxvf pip-18.0.tar.gz +WORKDIR pip-18.0 +RUN python setup.py install && \ + python3.8 setup.py install && \ + python3.7 setup.py install && \ + python3.6 setup.py install && \ + python3.5 setup.py install + +WORKDIR /home +RUN rm Python-$version.tgz setuptools-40.6.2.zip pip-18.0.tar.gz && \ + rm -r Python-$version setuptools-40.6.2 pip-18.0 + +# Install Go and glide +RUN wget -qO- https://dl.google.com/go/go1.14.linux-amd64.tar.gz | \ + tar -xz -C /usr/local && \ + mkdir /root/go && \ + mkdir /root/go/bin && \ + mkdir /root/go/src && \ + echo "GOROOT=/usr/local/go" >> /root/.bashrc && \ + echo "GOPATH=/root/go" >> /root/.bashrc && \ + echo "PATH=/usr/local/go/bin:/root/go/bin:$PATH" >> /root/.bashrc +ENV GOROOT=/usr/local/go GOPATH=/root/go +# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. +ENV PATH=usr/local/go/bin:/root/go/bin:${PATH} + +# Install TensorRT +# following TensorRT.tar.gz is not the default official one, we do two miny changes: +# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, +# and its size is only one-third of the official one. +# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. +# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. + +# Downgrade TensorRT +COPY tools/dockerfile/build_scripts /build_scripts +RUN bash /build_scripts/install_trt.sh +RUN rm -rf /build_scripts + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN apt-get install libprotobuf-dev -y + +# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. +# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa +# So install a newer version here. +RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \ + dpkg -i patchelf_0.10-2_amd64.deb + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +CMD source ~/.bashrc + +# ccache 3.7.9 +RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ + tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ + ./configure -prefix=/usr/local/ccache-3.7.9 && \ + make -j8 && make install && \ + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache + +RUN python3.8 -m pip install --upgrade pip requests && \ + python3.7 -m pip install --upgrade pip requests && \ + python3.6 -m pip install --upgrade pip requests && \ + python3.5 -m pip install --upgrade pip requests && \ + python2.7 -m pip install --upgrade pip requests + +RUN wget https://paddle-serving.bj.bcebos.com/others/centos_ssl.tar && \ + tar xf centos_ssl.tar && rm -rf centos_ssl.tar && \ + mv libcrypto.so.1.0.2k /usr/lib/libcrypto.so.1.0.2k && mv libssl.so.1.0.2k /usr/lib/libssl.so.1.0.2k && \ + ln -sf /usr/lib/libcrypto.so.1.0.2k /usr/lib/libcrypto.so.10 && \ + ln -sf /usr/lib/libssl.so.1.0.2k /usr/lib/libssl.so.10 && \ + ln -sf /usr/lib/libcrypto.so.10 /usr/lib/libcrypto.so && \ + ln -sf /usr/lib/libssl.so.10 /usr/lib/libssl.so + +EXPOSE 22 diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..7d5e019443229e116599f804f714e599fcc72fbc --- /dev/null +++ b/tools/dockerfile/build_scripts/build.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -ex + +# Python versions to be installed in /opt/$VERSION_NO +# NOTE Only need python 2.7.11 for nupic.core/nupic.bindings at this time, so +# remove others to expedite build and reduce docker image size. The original +# manylinux docker image project builds many python versions. +# NOTE We added back 3.5.1, since auditwheel requires python 3.3+ +CPYTHON_VERSIONS="3.8.0 3.7.0 3.6.0 3.5.1 2.7.15" + +# openssl version to build, with expected sha256 hash of .tar.gz +# archive +OPENSSL_ROOT=openssl-1.0.2g +OPENSSL_HASH=b784b1b3907ce39abf4098702dade6365522a253ad1552e267a9a0e89594aa33 +PATCHELF_HASH=f2aa40a6148cb3b0ca807a1bf836b081793e55ec9e5540a5356d800132be7e0a +CURL_ROOT=curl-7.49.1 +CURL_HASH=eb63cec4bef692eab9db459033f409533e6d10e20942f4b060b32819e81885f1 +AUTOCONF_ROOT=autoconf-2.69 +AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 + +# Dependencies for compiling Python that we want to remove from +# the final image after compiling Python +PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel" + +# Libraries that are allowed as part of the manylinux1 profile +MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel" + +# Get build utilities +MY_DIR=$(dirname "${BASH_SOURCE[0]}") +source $MY_DIR/build_utils.sh + +# EPEL support +yum -y install wget curl epel-release + +# Development tools and libraries +yum -y install bzip2 make git patch unzip bison yasm diffutils \ + automake which file \ + kernel-devel-`uname -r` \ + devtoolset-2-binutils devtoolset-2-gcc \ + devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran \ + ${PYTHON_COMPILE_DEPS} + +# Install more recent version of cmake +# curl -O https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.sh +# /bin/sh cmake-3.8.1-Linux-x86_64.sh --prefix=/usr/local --skip-license +# rm cmake-3.8.1-Linux-x86_64.sh + +wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && tar xzf cmake-3.16.0.tar.gz && \ +cd cmake-3.16.0 && ./bootstrap && \ +make -j8 && make install && cd .. && rm cmake-3.16.0.tar.gz && rm -rf cmake-3.16.0 + +# Install newest autoconf +build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH +autoconf --version + +# Compile the latest Python releases. +# (In order to have a proper SSL module, Python is compiled +# against a recent openssl [see env vars above], which is linked +# statically. We delete openssl afterwards.) +build_openssl $OPENSSL_ROOT $OPENSSL_HASH +mkdir -p /opt/python +build_cpythons $CPYTHON_VERSIONS + +PY35_BIN=/opt/python/cp35-cp35m/bin +PY36_BIN=/opt/python/cp36-cp36m/bin +PY37_BIN=/opt/python/cp37-cp37m/bin +PY38_BIN=/opt/python/cp38-cp38m/bin +# NOTE Since our custom manylinux image builds pythons with shared +# libpython, we need to add libpython's dir to LD_LIBRARY_PATH before running +# python. +ORIGINAL_LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib:$(dirname ${PY36_BIN})/lib:$(dirname ${PY37_BIN})/lib:$(dirname ${PY38_BIN})/lib" + +# Our openssl doesn't know how to find the system CA trust store +# (https://github.com/pypa/manylinux/issues/53) +# And it's not clear how up-to-date that is anyway +# So let's just use the same one pip and everyone uses +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install certifi +ln -s $($PY35_BIN/python -c 'import certifi; print(certifi.where())') \ + /opt/_internal/certs.pem +# If you modify this line you also have to modify the versions in the +# Dockerfiles: +export SSL_CERT_FILE=/opt/_internal/certs.pem + +# Install newest curl +build_curl $CURL_ROOT $CURL_HASH +rm -rf /usr/local/include/curl /usr/local/lib/libcurl* /usr/local/lib/pkgconfig/libcurl.pc +hash -r +curl --version +curl-config --features + +# Install patchelf (latest with unreleased bug fixes) +# FIXME(typhoonzero): restore this when the link is fixed. +# curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz +# check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH +# tar -xzf patchelf-0.9njs2.tar.gz +# (cd patchelf-0.9njs2 && ./configure && make && make install) +# rm -rf patchelf-0.9njs2.tar.gz patchelf-0.9njs2 +yum install -y patchelf + +# Install latest pypi release of auditwheel +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install auditwheel +ln -s $PY35_BIN/auditwheel /usr/local/bin/auditwheel + +# Clean up development headers and other unnecessary stuff for +# final image +yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \ + avahi freetype bitstream-vera-fonts \ + ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 || true +yum -y install ${MANYLINUX1_DEPS} && yum -y clean all > /dev/null 2>&1 || true +yum list installed +# we don't need libpython*.a, and they're many megabytes +find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f +# Strip what we can -- and ignore errors, because this just attempts to strip +# *everything*, including non-ELF files: +find /opt/_internal -type f -print0 \ + | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true +# We do not need the Python test suites, or indeed the precompiled .pyc and +# .pyo files. Partially cribbed from: +# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile +find /opt/_internal \ + \( -type d -a -name test -o -name tests \) \ + -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + -print0 | xargs -0 rm -f + +for PYTHON in /opt/python/*/bin/python; do + # Add matching directory of libpython shared library to library lookup path + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" + + # Smoke test to make sure that our Pythons work, and do indeed detect as + # being manylinux compatible: + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/manylinux1-check.py + # Make sure that SSL cert checking works + LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname $(dirname ${PYTHON}))/lib" $PYTHON $MY_DIR/ssl-check.py +done + +# Restore LD_LIBRARY_PATH +LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" + +# According to ar issues: https://lists.gnu.org/archive/html/bug-binutils/2016-05/msg00211.html +# we should install new version ar with 64-bit supported here +wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz +tar xzf binutils-2.27.tar.gz && cd binutils-2.27 +./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install +cd .. && rm binutils-2.27.tar.gz && rm -rf binutils-2.27 diff --git a/tools/dockerfile/build_scripts/build_utils.sh b/tools/dockerfile/build_scripts/build_utils.sh new file mode 100755 index 0000000000000000000000000000000000000000..b69ef8d628e4638cf07cc238d8b679d2874a5c67 --- /dev/null +++ b/tools/dockerfile/build_scripts/build_utils.sh @@ -0,0 +1,223 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper utilities for build + +PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python +# XXX: the official https server at www.openssl.org cannot be reached +# with the old versions of openssl and curl in Centos 5.11 hence the fallback +# to the ftp mirror: +# OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source +# Ditto the curl sources +CURL_DOWNLOAD_URL=http://curl.askapache.com/download + +GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py + +AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf + + +function check_var { + if [ -z "$1" ]; then + echo "required variable not defined" + exit 1 + fi +} + + +function lex_pyver { + # Echoes Python version string padded with zeros + # Thus: + # 3.2.1 -> 003002001 + # 3 -> 003000000 + echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' +} + + +function do_cpython_build { + local py_ver=$1 + check_var $py_ver + local ucs_setting=$2 + check_var $ucs_setting + tar -xzf Python-$py_ver.tgz + pushd Python-$py_ver + if [ "$ucs_setting" = "none" ]; then + unicode_flags="" + dir_suffix="" + else + local unicode_flags="--enable-unicode=$ucs_setting" + local dir_suffix="-$ucs_setting" + fi + local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}" + mkdir -p ${prefix}/lib + # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6 + + if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.6) ]; then + wget -q https://www.sqlite.org/2018/sqlite-autoconf-3250300.tar.gz + tar -zxf sqlite-autoconf-3250300.tar.gz + cd sqlite-autoconf-3250300 + ./configure --prefix=/usr/local + make -j8 && make install + cd ../ && rm sqlite-autoconf-3250300.tar.gz + fi + + # NOTE --enable-shared for generating libpython shared library needed for + # linking of some of the nupic.core test executables. + if [ $(lex_pyver $py_ver) -ge $(lex_pyver 3.7) ]; then + # NOTE python 3.7 should be installed via make altinstall rather than + # make install, and we should specify the location of ssl + CFLAGS="-Wformat" ./configure --prefix=${prefix} --with-openssl=/usr/local/ssl --enable-shared $unicode_flags > /dev/null + make -j8 > /dev/null + make altinstall > /dev/null + else + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} CFLAGS="-Wformat" ./configure --prefix=${prefix} --enable-shared $unicode_flags > /dev/null + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make -j8 > /dev/null + LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make install > /dev/null + fi + popd + echo "ZZZ looking for libpython" + find / -name 'libpython*.so*' + rm -rf Python-$py_ver + # Some python's install as bin/python3. Make them available as + # bin/python. + if [ -e ${prefix}/bin/python3 ]; then + ln -s python3 ${prefix}/bin/python + fi + if [ -e ${prefix}/bin/python3.7 ]; then + ln -s python3.7 ${prefix}/bin/python + fi + if [ -e ${prefix}/bin/python3.8 ]; then + ln -s python3.8 ${prefix}/bin/python + fi + # NOTE Make libpython shared library visible to python calls below + LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py + LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2 + cd / + ls ${MY_DIR} + local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${prefix} /opt/python/${abi_tag} +} + + +function build_cpython { + local py_ver=$1 + check_var $py_ver + check_var $PYTHON_DOWNLOAD_URL + wget -q $PYTHON_DOWNLOAD_URL/$py_ver/Python-$py_ver.tgz + if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then + # NOTE We only need wide unicode for nupic.bindings wheel + do_cpython_build $py_ver ucs2 + do_cpython_build $py_ver ucs4 + else + do_cpython_build $py_ver none + fi + rm -f Python-$py_ver.tgz +} + + +function build_cpythons { + for py_ver in $@; do + check_var $GET_PIP_URL + curl -sLO $GET_PIP_URL + if [ $py_ver == "2.7.15" ]; then + curl -sLO https://bootstrap.pypa.io/2.7/get-pip.py + fi + if [ $py_ver == "3.5.1" ]; then + curl -sLO https://bootstrap.pypa.io/3.5/get-pip.py + fi + build_cpython $py_ver + done + rm get-pip.py +} + + +function do_openssl_build { + ./config -fPIC --prefix=/usr/local/ssl > /dev/null + make > /dev/null + make install > /dev/null + +} + + +function check_sha256sum { + local fname=$1 + check_var ${fname} + local sha256=$2 + check_var ${sha256} + + echo "${sha256} ${fname}" > ${fname}.sha256 + sha256sum -c ${fname}.sha256 + rm ${fname}.sha256 +} + + +function build_openssl { + local openssl_fname=$1 + check_var ${openssl_fname} + local openssl_sha256=$2 + check_var ${openssl_sha256} + check_var ${OPENSSL_DOWNLOAD_URL} + curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz + check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} + tar -xzf ${openssl_fname}.tar.gz + (cd ${openssl_fname} && do_openssl_build) + rm -rf ${openssl_fname} ${openssl_fname}.tar.gz +} + + +function do_curl_build { + LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null + make > /dev/null + make install > /dev/null + ln -s /usr/local/ssl/lib/libcrypto.so /usr/lib/libcrypto.so + ln -s /usr/local/ssl/lib/libssl.so /usr/lib/libssl.so + ln -s /usr/local/ssl/bin/openssl /usr/local/bin/openssl +} + + +function build_curl { + local curl_fname=$1 + check_var ${curl_fname} + local curl_sha256=$2 + check_var ${curl_sha256} + check_var ${CURL_DOWNLOAD_URL} + curl -sLO ${CURL_DOWNLOAD_URL}/${curl_fname}.tar.bz2 + check_sha256sum ${curl_fname}.tar.bz2 ${curl_sha256} + tar -jxf ${curl_fname}.tar.bz2 + (cd ${curl_fname} && do_curl_build) + rm -rf ${curl_fname} ${curl_fname}.tar.bz2 +} + + +function do_standard_install { + ./configure > /dev/null + make > /dev/null + make install > /dev/null +} + + +function build_autoconf { + local autoconf_fname=$1 + check_var ${autoconf_fname} + local autoconf_sha256=$2 + check_var ${autoconf_sha256} + check_var ${AUTOCONF_DOWNLOAD_URL} + curl -sLO ${AUTOCONF_DOWNLOAD_URL}/${autoconf_fname}.tar.gz + check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256} + tar -zxf ${autoconf_fname}.tar.gz + (cd ${autoconf_fname} && do_standard_install) + rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz +} diff --git a/tools/dockerfile/build_scripts/install_gcc.sh b/tools/dockerfile/build_scripts/install_gcc.sh new file mode 100644 index 0000000000000000000000000000000000000000..e75021b2a9b6531701ef1365e9e6195b61770632 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_gcc.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -ex + +if [ -f "/etc/redhat-release" ];then + lib_so_5=/usr/lib64/libgfortran.so.5 + lib_so_6=/usr/lib64/libstdc++.so.6 + lib_path=/usr/lib64 +else + lib_so_5=/usr/lib/x86_64-linux-gnu/libstdc++.so.5 + lib_so_6=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 + lib_path=/usr/lib/x86_64-linux-gnu +fi + +if [ "$1" == "gcc82" ]; then + wget -q https://paddle-ci.gz.bcebos.com/gcc-8.2.0.tar.xz + tar -xvf gcc-8.2.0.tar.xz && \ + cd gcc-8.2.0 && \ + unset LIBRARY_PATH CPATH C_INCLUDE_PATH PKG_CONFIG_PATH CPLUS_INCLUDE_PATH INCLUDE && \ + ./contrib/download_prerequisites && \ + cd .. && mkdir temp_gcc82 && cd temp_gcc82 && \ + ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ + make -j8 && make install + cd .. && rm -rf temp_gcc82 + cp ${lib_so_6} ${lib_so_6}.bak && rm -f ${lib_so_6} && + ln -s /usr/local/gcc-8.2/lib64/libgfortran.so.5 ${lib_so_5} && \ + ln -s /usr/local/gcc-8.2/lib64/libstdc++.so.6 ${lib_so_6} && \ + cp /usr/local/gcc-8.2/lib64/libstdc++.so.6.0.25 ${lib_path} +fi diff --git a/tools/dockerfile/build_scripts/install_nccl2.sh b/tools/dockerfile/build_scripts/install_nccl2.sh new file mode 100644 index 0000000000000000000000000000000000000000..b06b3d44c6ec6b32688521a76f26b82acb4e0998 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_nccl2.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| sed "s/release //") +if [ "$VERSION" == "10.0" ]; then + DEB="nccl-repo-ubuntu1604-2.4.7-ga-cuda10.0_1-1_amd64.deb" +elif [ "$VERSION" == "10.2" ] || [ "$VERSION" == "10.1" ] || [ "$VERSION" == "11.0" ]; then + if [ -f "/etc/redhat-release" ];then + rm -f /usr/local/lib/libnccl.so + wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.7.8-1+cuda10.2.x86_64.rpm + wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm + wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm + rpm -ivh libnccl-2.7.8-1+cuda10.2.x86_64.rpm + rpm -ivh libnccl-devel-2.7.8-1+cuda10.2.x86_64.rpm + rpm -ivh libnccl-static-2.7.8-1+cuda10.2.x86_64.rpm && rm -f libnccl-* + exit 0 + fi + DEB="nccl-repo-ubuntu1604-2.7.8-ga-cuda10.2_1-1_amd64.deb" +elif [ "$VERSION" == "9.0" ]; then + DEB="nccl-repo-ubuntu1604-2.3.7-ga-cuda9.0_1-1_amd64.deb" +else + echo "nccl not found" + exit 2 +fi + +URL="http://nccl2-deb.cdn.bcebos.com/$DEB" + +DIR="/nccl2" +mkdir -p $DIR +# we cached the nccl2 deb package in BOS, so we can download it with wget +# install nccl2: http://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html#down +wget -q -O $DIR/$DEB $URL + +cd $DIR && ar x $DEB && tar xf data.tar.xz +DEBS=$(find ./var/ -name "*.deb") +for sub_deb in $DEBS; do + echo $sub_deb + ar x $sub_deb && tar xf data.tar.xz +done +mv -f usr/include/nccl.h /usr/local/include/ +mv -f usr/lib/x86_64-linux-gnu/libnccl* /usr/local/lib/ +rm /usr/include/nccl.h +rm -rf $DIR diff --git a/tools/dockerfile/build_scripts/install_trt.sh b/tools/dockerfile/build_scripts/install_trt.sh new file mode 100644 index 0000000000000000000000000000000000000000..e5ec70d2f378d180a08a86d705f3e662a211dc91 --- /dev/null +++ b/tools/dockerfile/build_scripts/install_trt.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VERSION=$(nvcc --version | grep release | grep -oEi "release ([0-9]+)\.([0-9])"| sed "s/release //") + +if [[ "$VERSION" == "10.1" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.1-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda10.1-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda10.1-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda10.1-cudnn7/lib/* /usr/lib/ + rm TensorRT6-cuda10.1-cudnn7.tar.gz +elif [[ "$VERSION" == "11.0" ]];then + wget -q https://paddle-ci.cdn.bcebos.com/TRT/TensorRT-7.1.3.4.Ubuntu-16.04.x86_64-gnu.cuda-11.0.cudnn8.0.tar.gz --no-check-certificate + tar -zxf TensorRT-7.1.3.4.Ubuntu-16.04.x86_64-gnu.cuda-11.0.cudnn8.0.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/ + rm TensorRT-7.1.3.4.Ubuntu-16.04.x86_64-gnu.cuda-11.0.cudnn8.0.tar.gz +elif [[ "$VERSION" == "10.2" ]];then + wget https://paddle-ci.gz.bcebos.com/TRT/TensorRT7-cuda10.2-cudnn8.tar.gz --no-check-certificate + tar -zxf TensorRT7-cuda10.2-cudnn8.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-7.1.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-7.1.3.4/lib/* /usr/lib/ + rm TensorRT7-cuda10.2-cudnn8.tar.gz +elif [[ "$VERSION" == "10.0" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda10.0-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda10.0-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda10.0-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda10.0-cudnn7/lib/* /usr/lib/ + rm TensorRT6-cuda10.0-cudnn7.tar.gz +elif [[ "$VERSION" == "9.0" ]];then + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT6-cuda9.0-cudnn7.tar.gz --no-check-certificate + tar -zxf TensorRT6-cuda9.0-cudnn7.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT6-cuda9.0-cudnn7/include/* /usr/include/ && cp -rf /usr/local/TensorRT6-cuda9.0-cudnn7/lib/* /usr/lib/ + rm TensorRT6-cuda9.0-cudnn7.tar.gz +fi diff --git a/tools/dockerfile/build_scripts/manylinux1-check.py b/tools/dockerfile/build_scripts/manylinux1-check.py new file mode 100644 index 0000000000000000000000000000000000000000..0d1a6df4eec98c72e493517d54fae7c416727d38 --- /dev/null +++ b/tools/dockerfile/build_scripts/manylinux1-check.py @@ -0,0 +1,70 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Logic copied from PEP 513 + + +def is_manylinux1_compatible(): + # Only Linux, and only x86-64 / i686 + from distutils.util import get_platform + if get_platform() not in ["linux-x86_64", "linux-i686"]: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. + return have_compatible_glibc(2, 5) + + +def have_compatible_glibc(major, minimum_minor): + import ctypes + + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return False + + # Call gnu_get_libc_version, which returns a string like "2.5". + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + # Parse string and check against requested version. + version = [int(piece) for piece in version_str.split(".")] + assert len(version) == 2 + if major != version[0]: + return False + if minimum_minor > version[1]: + return False + return True + + +import sys +if is_manylinux1_compatible(): + print("%s is manylinux1 compatible" % (sys.executable, )) + sys.exit(0) +else: + print("%s is NOT manylinux1 compatible" % (sys.executable, )) + sys.exit(1) diff --git a/tools/dockerfile/build_scripts/python-tag-abi-tag.py b/tools/dockerfile/build_scripts/python-tag-abi-tag.py new file mode 100644 index 0000000000000000000000000000000000000000..0364ab3659e49dd59ff57764251408ae4359a43f --- /dev/null +++ b/tools/dockerfile/build_scripts/python-tag-abi-tag.py @@ -0,0 +1,21 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Utility script to print the python tag + the abi tag for a Python +# See PEP 425 for exactly what these are, but an example would be: +# cp27-cp27mu + +from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag + +print("{0}{1}-{2}".format(get_abbr_impl(), get_impl_ver(), get_abi_tag())) diff --git a/tools/dockerfile/build_scripts/ssl-check.py b/tools/dockerfile/build_scripts/ssl-check.py new file mode 100644 index 0000000000000000000000000000000000000000..afef2812f3fb4e9298ec8ab2d97e790ecc455d1c --- /dev/null +++ b/tools/dockerfile/build_scripts/ssl-check.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cf. https://github.com/pypa/manylinux/issues/53 + +GOOD_SSL = "https://google.com" +BAD_SSL = "https://self-signed.badssl.com" + +import sys + +print("Testing SSL certificate checking for Python:", sys.version) + +if (sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4)): + print("This version never checks SSL certs; skipping tests") + sys.exit(0) + +if sys.version_info[0] >= 3: + from urllib.request import urlopen + EXC = OSError +else: + from urllib import urlopen + EXC = IOError + +print("Connecting to %s should work" % (GOOD_SSL, )) +urlopen(GOOD_SSL) +print("...it did, yay.") + +print("Connecting to %s should fail" % (BAD_SSL, )) +try: + urlopen(BAD_SSL) + # If we get here then we failed: + print("...it DIDN'T!!!!!11!!1one!") + sys.exit(1) +except EXC: + print("...it did, yay.") diff --git a/tools/dockerfile/root/.bashrc b/tools/dockerfile/root/.bashrc new file mode 100755 index 0000000000000000000000000000000000000000..4b3024e4e81a0fa206a796c12a8b9d72f1a8f5d9 --- /dev/null +++ b/tools/dockerfile/root/.bashrc @@ -0,0 +1,46 @@ +# Locales + +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 +export LANGUAGE=en_US.UTF-8 + +# Aliases + +alias rm='rm -i' +alias cp='cp -i' +alias mv='mv -i' + +alias ls='ls -hFG' +alias l='ls -lF' +alias ll='ls -alF' +alias lt='ls -ltrF' +alias ll='ls -alF' +alias lls='ls -alSrF' +alias llt='ls -altrF' + +# Colorize directory listing + +alias ls="ls -ph --color=auto" + +# Colorize grep + +if echo hello|grep --color=auto l >/dev/null 2>&1; then + export GREP_OPTIONS="--color=auto" GREP_COLOR="1;31" +fi + +# Shell + +export CLICOLOR="1" + +YELLOW="\[\033[1;33m\]" +NO_COLOUR="\[\033[0m\]" +GREEN="\[\033[1;32m\]" +WHITE="\[\033[1;37m\]" + +source ~/.scripts/git-prompt.sh + +export PS1="\[\033[1;33m\]λ $WHITE\h $GREEN\w$YELLOW\$(__git_ps1 \" \[\033[35m\]{\[\033[36m\]%s\[\033[35m\]}\")$NO_COLOUR " + +# Git + +source ~/.scripts/git-completion.sh diff --git a/tools/dockerfile/root/.gitconfig b/tools/dockerfile/root/.gitconfig new file mode 100755 index 0000000000000000000000000000000000000000..6c249803a50403b9b79e36a13abe7fe88a35729d --- /dev/null +++ b/tools/dockerfile/root/.gitconfig @@ -0,0 +1,43 @@ +[user] + name = + email = + +[alias] + st = status --branch --short + ci = commit + br = branch + co = checkout + df = diff + l = log --pretty=format:\"%h %ad | %s%d [%an]\" --graph --date=short + ll = log --stat + +[merge] + tool = vimdiff + +[core] + excludesfile = ~/.gitignore + editor = vim + +[color] + branch = auto + diff = auto + status = auto + +[color "branch"] + current = yellow reverse + local = yellow + remote = green + +[color "diff"] + meta = yellow bold + frag = magenta bold + old = red bold + new = green bold + +[color "status"] + added = yellow + changed = green + untracked = cyan + +[push] + default = matching \ No newline at end of file diff --git a/tools/dockerfile/root/.scripts/git-completion.sh b/tools/dockerfile/root/.scripts/git-completion.sh new file mode 100755 index 0000000000000000000000000000000000000000..bdddef5ac2faf50b47dd03539dae8912bec8a16c --- /dev/null +++ b/tools/dockerfile/root/.scripts/git-completion.sh @@ -0,0 +1,2663 @@ +#!bash +# +# bash/zsh completion support for core Git. +# +# Copyright (C) 2006,2007 Shawn O. Pearce +# Conceptually based on gitcompletion (http://gitweb.hawaga.org.uk/). +# Distributed under the GNU General Public License, version 2.0. +# +# The contained completion routines provide support for completing: +# +# *) local and remote branch names +# *) local and remote tag names +# *) .git/remotes file names +# *) git 'subcommands' +# *) tree paths within 'ref:path/to/file' expressions +# *) file paths within current working directory and index +# *) common --long-options +# +# To use these routines: +# +# 1) Copy this file to somewhere (e.g. ~/.git-completion.sh). +# 2) Add the following line to your .bashrc/.zshrc: +# source ~/.git-completion.sh +# 3) Consider changing your PS1 to also show the current branch, +# see git-prompt.sh for details. + +case "$COMP_WORDBREAKS" in +*:*) : great ;; +*) COMP_WORDBREAKS="$COMP_WORDBREAKS:" +esac + +# __gitdir accepts 0 or 1 arguments (i.e., location) +# returns location of .git repo +__gitdir () +{ + if [ -z "${1-}" ]; then + if [ -n "${__git_dir-}" ]; then + echo "$__git_dir" + elif [ -n "${GIT_DIR-}" ]; then + test -d "${GIT_DIR-}" || return 1 + echo "$GIT_DIR" + elif [ -d .git ]; then + echo .git + else + git rev-parse --git-dir 2>/dev/null + fi + elif [ -d "$1/.git" ]; then + echo "$1/.git" + else + echo "$1" + fi +} + +# The following function is based on code from: +# +# bash_completion - programmable completion functions for bash 3.2+ +# +# Copyright © 2006-2008, Ian Macdonald +# © 2009-2010, Bash Completion Maintainers +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# The latest version of this software can be obtained here: +# +# http://bash-completion.alioth.debian.org/ +# +# RELEASE: 2.x + +# This function can be used to access a tokenized list of words +# on the command line: +# +# __git_reassemble_comp_words_by_ref '=:' +# if test "${words_[cword_-1]}" = -w +# then +# ... +# fi +# +# The argument should be a collection of characters from the list of +# word completion separators (COMP_WORDBREAKS) to treat as ordinary +# characters. +# +# This is roughly equivalent to going back in time and setting +# COMP_WORDBREAKS to exclude those characters. The intent is to +# make option types like --date= and : easy to +# recognize by treating each shell word as a single token. +# +# It is best not to set COMP_WORDBREAKS directly because the value is +# shared with other completion scripts. By the time the completion +# function gets called, COMP_WORDS has already been populated so local +# changes to COMP_WORDBREAKS have no effect. +# +# Output: words_, cword_, cur_. + +__git_reassemble_comp_words_by_ref() +{ + local exclude i j first + # Which word separators to exclude? + exclude="${1//[^$COMP_WORDBREAKS]}" + cword_=$COMP_CWORD + if [ -z "$exclude" ]; then + words_=("${COMP_WORDS[@]}") + return + fi + # List of word completion separators has shrunk; + # re-assemble words to complete. + for ((i=0, j=0; i < ${#COMP_WORDS[@]}; i++, j++)); do + # Append each nonempty word consisting of just + # word separator characters to the current word. + first=t + while + [ $i -gt 0 ] && + [ -n "${COMP_WORDS[$i]}" ] && + # word consists of excluded word separators + [ "${COMP_WORDS[$i]//[^$exclude]}" = "${COMP_WORDS[$i]}" ] + do + # Attach to the previous token, + # unless the previous token is the command name. + if [ $j -ge 2 ] && [ -n "$first" ]; then + ((j--)) + fi + first= + words_[$j]=${words_[j]}${COMP_WORDS[i]} + if [ $i = $COMP_CWORD ]; then + cword_=$j + fi + if (($i < ${#COMP_WORDS[@]} - 1)); then + ((i++)) + else + # Done. + return + fi + done + words_[$j]=${words_[j]}${COMP_WORDS[i]} + if [ $i = $COMP_CWORD ]; then + cword_=$j + fi + done +} + +if ! type _get_comp_words_by_ref >/dev/null 2>&1; then +_get_comp_words_by_ref () +{ + local exclude cur_ words_ cword_ + if [ "$1" = "-n" ]; then + exclude=$2 + shift 2 + fi + __git_reassemble_comp_words_by_ref "$exclude" + cur_=${words_[cword_]} + while [ $# -gt 0 ]; do + case "$1" in + cur) + cur=$cur_ + ;; + prev) + prev=${words_[$cword_-1]} + ;; + words) + words=("${words_[@]}") + ;; + cword) + cword=$cword_ + ;; + esac + shift + done +} +fi + +__gitcompadd () +{ + local i=0 + for x in $1; do + if [[ "$x" == "$3"* ]]; then + COMPREPLY[i++]="$2$x$4" + fi + done +} + +# Generates completion reply, appending a space to possible completion words, +# if necessary. +# It accepts 1 to 4 arguments: +# 1: List of possible completion words. +# 2: A prefix to be added to each possible completion word (optional). +# 3: Generate possible completion matches for this word (optional). +# 4: A suffix to be appended to each possible completion word (optional). +__gitcomp () +{ + local cur_="${3-$cur}" + + case "$cur_" in + --*=) + ;; + *) + local c i=0 IFS=$' \t\n' + for c in $1; do + c="$c${4-}" + if [[ $c == "$cur_"* ]]; then + case $c in + --*=*|*.) ;; + *) c="$c " ;; + esac + COMPREPLY[i++]="${2-}$c" + fi + done + ;; + esac +} + +# Generates completion reply from newline-separated possible completion words +# by appending a space to all of them. +# It accepts 1 to 4 arguments: +# 1: List of possible completion words, separated by a single newline. +# 2: A prefix to be added to each possible completion word (optional). +# 3: Generate possible completion matches for this word (optional). +# 4: A suffix to be appended to each possible completion word instead of +# the default space (optional). If specified but empty, nothing is +# appended. +__gitcomp_nl () +{ + local IFS=$'\n' + __gitcompadd "$1" "${2-}" "${3-$cur}" "${4- }" +} + +# Generates completion reply with compgen from newline-separated possible +# completion filenames. +# It accepts 1 to 3 arguments: +# 1: List of possible completion filenames, separated by a single newline. +# 2: A directory prefix to be added to each possible completion filename +# (optional). +# 3: Generate possible completion matches for this word (optional). +__gitcomp_file () +{ + local IFS=$'\n' + + # XXX does not work when the directory prefix contains a tilde, + # since tilde expansion is not applied. + # This means that COMPREPLY will be empty and Bash default + # completion will be used. + __gitcompadd "$1" "${2-}" "${3-$cur}" "" + + # use a hack to enable file mode in bash < 4 + compopt -o filenames +o nospace 2>/dev/null || + compgen -f /non-existing-dir/ > /dev/null +} + +# Execute 'git ls-files', unless the --committable option is specified, in +# which case it runs 'git diff-index' to find out the files that can be +# committed. It return paths relative to the directory specified in the first +# argument, and using the options specified in the second argument. +__git_ls_files_helper () +{ + ( + test -n "${CDPATH+set}" && unset CDPATH + cd "$1" + if [ "$2" == "--committable" ]; then + git diff-index --name-only --relative HEAD + else + # NOTE: $2 is not quoted in order to support multiple options + git ls-files --exclude-standard $2 + fi + ) 2>/dev/null +} + + +# __git_index_files accepts 1 or 2 arguments: +# 1: Options to pass to ls-files (required). +# 2: A directory path (optional). +# If provided, only files within the specified directory are listed. +# Sub directories are never recursed. Path must have a trailing +# slash. +__git_index_files () +{ + local dir="$(__gitdir)" root="${2-.}" file + + if [ -d "$dir" ]; then + __git_ls_files_helper "$root" "$1" | + while read -r file; do + case "$file" in + ?*/*) echo "${file%%/*}" ;; + *) echo "$file" ;; + esac + done | sort | uniq + fi +} + +__git_heads () +{ + local dir="$(__gitdir)" + if [ -d "$dir" ]; then + git --git-dir="$dir" for-each-ref --format='%(refname:short)' \ + refs/heads + return + fi +} + +__git_tags () +{ + local dir="$(__gitdir)" + if [ -d "$dir" ]; then + git --git-dir="$dir" for-each-ref --format='%(refname:short)' \ + refs/tags + return + fi +} + +# __git_refs accepts 0, 1 (to pass to __gitdir), or 2 arguments +# presence of 2nd argument means use the guess heuristic employed +# by checkout for tracking branches +__git_refs () +{ + local i hash dir="$(__gitdir "${1-}")" track="${2-}" + local format refs + if [ -d "$dir" ]; then + case "$cur" in + refs|refs/*) + format="refname" + refs="${cur%/*}" + track="" + ;; + *) + for i in HEAD FETCH_HEAD ORIG_HEAD MERGE_HEAD; do + if [ -e "$dir/$i" ]; then echo $i; fi + done + format="refname:short" + refs="refs/tags refs/heads refs/remotes" + ;; + esac + git --git-dir="$dir" for-each-ref --format="%($format)" \ + $refs + if [ -n "$track" ]; then + # employ the heuristic used by git checkout + # Try to find a remote branch that matches the completion word + # but only output if the branch name is unique + local ref entry + git --git-dir="$dir" for-each-ref --shell --format="ref=%(refname:short)" \ + "refs/remotes/" | \ + while read -r entry; do + eval "$entry" + ref="${ref#*/}" + if [[ "$ref" == "$cur"* ]]; then + echo "$ref" + fi + done | sort | uniq -u + fi + return + fi + case "$cur" in + refs|refs/*) + git ls-remote "$dir" "$cur*" 2>/dev/null | \ + while read -r hash i; do + case "$i" in + *^{}) ;; + *) echo "$i" ;; + esac + done + ;; + *) + echo "HEAD" + git for-each-ref --format="%(refname:short)" -- "refs/remotes/$dir/" | sed -e "s#^$dir/##" + ;; + esac +} + +# __git_refs2 requires 1 argument (to pass to __git_refs) +__git_refs2 () +{ + local i + for i in $(__git_refs "$1"); do + echo "$i:$i" + done +} + +# __git_refs_remotes requires 1 argument (to pass to ls-remote) +__git_refs_remotes () +{ + local i hash + git ls-remote "$1" 'refs/heads/*' 2>/dev/null | \ + while read -r hash i; do + echo "$i:refs/remotes/$1/${i#refs/heads/}" + done +} + +__git_remotes () +{ + local i IFS=$'\n' d="$(__gitdir)" + test -d "$d/remotes" && ls -1 "$d/remotes" + for i in $(git --git-dir="$d" config --get-regexp 'remote\..*\.url' 2>/dev/null); do + i="${i#remote.}" + echo "${i/.url*/}" + done +} + +__git_list_merge_strategies () +{ + git merge -s help 2>&1 | + sed -n -e '/[Aa]vailable strategies are: /,/^$/{ + s/\.$// + s/.*:// + s/^[ ]*// + s/[ ]*$// + p + }' +} + +__git_merge_strategies= +# 'git merge -s help' (and thus detection of the merge strategy +# list) fails, unfortunately, if run outside of any git working +# tree. __git_merge_strategies is set to the empty string in +# that case, and the detection will be repeated the next time it +# is needed. +__git_compute_merge_strategies () +{ + test -n "$__git_merge_strategies" || + __git_merge_strategies=$(__git_list_merge_strategies) +} + +__git_complete_revlist_file () +{ + local pfx ls ref cur_="$cur" + case "$cur_" in + *..?*:*) + return + ;; + ?*:*) + ref="${cur_%%:*}" + cur_="${cur_#*:}" + case "$cur_" in + ?*/*) + pfx="${cur_%/*}" + cur_="${cur_##*/}" + ls="$ref:$pfx" + pfx="$pfx/" + ;; + *) + ls="$ref" + ;; + esac + + case "$COMP_WORDBREAKS" in + *:*) : great ;; + *) pfx="$ref:$pfx" ;; + esac + + __gitcomp_nl "$(git --git-dir="$(__gitdir)" ls-tree "$ls" 2>/dev/null \ + | sed '/^100... blob /{ + s,^.* ,, + s,$, , + } + /^120000 blob /{ + s,^.* ,, + s,$, , + } + /^040000 tree /{ + s,^.* ,, + s,$,/, + } + s/^.* //')" \ + "$pfx" "$cur_" "" + ;; + *...*) + pfx="${cur_%...*}..." + cur_="${cur_#*...}" + __gitcomp_nl "$(__git_refs)" "$pfx" "$cur_" + ;; + *..*) + pfx="${cur_%..*}.." + cur_="${cur_#*..}" + __gitcomp_nl "$(__git_refs)" "$pfx" "$cur_" + ;; + *) + __gitcomp_nl "$(__git_refs)" + ;; + esac +} + + +# __git_complete_index_file requires 1 argument: +# 1: the options to pass to ls-file +# +# The exception is --committable, which finds the files appropriate commit. +__git_complete_index_file () +{ + local pfx="" cur_="$cur" + + case "$cur_" in + ?*/*) + pfx="${cur_%/*}" + cur_="${cur_##*/}" + pfx="${pfx}/" + ;; + esac + + __gitcomp_file "$(__git_index_files "$1" "$pfx")" "$pfx" "$cur_" +} + +__git_complete_file () +{ + __git_complete_revlist_file +} + +__git_complete_revlist () +{ + __git_complete_revlist_file +} + +__git_complete_remote_or_refspec () +{ + local cur_="$cur" cmd="${words[1]}" + local i c=2 remote="" pfx="" lhs=1 no_complete_refspec=0 + if [ "$cmd" = "remote" ]; then + ((c++)) + fi + while [ $c -lt $cword ]; do + i="${words[c]}" + case "$i" in + --mirror) [ "$cmd" = "push" ] && no_complete_refspec=1 ;; + --all) + case "$cmd" in + push) no_complete_refspec=1 ;; + fetch) + return + ;; + *) ;; + esac + ;; + -*) ;; + *) remote="$i"; break ;; + esac + ((c++)) + done + if [ -z "$remote" ]; then + __gitcomp_nl "$(__git_remotes)" + return + fi + if [ $no_complete_refspec = 1 ]; then + return + fi + [ "$remote" = "." ] && remote= + case "$cur_" in + *:*) + case "$COMP_WORDBREAKS" in + *:*) : great ;; + *) pfx="${cur_%%:*}:" ;; + esac + cur_="${cur_#*:}" + lhs=0 + ;; + +*) + pfx="+" + cur_="${cur_#+}" + ;; + esac + case "$cmd" in + fetch) + if [ $lhs = 1 ]; then + __gitcomp_nl "$(__git_refs2 "$remote")" "$pfx" "$cur_" + else + __gitcomp_nl "$(__git_refs)" "$pfx" "$cur_" + fi + ;; + pull|remote) + if [ $lhs = 1 ]; then + __gitcomp_nl "$(__git_refs "$remote")" "$pfx" "$cur_" + else + __gitcomp_nl "$(__git_refs)" "$pfx" "$cur_" + fi + ;; + push) + if [ $lhs = 1 ]; then + __gitcomp_nl "$(__git_refs)" "$pfx" "$cur_" + else + __gitcomp_nl "$(__git_refs "$remote")" "$pfx" "$cur_" + fi + ;; + esac +} + +__git_complete_strategy () +{ + __git_compute_merge_strategies + case "$prev" in + -s|--strategy) + __gitcomp "$__git_merge_strategies" + return 0 + esac + case "$cur" in + --strategy=*) + __gitcomp "$__git_merge_strategies" "" "${cur##--strategy=}" + return 0 + ;; + esac + return 1 +} + +__git_commands () { + if test -n "${GIT_TESTING_COMMAND_COMPLETION:-}" + then + printf "%s" "${GIT_TESTING_COMMAND_COMPLETION}" + else + git help -a|egrep '^ [a-zA-Z0-9]' + fi +} + +__git_list_all_commands () +{ + local i IFS=" "$'\n' + for i in $(__git_commands) + do + case $i in + *--*) : helper pattern;; + *) echo $i;; + esac + done +} + +__git_all_commands= +__git_compute_all_commands () +{ + test -n "$__git_all_commands" || + __git_all_commands=$(__git_list_all_commands) +} + +__git_list_porcelain_commands () +{ + local i IFS=" "$'\n' + __git_compute_all_commands + for i in $__git_all_commands + do + case $i in + *--*) : helper pattern;; + applymbox) : ask gittus;; + applypatch) : ask gittus;; + archimport) : import;; + cat-file) : plumbing;; + check-attr) : plumbing;; + check-ignore) : plumbing;; + check-mailmap) : plumbing;; + check-ref-format) : plumbing;; + checkout-index) : plumbing;; + commit-tree) : plumbing;; + count-objects) : infrequent;; + credential-cache) : credentials helper;; + credential-store) : credentials helper;; + cvsexportcommit) : export;; + cvsimport) : import;; + cvsserver) : daemon;; + daemon) : daemon;; + diff-files) : plumbing;; + diff-index) : plumbing;; + diff-tree) : plumbing;; + fast-import) : import;; + fast-export) : export;; + fsck-objects) : plumbing;; + fetch-pack) : plumbing;; + fmt-merge-msg) : plumbing;; + for-each-ref) : plumbing;; + hash-object) : plumbing;; + http-*) : transport;; + index-pack) : plumbing;; + init-db) : deprecated;; + local-fetch) : plumbing;; + lost-found) : infrequent;; + ls-files) : plumbing;; + ls-remote) : plumbing;; + ls-tree) : plumbing;; + mailinfo) : plumbing;; + mailsplit) : plumbing;; + merge-*) : plumbing;; + mktree) : plumbing;; + mktag) : plumbing;; + pack-objects) : plumbing;; + pack-redundant) : plumbing;; + pack-refs) : plumbing;; + parse-remote) : plumbing;; + patch-id) : plumbing;; + peek-remote) : plumbing;; + prune) : plumbing;; + prune-packed) : plumbing;; + quiltimport) : import;; + read-tree) : plumbing;; + receive-pack) : plumbing;; + remote-*) : transport;; + repo-config) : deprecated;; + rerere) : plumbing;; + rev-list) : plumbing;; + rev-parse) : plumbing;; + runstatus) : plumbing;; + sh-setup) : internal;; + shell) : daemon;; + show-ref) : plumbing;; + send-pack) : plumbing;; + show-index) : plumbing;; + ssh-*) : transport;; + stripspace) : plumbing;; + symbolic-ref) : plumbing;; + tar-tree) : deprecated;; + unpack-file) : plumbing;; + unpack-objects) : plumbing;; + update-index) : plumbing;; + update-ref) : plumbing;; + update-server-info) : daemon;; + upload-archive) : plumbing;; + upload-pack) : plumbing;; + write-tree) : plumbing;; + var) : infrequent;; + verify-pack) : infrequent;; + verify-tag) : plumbing;; + *) echo $i;; + esac + done +} + +__git_porcelain_commands= +__git_compute_porcelain_commands () +{ + __git_compute_all_commands + test -n "$__git_porcelain_commands" || + __git_porcelain_commands=$(__git_list_porcelain_commands) +} + +__git_pretty_aliases () +{ + local i IFS=$'\n' + for i in $(git --git-dir="$(__gitdir)" config --get-regexp "pretty\..*" 2>/dev/null); do + case "$i" in + pretty.*) + i="${i#pretty.}" + echo "${i/ */}" + ;; + esac + done +} + +__git_aliases () +{ + local i IFS=$'\n' + for i in $(git --git-dir="$(__gitdir)" config --get-regexp "alias\..*" 2>/dev/null); do + case "$i" in + alias.*) + i="${i#alias.}" + echo "${i/ */}" + ;; + esac + done +} + +# __git_aliased_command requires 1 argument +__git_aliased_command () +{ + local word cmdline=$(git --git-dir="$(__gitdir)" \ + config --get "alias.$1") + for word in $cmdline; do + case "$word" in + \!gitk|gitk) + echo "gitk" + return + ;; + \!*) : shell command alias ;; + -*) : option ;; + *=*) : setting env ;; + git) : git itself ;; + *) + echo "$word" + return + esac + done +} + +# __git_find_on_cmdline requires 1 argument +__git_find_on_cmdline () +{ + local word subcommand c=1 + while [ $c -lt $cword ]; do + word="${words[c]}" + for subcommand in $1; do + if [ "$subcommand" = "$word" ]; then + echo "$subcommand" + return + fi + done + ((c++)) + done +} + +__git_has_doubledash () +{ + local c=1 + while [ $c -lt $cword ]; do + if [ "--" = "${words[c]}" ]; then + return 0 + fi + ((c++)) + done + return 1 +} + +# Try to count non option arguments passed on the command line for the +# specified git command. +# When options are used, it is necessary to use the special -- option to +# tell the implementation were non option arguments begin. +# XXX this can not be improved, since options can appear everywhere, as +# an example: +# git mv x -n y +# +# __git_count_arguments requires 1 argument: the git command executed. +__git_count_arguments () +{ + local word i c=0 + + # Skip "git" (first argument) + for ((i=1; i < ${#words[@]}; i++)); do + word="${words[i]}" + + case "$word" in + --) + # Good; we can assume that the following are only non + # option arguments. + ((c = 0)) + ;; + "$1") + # Skip the specified git command and discard git + # main options + ((c = 0)) + ;; + ?*) + ((c++)) + ;; + esac + done + + printf "%d" $c +} + +__git_whitespacelist="nowarn warn error error-all fix" + +_git_am () +{ + local dir="$(__gitdir)" + if [ -d "$dir"/rebase-apply ]; then + __gitcomp "--skip --continue --resolved --abort" + return + fi + case "$cur" in + --whitespace=*) + __gitcomp "$__git_whitespacelist" "" "${cur##--whitespace=}" + return + ;; + --*) + __gitcomp " + --3way --committer-date-is-author-date --ignore-date + --ignore-whitespace --ignore-space-change + --interactive --keep --no-utf8 --signoff --utf8 + --whitespace= --scissors + " + return + esac +} + +_git_apply () +{ + case "$cur" in + --whitespace=*) + __gitcomp "$__git_whitespacelist" "" "${cur##--whitespace=}" + return + ;; + --*) + __gitcomp " + --stat --numstat --summary --check --index + --cached --index-info --reverse --reject --unidiff-zero + --apply --no-add --exclude= + --ignore-whitespace --ignore-space-change + --whitespace= --inaccurate-eof --verbose + " + return + esac +} + +_git_add () +{ + case "$cur" in + --*) + __gitcomp " + --interactive --refresh --patch --update --dry-run + --ignore-errors --intent-to-add + " + return + esac + + # XXX should we check for --update and --all options ? + __git_complete_index_file "--others --modified" +} + +_git_archive () +{ + case "$cur" in + --format=*) + __gitcomp "$(git archive --list)" "" "${cur##--format=}" + return + ;; + --remote=*) + __gitcomp_nl "$(__git_remotes)" "" "${cur##--remote=}" + return + ;; + --*) + __gitcomp " + --format= --list --verbose + --prefix= --remote= --exec= + " + return + ;; + esac + __git_complete_file +} + +_git_bisect () +{ + __git_has_doubledash && return + + local subcommands="start bad good skip reset visualize replay log run" + local subcommand="$(__git_find_on_cmdline "$subcommands")" + if [ -z "$subcommand" ]; then + if [ -f "$(__gitdir)"/BISECT_START ]; then + __gitcomp "$subcommands" + else + __gitcomp "replay start" + fi + return + fi + + case "$subcommand" in + bad|good|reset|skip|start) + __gitcomp_nl "$(__git_refs)" + ;; + *) + ;; + esac +} + +_git_branch () +{ + local i c=1 only_local_ref="n" has_r="n" + + while [ $c -lt $cword ]; do + i="${words[c]}" + case "$i" in + -d|-m) only_local_ref="y" ;; + -r) has_r="y" ;; + esac + ((c++)) + done + + case "$cur" in + --set-upstream-to=*) + __gitcomp "$(__git_refs)" "" "${cur##--set-upstream-to=}" + ;; + --*) + __gitcomp " + --color --no-color --verbose --abbrev= --no-abbrev + --track --no-track --contains --merged --no-merged + --set-upstream-to= --edit-description --list + --unset-upstream + " + ;; + *) + if [ $only_local_ref = "y" -a $has_r = "n" ]; then + __gitcomp_nl "$(__git_heads)" + else + __gitcomp_nl "$(__git_refs)" + fi + ;; + esac +} + +_git_bundle () +{ + local cmd="${words[2]}" + case "$cword" in + 2) + __gitcomp "create list-heads verify unbundle" + ;; + 3) + # looking for a file + ;; + *) + case "$cmd" in + create) + __git_complete_revlist + ;; + esac + ;; + esac +} + +_git_checkout () +{ + __git_has_doubledash && return + + case "$cur" in + --conflict=*) + __gitcomp "diff3 merge" "" "${cur##--conflict=}" + ;; + --*) + __gitcomp " + --quiet --ours --theirs --track --no-track --merge + --conflict= --orphan --patch + " + ;; + *) + # check if --track, --no-track, or --no-guess was specified + # if so, disable DWIM mode + local flags="--track --no-track --no-guess" track=1 + if [ -n "$(__git_find_on_cmdline "$flags")" ]; then + track='' + fi + __gitcomp_nl "$(__git_refs '' $track)" + ;; + esac +} + +_git_cherry () +{ + __gitcomp "$(__git_refs)" +} + +_git_cherry_pick () +{ + local dir="$(__gitdir)" + if [ -f "$dir"/CHERRY_PICK_HEAD ]; then + __gitcomp "--continue --quit --abort" + return + fi + case "$cur" in + --*) + __gitcomp "--edit --no-commit --signoff --strategy= --mainline" + ;; + *) + __gitcomp_nl "$(__git_refs)" + ;; + esac +} + +_git_clean () +{ + case "$cur" in + --*) + __gitcomp "--dry-run --quiet" + return + ;; + esac + + # XXX should we check for -x option ? + __git_complete_index_file "--others" +} + +_git_clone () +{ + case "$cur" in + --*) + __gitcomp " + --local + --no-hardlinks + --shared + --reference + --quiet + --no-checkout + --bare + --mirror + --origin + --upload-pack + --template= + --depth + --single-branch + --branch + " + return + ;; + esac +} + +_git_commit () +{ + case "$prev" in + -c|-C) + __gitcomp_nl "$(__git_refs)" "" "${cur}" + return + ;; + esac + + case "$cur" in + --cleanup=*) + __gitcomp "default strip verbatim whitespace + " "" "${cur##--cleanup=}" + return + ;; + --reuse-message=*|--reedit-message=*|\ + --fixup=*|--squash=*) + __gitcomp_nl "$(__git_refs)" "" "${cur#*=}" + return + ;; + --untracked-files=*) + __gitcomp "all no normal" "" "${cur##--untracked-files=}" + return + ;; + --*) + __gitcomp " + --all --author= --signoff --verify --no-verify + --edit --no-edit + --amend --include --only --interactive + --dry-run --reuse-message= --reedit-message= + --reset-author --file= --message= --template= + --cleanup= --untracked-files --untracked-files= + --verbose --quiet --fixup= --squash= + " + return + esac + + if git rev-parse --verify --quiet HEAD >/dev/null; then + __git_complete_index_file "--committable" + else + # This is the first commit + __git_complete_index_file "--cached" + fi +} + +_git_describe () +{ + case "$cur" in + --*) + __gitcomp " + --all --tags --contains --abbrev= --candidates= + --exact-match --debug --long --match --always + " + return + esac + __gitcomp_nl "$(__git_refs)" +} + +__git_diff_algorithms="myers minimal patience histogram" + +__git_diff_common_options="--stat --numstat --shortstat --summary + --patch-with-stat --name-only --name-status --color + --no-color --color-words --no-renames --check + --full-index --binary --abbrev --diff-filter= + --find-copies-harder + --text --ignore-space-at-eol --ignore-space-change + --ignore-all-space --exit-code --quiet --ext-diff + --no-ext-diff + --no-prefix --src-prefix= --dst-prefix= + --inter-hunk-context= + --patience --histogram --minimal + --raw --word-diff + --dirstat --dirstat= --dirstat-by-file + --dirstat-by-file= --cumulative + --diff-algorithm= +" + +_git_diff () +{ + __git_has_doubledash && return + + case "$cur" in + --diff-algorithm=*) + __gitcomp "$__git_diff_algorithms" "" "${cur##--diff-algorithm=}" + return + ;; + --*) + __gitcomp "--cached --staged --pickaxe-all --pickaxe-regex + --base --ours --theirs --no-index + $__git_diff_common_options + " + return + ;; + esac + __git_complete_revlist_file +} + +__git_mergetools_common="diffuse ecmerge emerge kdiff3 meld opendiff + tkdiff vimdiff gvimdiff xxdiff araxis p4merge bc3 codecompare +" + +_git_difftool () +{ + __git_has_doubledash && return + + case "$cur" in + --tool=*) + __gitcomp "$__git_mergetools_common kompare" "" "${cur##--tool=}" + return + ;; + --*) + __gitcomp "--cached --staged --pickaxe-all --pickaxe-regex + --base --ours --theirs + --no-renames --diff-filter= --find-copies-harder + --relative --ignore-submodules + --tool=" + return + ;; + esac + __git_complete_revlist_file +} + +__git_fetch_options=" + --quiet --verbose --append --upload-pack --force --keep --depth= + --tags --no-tags --all --prune --dry-run +" + +_git_fetch () +{ + case "$cur" in + --*) + __gitcomp "$__git_fetch_options" + return + ;; + esac + __git_complete_remote_or_refspec +} + +__git_format_patch_options=" + --stdout --attach --no-attach --thread --thread= --no-thread + --numbered --start-number --numbered-files --keep-subject --signoff + --signature --no-signature --in-reply-to= --cc= --full-index --binary + --not --all --cover-letter --no-prefix --src-prefix= --dst-prefix= + --inline --suffix= --ignore-if-in-upstream --subject-prefix= + --output-directory --reroll-count --to= --quiet --notes +" + +_git_format_patch () +{ + case "$cur" in + --thread=*) + __gitcomp " + deep shallow + " "" "${cur##--thread=}" + return + ;; + --*) + __gitcomp "$__git_format_patch_options" + return + ;; + esac + __git_complete_revlist +} + +_git_fsck () +{ + case "$cur" in + --*) + __gitcomp " + --tags --root --unreachable --cache --no-reflogs --full + --strict --verbose --lost-found + " + return + ;; + esac +} + +_git_gc () +{ + case "$cur" in + --*) + __gitcomp "--prune --aggressive" + return + ;; + esac +} + +_git_gitk () +{ + _gitk +} + +__git_match_ctag() { + awk "/^${1////\\/}/ { print \$1 }" "$2" +} + +_git_grep () +{ + __git_has_doubledash && return + + case "$cur" in + --*) + __gitcomp " + --cached + --text --ignore-case --word-regexp --invert-match + --full-name --line-number + --extended-regexp --basic-regexp --fixed-strings + --perl-regexp + --files-with-matches --name-only + --files-without-match + --max-depth + --count + --and --or --not --all-match + " + return + ;; + esac + + case "$cword,$prev" in + 2,*|*,-*) + if test -r tags; then + __gitcomp_nl "$(__git_match_ctag "$cur" tags)" + return + fi + ;; + esac + + __gitcomp_nl "$(__git_refs)" +} + +_git_help () +{ + case "$cur" in + --*) + __gitcomp "--all --info --man --web" + return + ;; + esac + __git_compute_all_commands + __gitcomp "$__git_all_commands $(__git_aliases) + attributes cli core-tutorial cvs-migration + diffcore gitk glossary hooks ignore modules + namespaces repository-layout tutorial tutorial-2 + workflows + " +} + +_git_init () +{ + case "$cur" in + --shared=*) + __gitcomp " + false true umask group all world everybody + " "" "${cur##--shared=}" + return + ;; + --*) + __gitcomp "--quiet --bare --template= --shared --shared=" + return + ;; + esac +} + +_git_ls_files () +{ + case "$cur" in + --*) + __gitcomp "--cached --deleted --modified --others --ignored + --stage --directory --no-empty-directory --unmerged + --killed --exclude= --exclude-from= + --exclude-per-directory= --exclude-standard + --error-unmatch --with-tree= --full-name + --abbrev --ignored --exclude-per-directory + " + return + ;; + esac + + # XXX ignore options like --modified and always suggest all cached + # files. + __git_complete_index_file "--cached" +} + +_git_ls_remote () +{ + __gitcomp_nl "$(__git_remotes)" +} + +_git_ls_tree () +{ + __git_complete_file +} + +# Options that go well for log, shortlog and gitk +__git_log_common_options=" + --not --all + --branches --tags --remotes + --first-parent --merges --no-merges + --max-count= + --max-age= --since= --after= + --min-age= --until= --before= + --min-parents= --max-parents= + --no-min-parents --no-max-parents +" +# Options that go well for log and gitk (not shortlog) +__git_log_gitk_options=" + --dense --sparse --full-history + --simplify-merges --simplify-by-decoration + --left-right --notes --no-notes +" +# Options that go well for log and shortlog (not gitk) +__git_log_shortlog_options=" + --author= --committer= --grep= + --all-match +" + +__git_log_pretty_formats="oneline short medium full fuller email raw format:" +__git_log_date_formats="relative iso8601 rfc2822 short local default raw" + +_git_log () +{ + __git_has_doubledash && return + + local g="$(git rev-parse --git-dir 2>/dev/null)" + local merge="" + if [ -f "$g/MERGE_HEAD" ]; then + merge="--merge" + fi + case "$cur" in + --pretty=*|--format=*) + __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) + " "" "${cur#*=}" + return + ;; + --date=*) + __gitcomp "$__git_log_date_formats" "" "${cur##--date=}" + return + ;; + --decorate=*) + __gitcomp "long short" "" "${cur##--decorate=}" + return + ;; + --*) + __gitcomp " + $__git_log_common_options + $__git_log_shortlog_options + $__git_log_gitk_options + --root --topo-order --date-order --reverse + --follow --full-diff + --abbrev-commit --abbrev= + --relative-date --date= + --pretty= --format= --oneline + --cherry-pick + --graph + --decorate --decorate= + --walk-reflogs + --parents --children + $merge + $__git_diff_common_options + --pickaxe-all --pickaxe-regex + " + return + ;; + esac + __git_complete_revlist +} + +__git_merge_options=" + --no-commit --no-stat --log --no-log --squash --strategy + --commit --stat --no-squash --ff --no-ff --ff-only --edit --no-edit +" + +_git_merge () +{ + __git_complete_strategy && return + + case "$cur" in + --*) + __gitcomp "$__git_merge_options" + return + esac + __gitcomp_nl "$(__git_refs)" +} + +_git_mergetool () +{ + case "$cur" in + --tool=*) + __gitcomp "$__git_mergetools_common tortoisemerge" "" "${cur##--tool=}" + return + ;; + --*) + __gitcomp "--tool=" + return + ;; + esac +} + +_git_merge_base () +{ + __gitcomp_nl "$(__git_refs)" +} + +_git_mv () +{ + case "$cur" in + --*) + __gitcomp "--dry-run" + return + ;; + esac + + if [ $(__git_count_arguments "mv") -gt 0 ]; then + # We need to show both cached and untracked files (including + # empty directories) since this may not be the last argument. + __git_complete_index_file "--cached --others --directory" + else + __git_complete_index_file "--cached" + fi +} + +_git_name_rev () +{ + __gitcomp "--tags --all --stdin" +} + +_git_notes () +{ + local subcommands='add append copy edit list prune remove show' + local subcommand="$(__git_find_on_cmdline "$subcommands")" + + case "$subcommand,$cur" in + ,--*) + __gitcomp '--ref' + ;; + ,*) + case "$prev" in + --ref) + __gitcomp_nl "$(__git_refs)" + ;; + *) + __gitcomp "$subcommands --ref" + ;; + esac + ;; + add,--reuse-message=*|append,--reuse-message=*|\ + add,--reedit-message=*|append,--reedit-message=*) + __gitcomp_nl "$(__git_refs)" "" "${cur#*=}" + ;; + add,--*|append,--*) + __gitcomp '--file= --message= --reedit-message= + --reuse-message=' + ;; + copy,--*) + __gitcomp '--stdin' + ;; + prune,--*) + __gitcomp '--dry-run --verbose' + ;; + prune,*) + ;; + *) + case "$prev" in + -m|-F) + ;; + *) + __gitcomp_nl "$(__git_refs)" + ;; + esac + ;; + esac +} + +_git_pull () +{ + __git_complete_strategy && return + + case "$cur" in + --*) + __gitcomp " + --rebase --no-rebase + $__git_merge_options + $__git_fetch_options + " + return + ;; + esac + __git_complete_remote_or_refspec +} + +_git_push () +{ + case "$prev" in + --repo) + __gitcomp_nl "$(__git_remotes)" + return + esac + case "$cur" in + --repo=*) + __gitcomp_nl "$(__git_remotes)" "" "${cur##--repo=}" + return + ;; + --*) + __gitcomp " + --all --mirror --tags --dry-run --force --verbose + --receive-pack= --repo= --set-upstream + " + return + ;; + esac + __git_complete_remote_or_refspec +} + +_git_rebase () +{ + local dir="$(__gitdir)" + if [ -d "$dir"/rebase-apply ] || [ -d "$dir"/rebase-merge ]; then + __gitcomp "--continue --skip --abort" + return + fi + __git_complete_strategy && return + case "$cur" in + --whitespace=*) + __gitcomp "$__git_whitespacelist" "" "${cur##--whitespace=}" + return + ;; + --*) + __gitcomp " + --onto --merge --strategy --interactive + --preserve-merges --stat --no-stat + --committer-date-is-author-date --ignore-date + --ignore-whitespace --whitespace= + --autosquash + " + + return + esac + __gitcomp_nl "$(__git_refs)" +} + +_git_reflog () +{ + local subcommands="show delete expire" + local subcommand="$(__git_find_on_cmdline "$subcommands")" + + if [ -z "$subcommand" ]; then + __gitcomp "$subcommands" + else + __gitcomp_nl "$(__git_refs)" + fi +} + +__git_send_email_confirm_options="always never auto cc compose" +__git_send_email_suppresscc_options="author self cc bodycc sob cccmd body all" + +_git_send_email () +{ + case "$cur" in + --confirm=*) + __gitcomp " + $__git_send_email_confirm_options + " "" "${cur##--confirm=}" + return + ;; + --suppress-cc=*) + __gitcomp " + $__git_send_email_suppresscc_options + " "" "${cur##--suppress-cc=}" + + return + ;; + --smtp-encryption=*) + __gitcomp "ssl tls" "" "${cur##--smtp-encryption=}" + return + ;; + --thread=*) + __gitcomp " + deep shallow + " "" "${cur##--thread=}" + return + ;; + --*) + __gitcomp "--annotate --bcc --cc --cc-cmd --chain-reply-to + --compose --confirm= --dry-run --envelope-sender + --from --identity + --in-reply-to --no-chain-reply-to --no-signed-off-by-cc + --no-suppress-from --no-thread --quiet + --signed-off-by-cc --smtp-pass --smtp-server + --smtp-server-port --smtp-encryption= --smtp-user + --subject --suppress-cc= --suppress-from --thread --to + --validate --no-validate + $__git_format_patch_options" + return + ;; + esac + __git_complete_revlist +} + +_git_stage () +{ + _git_add +} + +__git_config_get_set_variables () +{ + local prevword word config_file= c=$cword + while [ $c -gt 1 ]; do + word="${words[c]}" + case "$word" in + --system|--global|--local|--file=*) + config_file="$word" + break + ;; + -f|--file) + config_file="$word $prevword" + break + ;; + esac + prevword=$word + c=$((--c)) + done + + git --git-dir="$(__gitdir)" config $config_file --list 2>/dev/null | + while read -r line + do + case "$line" in + *.*=*) + echo "${line/=*/}" + ;; + esac + done +} + +_git_config () +{ + case "$prev" in + branch.*.remote|branch.*.pushremote) + __gitcomp_nl "$(__git_remotes)" + return + ;; + branch.*.merge) + __gitcomp_nl "$(__git_refs)" + return + ;; + branch.*.rebase) + __gitcomp "false true" + return + ;; + remote.pushdefault) + __gitcomp_nl "$(__git_remotes)" + return + ;; + remote.*.fetch) + local remote="${prev#remote.}" + remote="${remote%.fetch}" + if [ -z "$cur" ]; then + __gitcomp_nl "refs/heads/" "" "" "" + return + fi + __gitcomp_nl "$(__git_refs_remotes "$remote")" + return + ;; + remote.*.push) + local remote="${prev#remote.}" + remote="${remote%.push}" + __gitcomp_nl "$(git --git-dir="$(__gitdir)" \ + for-each-ref --format='%(refname):%(refname)' \ + refs/heads)" + return + ;; + pull.twohead|pull.octopus) + __git_compute_merge_strategies + __gitcomp "$__git_merge_strategies" + return + ;; + color.branch|color.diff|color.interactive|\ + color.showbranch|color.status|color.ui) + __gitcomp "always never auto" + return + ;; + color.pager) + __gitcomp "false true" + return + ;; + color.*.*) + __gitcomp " + normal black red green yellow blue magenta cyan white + bold dim ul blink reverse + " + return + ;; + diff.submodule) + __gitcomp "log short" + return + ;; + help.format) + __gitcomp "man info web html" + return + ;; + log.date) + __gitcomp "$__git_log_date_formats" + return + ;; + sendemail.aliasesfiletype) + __gitcomp "mutt mailrc pine elm gnus" + return + ;; + sendemail.confirm) + __gitcomp "$__git_send_email_confirm_options" + return + ;; + sendemail.suppresscc) + __gitcomp "$__git_send_email_suppresscc_options" + return + ;; + --get|--get-all|--unset|--unset-all) + __gitcomp_nl "$(__git_config_get_set_variables)" + return + ;; + *.*) + return + ;; + esac + case "$cur" in + --*) + __gitcomp " + --system --global --local --file= + --list --replace-all + --get --get-all --get-regexp + --add --unset --unset-all + --remove-section --rename-section + " + return + ;; + branch.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp "remote pushremote merge mergeoptions rebase" "$pfx" "$cur_" + return + ;; + branch.*) + local pfx="${cur%.*}." cur_="${cur#*.}" + __gitcomp_nl "$(__git_heads)" "$pfx" "$cur_" "." + return + ;; + guitool.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp " + argprompt cmd confirm needsfile noconsole norescan + prompt revprompt revunmerged title + " "$pfx" "$cur_" + return + ;; + difftool.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp "cmd path" "$pfx" "$cur_" + return + ;; + man.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp "cmd path" "$pfx" "$cur_" + return + ;; + mergetool.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp "cmd path trustExitCode" "$pfx" "$cur_" + return + ;; + pager.*) + local pfx="${cur%.*}." cur_="${cur#*.}" + __git_compute_all_commands + __gitcomp_nl "$__git_all_commands" "$pfx" "$cur_" + return + ;; + remote.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp " + url proxy fetch push mirror skipDefaultUpdate + receivepack uploadpack tagopt pushurl + " "$pfx" "$cur_" + return + ;; + remote.*) + local pfx="${cur%.*}." cur_="${cur#*.}" + __gitcomp_nl "$(__git_remotes)" "$pfx" "$cur_" "." + return + ;; + url.*.*) + local pfx="${cur%.*}." cur_="${cur##*.}" + __gitcomp "insteadOf pushInsteadOf" "$pfx" "$cur_" + return + ;; + esac + __gitcomp " + add.ignoreErrors + advice.commitBeforeMerge + advice.detachedHead + advice.implicitIdentity + advice.pushNonFastForward + advice.resolveConflict + advice.statusHints + alias. + am.keepcr + apply.ignorewhitespace + apply.whitespace + branch.autosetupmerge + branch.autosetuprebase + browser. + clean.requireForce + color.branch + color.branch.current + color.branch.local + color.branch.plain + color.branch.remote + color.decorate.HEAD + color.decorate.branch + color.decorate.remoteBranch + color.decorate.stash + color.decorate.tag + color.diff + color.diff.commit + color.diff.frag + color.diff.func + color.diff.meta + color.diff.new + color.diff.old + color.diff.plain + color.diff.whitespace + color.grep + color.grep.context + color.grep.filename + color.grep.function + color.grep.linenumber + color.grep.match + color.grep.selected + color.grep.separator + color.interactive + color.interactive.error + color.interactive.header + color.interactive.help + color.interactive.prompt + color.pager + color.showbranch + color.status + color.status.added + color.status.changed + color.status.header + color.status.nobranch + color.status.untracked + color.status.updated + color.ui + commit.status + commit.template + core.abbrev + core.askpass + core.attributesfile + core.autocrlf + core.bare + core.bigFileThreshold + core.compression + core.createObject + core.deltaBaseCacheLimit + core.editor + core.eol + core.excludesfile + core.fileMode + core.fsyncobjectfiles + core.gitProxy + core.ignoreStat + core.ignorecase + core.logAllRefUpdates + core.loosecompression + core.notesRef + core.packedGitLimit + core.packedGitWindowSize + core.pager + core.preferSymlinkRefs + core.preloadindex + core.quotepath + core.repositoryFormatVersion + core.safecrlf + core.sharedRepository + core.sparseCheckout + core.symlinks + core.trustctime + core.warnAmbiguousRefs + core.whitespace + core.worktree + diff.autorefreshindex + diff.external + diff.ignoreSubmodules + diff.mnemonicprefix + diff.noprefix + diff.renameLimit + diff.renames + diff.statGraphWidth + diff.submodule + diff.suppressBlankEmpty + diff.tool + diff.wordRegex + diff.algorithm + difftool. + difftool.prompt + fetch.recurseSubmodules + fetch.unpackLimit + format.attach + format.cc + format.headers + format.numbered + format.pretty + format.signature + format.signoff + format.subjectprefix + format.suffix + format.thread + format.to + gc. + gc.aggressiveWindow + gc.auto + gc.autopacklimit + gc.packrefs + gc.pruneexpire + gc.reflogexpire + gc.reflogexpireunreachable + gc.rerereresolved + gc.rerereunresolved + gitcvs.allbinary + gitcvs.commitmsgannotation + gitcvs.dbTableNamePrefix + gitcvs.dbdriver + gitcvs.dbname + gitcvs.dbpass + gitcvs.dbuser + gitcvs.enabled + gitcvs.logfile + gitcvs.usecrlfattr + guitool. + gui.blamehistoryctx + gui.commitmsgwidth + gui.copyblamethreshold + gui.diffcontext + gui.encoding + gui.fastcopyblame + gui.matchtrackingbranch + gui.newbranchtemplate + gui.pruneduringfetch + gui.spellingdictionary + gui.trustmtime + help.autocorrect + help.browser + help.format + http.lowSpeedLimit + http.lowSpeedTime + http.maxRequests + http.minSessions + http.noEPSV + http.postBuffer + http.proxy + http.sslCAInfo + http.sslCAPath + http.sslCert + http.sslCertPasswordProtected + http.sslKey + http.sslVerify + http.useragent + i18n.commitEncoding + i18n.logOutputEncoding + imap.authMethod + imap.folder + imap.host + imap.pass + imap.port + imap.preformattedHTML + imap.sslverify + imap.tunnel + imap.user + init.templatedir + instaweb.browser + instaweb.httpd + instaweb.local + instaweb.modulepath + instaweb.port + interactive.singlekey + log.date + log.decorate + log.showroot + mailmap.file + man. + man.viewer + merge. + merge.conflictstyle + merge.log + merge.renameLimit + merge.renormalize + merge.stat + merge.tool + merge.verbosity + mergetool. + mergetool.keepBackup + mergetool.keepTemporaries + mergetool.prompt + notes.displayRef + notes.rewrite. + notes.rewrite.amend + notes.rewrite.rebase + notes.rewriteMode + notes.rewriteRef + pack.compression + pack.deltaCacheLimit + pack.deltaCacheSize + pack.depth + pack.indexVersion + pack.packSizeLimit + pack.threads + pack.window + pack.windowMemory + pager. + pretty. + pull.octopus + pull.twohead + push.default + rebase.autosquash + rebase.stat + receive.autogc + receive.denyCurrentBranch + receive.denyDeleteCurrent + receive.denyDeletes + receive.denyNonFastForwards + receive.fsckObjects + receive.unpackLimit + receive.updateserverinfo + remote.pushdefault + remotes. + repack.usedeltabaseoffset + rerere.autoupdate + rerere.enabled + sendemail. + sendemail.aliasesfile + sendemail.aliasfiletype + sendemail.bcc + sendemail.cc + sendemail.cccmd + sendemail.chainreplyto + sendemail.confirm + sendemail.envelopesender + sendemail.from + sendemail.identity + sendemail.multiedit + sendemail.signedoffbycc + sendemail.smtpdomain + sendemail.smtpencryption + sendemail.smtppass + sendemail.smtpserver + sendemail.smtpserveroption + sendemail.smtpserverport + sendemail.smtpuser + sendemail.suppresscc + sendemail.suppressfrom + sendemail.thread + sendemail.to + sendemail.validate + showbranch.default + status.relativePaths + status.showUntrackedFiles + status.submodulesummary + submodule. + tar.umask + transfer.unpackLimit + url. + user.email + user.name + user.signingkey + web.browser + branch. remote. + " +} + +_git_remote () +{ + local subcommands="add rename remove set-head set-branches set-url show prune update" + local subcommand="$(__git_find_on_cmdline "$subcommands")" + if [ -z "$subcommand" ]; then + __gitcomp "$subcommands" + return + fi + + case "$subcommand" in + rename|remove|set-url|show|prune) + __gitcomp_nl "$(__git_remotes)" + ;; + set-head|set-branches) + __git_complete_remote_or_refspec + ;; + update) + local i c='' IFS=$'\n' + for i in $(git --git-dir="$(__gitdir)" config --get-regexp "remotes\..*" 2>/dev/null); do + i="${i#remotes.}" + c="$c ${i/ */}" + done + __gitcomp "$c" + ;; + *) + ;; + esac +} + +_git_replace () +{ + __gitcomp_nl "$(__git_refs)" +} + +_git_reset () +{ + __git_has_doubledash && return + + case "$cur" in + --*) + __gitcomp "--merge --mixed --hard --soft --patch" + return + ;; + esac + __gitcomp_nl "$(__git_refs)" +} + +_git_revert () +{ + case "$cur" in + --*) + __gitcomp "--edit --mainline --no-edit --no-commit --signoff" + return + ;; + esac + __gitcomp_nl "$(__git_refs)" +} + +_git_rm () +{ + case "$cur" in + --*) + __gitcomp "--cached --dry-run --ignore-unmatch --quiet" + return + ;; + esac + + __git_complete_index_file "--cached" +} + +_git_shortlog () +{ + __git_has_doubledash && return + + case "$cur" in + --*) + __gitcomp " + $__git_log_common_options + $__git_log_shortlog_options + --numbered --summary + " + return + ;; + esac + __git_complete_revlist +} + +_git_show () +{ + __git_has_doubledash && return + + case "$cur" in + --pretty=*|--format=*) + __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) + " "" "${cur#*=}" + return + ;; + --diff-algorithm=*) + __gitcomp "$__git_diff_algorithms" "" "${cur##--diff-algorithm=}" + return + ;; + --*) + __gitcomp "--pretty= --format= --abbrev-commit --oneline + $__git_diff_common_options + " + return + ;; + esac + __git_complete_revlist_file +} + +_git_show_branch () +{ + case "$cur" in + --*) + __gitcomp " + --all --remotes --topo-order --current --more= + --list --independent --merge-base --no-name + --color --no-color + --sha1-name --sparse --topics --reflog + " + return + ;; + esac + __git_complete_revlist +} + +_git_stash () +{ + local save_opts='--keep-index --no-keep-index --quiet --patch' + local subcommands='save list show apply clear drop pop create branch' + local subcommand="$(__git_find_on_cmdline "$subcommands")" + if [ -z "$subcommand" ]; then + case "$cur" in + --*) + __gitcomp "$save_opts" + ;; + *) + if [ -z "$(__git_find_on_cmdline "$save_opts")" ]; then + __gitcomp "$subcommands" + fi + ;; + esac + else + case "$subcommand,$cur" in + save,--*) + __gitcomp "$save_opts" + ;; + apply,--*|pop,--*) + __gitcomp "--index --quiet" + ;; + show,--*|drop,--*|branch,--*) + ;; + show,*|apply,*|drop,*|pop,*|branch,*) + __gitcomp_nl "$(git --git-dir="$(__gitdir)" stash list \ + | sed -n -e 's/:.*//p')" + ;; + *) + ;; + esac + fi +} + +_git_submodule () +{ + __git_has_doubledash && return + + local subcommands="add status init deinit update summary foreach sync" + if [ -z "$(__git_find_on_cmdline "$subcommands")" ]; then + case "$cur" in + --*) + __gitcomp "--quiet --cached" + ;; + *) + __gitcomp "$subcommands" + ;; + esac + return + fi +} + +_git_svn () +{ + local subcommands=" + init fetch clone rebase dcommit log find-rev + set-tree commit-diff info create-ignore propget + proplist show-ignore show-externals branch tag blame + migrate mkdirs reset gc + " + local subcommand="$(__git_find_on_cmdline "$subcommands")" + if [ -z "$subcommand" ]; then + __gitcomp "$subcommands" + else + local remote_opts="--username= --config-dir= --no-auth-cache" + local fc_opts=" + --follow-parent --authors-file= --repack= + --no-metadata --use-svm-props --use-svnsync-props + --log-window-size= --no-checkout --quiet + --repack-flags --use-log-author --localtime + --ignore-paths= --include-paths= $remote_opts + " + local init_opts=" + --template= --shared= --trunk= --tags= + --branches= --stdlayout --minimize-url + --no-metadata --use-svm-props --use-svnsync-props + --rewrite-root= --prefix= --use-log-author + --add-author-from $remote_opts + " + local cmt_opts=" + --edit --rmdir --find-copies-harder --copy-similarity= + " + + case "$subcommand,$cur" in + fetch,--*) + __gitcomp "--revision= --fetch-all $fc_opts" + ;; + clone,--*) + __gitcomp "--revision= $fc_opts $init_opts" + ;; + init,--*) + __gitcomp "$init_opts" + ;; + dcommit,--*) + __gitcomp " + --merge --strategy= --verbose --dry-run + --fetch-all --no-rebase --commit-url + --revision --interactive $cmt_opts $fc_opts + " + ;; + set-tree,--*) + __gitcomp "--stdin $cmt_opts $fc_opts" + ;; + create-ignore,--*|propget,--*|proplist,--*|show-ignore,--*|\ + show-externals,--*|mkdirs,--*) + __gitcomp "--revision=" + ;; + log,--*) + __gitcomp " + --limit= --revision= --verbose --incremental + --oneline --show-commit --non-recursive + --authors-file= --color + " + ;; + rebase,--*) + __gitcomp " + --merge --verbose --strategy= --local + --fetch-all --dry-run $fc_opts + " + ;; + commit-diff,--*) + __gitcomp "--message= --file= --revision= $cmt_opts" + ;; + info,--*) + __gitcomp "--url" + ;; + branch,--*) + __gitcomp "--dry-run --message --tag" + ;; + tag,--*) + __gitcomp "--dry-run --message" + ;; + blame,--*) + __gitcomp "--git-format" + ;; + migrate,--*) + __gitcomp " + --config-dir= --ignore-paths= --minimize + --no-auth-cache --username= + " + ;; + reset,--*) + __gitcomp "--revision= --parent" + ;; + *) + ;; + esac + fi +} + +_git_tag () +{ + local i c=1 f=0 + while [ $c -lt $cword ]; do + i="${words[c]}" + case "$i" in + -d|-v) + __gitcomp_nl "$(__git_tags)" + return + ;; + -f) + f=1 + ;; + esac + ((c++)) + done + + case "$prev" in + -m|-F) + ;; + -*|tag) + if [ $f = 1 ]; then + __gitcomp_nl "$(__git_tags)" + fi + ;; + *) + __gitcomp_nl "$(__git_refs)" + ;; + esac +} + +_git_whatchanged () +{ + _git_log +} + +__git_main () +{ + local i c=1 command __git_dir + + while [ $c -lt $cword ]; do + i="${words[c]}" + case "$i" in + --git-dir=*) __git_dir="${i#--git-dir=}" ;; + --git-dir) ((c++)) ; __git_dir="${words[c]}" ;; + --bare) __git_dir="." ;; + --help) command="help"; break ;; + -c|--work-tree|--namespace) ((c++)) ;; + -*) ;; + *) command="$i"; break ;; + esac + ((c++)) + done + + if [ -z "$command" ]; then + case "$cur" in + --*) __gitcomp " + --paginate + --no-pager + --git-dir= + --bare + --version + --exec-path + --exec-path= + --html-path + --man-path + --info-path + --work-tree= + --namespace= + --no-replace-objects + --help + " + ;; + *) __git_compute_porcelain_commands + __gitcomp "$__git_porcelain_commands $(__git_aliases)" ;; + esac + return + fi + + local completion_func="_git_${command//-/_}" + declare -f $completion_func >/dev/null && $completion_func && return + + local expansion=$(__git_aliased_command "$command") + if [ -n "$expansion" ]; then + completion_func="_git_${expansion//-/_}" + declare -f $completion_func >/dev/null && $completion_func + fi +} + +__gitk_main () +{ + __git_has_doubledash && return + + local g="$(__gitdir)" + local merge="" + if [ -f "$g/MERGE_HEAD" ]; then + merge="--merge" + fi + case "$cur" in + --*) + __gitcomp " + $__git_log_common_options + $__git_log_gitk_options + $merge + " + return + ;; + esac + __git_complete_revlist +} + +if [[ -n ${ZSH_VERSION-} ]]; then + echo "WARNING: this script is deprecated, please see git-completion.zsh" 1>&2 + + autoload -U +X compinit && compinit + + __gitcomp () + { + emulate -L zsh + + local cur_="${3-$cur}" + + case "$cur_" in + --*=) + ;; + *) + local c IFS=$' \t\n' + local -a array + for c in ${=1}; do + c="$c${4-}" + case $c in + --*=*|*.) ;; + *) c="$c " ;; + esac + array[$#array+1]="$c" + done + compset -P '*[=:]' + compadd -Q -S '' -p "${2-}" -a -- array && _ret=0 + ;; + esac + } + + __gitcomp_nl () + { + emulate -L zsh + + local IFS=$'\n' + compset -P '*[=:]' + compadd -Q -S "${4- }" -p "${2-}" -- ${=1} && _ret=0 + } + + __gitcomp_file () + { + emulate -L zsh + + local IFS=$'\n' + compset -P '*[=:]' + compadd -Q -p "${2-}" -f -- ${=1} && _ret=0 + } + + _git () + { + local _ret=1 cur cword prev + cur=${words[CURRENT]} + prev=${words[CURRENT-1]} + let cword=CURRENT-1 + emulate ksh -c __${service}_main + let _ret && _default && _ret=0 + return _ret + } + + compdef _git git gitk + return +fi + +__git_func_wrap () +{ + local cur words cword prev + _get_comp_words_by_ref -n =: cur words cword prev + $1 +} + +# Setup completion for certain functions defined above by setting common +# variables and workarounds. +# This is NOT a public function; use at your own risk. +__git_complete () +{ + local wrapper="__git_wrap${2}" + eval "$wrapper () { __git_func_wrap $2 ; }" + complete -o bashdefault -o default -o nospace -F $wrapper $1 2>/dev/null \ + || complete -o default -o nospace -F $wrapper $1 +} + +# wrapper for backwards compatibility +_git () +{ + __git_wrap__git_main +} + +# wrapper for backwards compatibility +_gitk () +{ + __git_wrap__gitk_main +} + +__git_complete git __git_main +__git_complete gitk __gitk_main + +# The following are necessary only for Cygwin, and only are needed +# when the user has tab-completed the executable name and consequently +# included the '.exe' suffix. +# +if [ Cygwin = "$(uname -o 2>/dev/null)" ]; then +__git_complete git.exe __git_main +fi diff --git a/tools/dockerfile/root/.scripts/git-prompt.sh b/tools/dockerfile/root/.scripts/git-prompt.sh new file mode 100755 index 0000000000000000000000000000000000000000..576f4ec14c94a24ebffa9e2620acf881e6b5ddaa --- /dev/null +++ b/tools/dockerfile/root/.scripts/git-prompt.sh @@ -0,0 +1,445 @@ +# bash/zsh git prompt support +# +# Copyright (C) 2006,2007 Shawn O. Pearce +# Distributed under the GNU General Public License, version 2.0. +# +# This script allows you to see repository status in your prompt. +# +# To enable: +# +# 1) Copy this file to somewhere (e.g. ~/.git-prompt.sh). +# 2) Add the following line to your .bashrc/.zshrc: +# source ~/.git-prompt.sh +# 3a) Change your PS1 to call __git_ps1 as +# command-substitution: +# Bash: PS1='[\u@\h \W$(__git_ps1 " (%s)")]\$ ' +# ZSH: setopt PROMPT_SUBST ; PS1='[%n@%m %c$(__git_ps1 " (%s)")]\$ ' +# the optional argument will be used as format string. +# 3b) Alternatively, for a slightly faster prompt, __git_ps1 can +# be used for PROMPT_COMMAND in Bash or for precmd() in Zsh +# with two parameters,
 and , which are strings
+#        you would put in $PS1 before and after the status string
+#        generated by the git-prompt machinery.  e.g.
+#        Bash: PROMPT_COMMAND='__git_ps1 "\u@\h:\w" "\\\$ "'
+#          will show username, at-sign, host, colon, cwd, then
+#          various status string, followed by dollar and SP, as
+#          your prompt.
+#        ZSH:  precmd () { __git_ps1 "%n" ":%~$ " "|%s" }
+#          will show username, pipe, then various status string,
+#          followed by colon, cwd, dollar and SP, as your prompt.
+#        Optionally, you can supply a third argument with a printf
+#        format string to finetune the output of the branch status
+#
+# The repository status will be displayed only if you are currently in a
+# git repository. The %s token is the placeholder for the shown status.
+#
+# The prompt status always includes the current branch name.
+#
+# In addition, if you set GIT_PS1_SHOWDIRTYSTATE to a nonempty value,
+# unstaged (*) and staged (+) changes will be shown next to the branch
+# name.  You can configure this per-repository with the
+# bash.showDirtyState variable, which defaults to true once
+# GIT_PS1_SHOWDIRTYSTATE is enabled.
+#
+# You can also see if currently something is stashed, by setting
+# GIT_PS1_SHOWSTASHSTATE to a nonempty value. If something is stashed,
+# then a '$' will be shown next to the branch name.
+#
+# If you would like to see if there're untracked files, then you can set
+# GIT_PS1_SHOWUNTRACKEDFILES to a nonempty value. If there're untracked
+# files, then a '%' will be shown next to the branch name.  You can
+# configure this per-repository with the bash.showUntrackedFiles
+# variable, which defaults to true once GIT_PS1_SHOWUNTRACKEDFILES is
+# enabled.
+#
+# If you would like to see the difference between HEAD and its upstream,
+# set GIT_PS1_SHOWUPSTREAM="auto".  A "<" indicates you are behind, ">"
+# indicates you are ahead, "<>" indicates you have diverged and "="
+# indicates that there is no difference. You can further control
+# behaviour by setting GIT_PS1_SHOWUPSTREAM to a space-separated list
+# of values:
+#
+#     verbose       show number of commits ahead/behind (+/-) upstream
+#     legacy        don't use the '--count' option available in recent
+#                   versions of git-rev-list
+#     git           always compare HEAD to @{upstream}
+#     svn           always compare HEAD to your SVN upstream
+#
+# By default, __git_ps1 will compare HEAD to your SVN upstream if it can
+# find one, or @{upstream} otherwise.  Once you have set
+# GIT_PS1_SHOWUPSTREAM, you can override it on a per-repository basis by
+# setting the bash.showUpstream config variable.
+#
+# If you would like to see more information about the identity of
+# commits checked out as a detached HEAD, set GIT_PS1_DESCRIBE_STYLE
+# to one of these values:
+#
+#     contains      relative to newer annotated tag (v1.6.3.2~35)
+#     branch        relative to newer tag or branch (master~4)
+#     describe      relative to older annotated tag (v1.6.3.1-13-gdd42c2f)
+#     default       exactly matching tag
+#
+# If you would like a colored hint about the current dirty state, set
+# GIT_PS1_SHOWCOLORHINTS to a nonempty value. The colors are based on
+# the colored output of "git status -sb" and are available only when
+# using __git_ps1 for PROMPT_COMMAND or precmd.
+
+# stores the divergence from upstream in $p
+# used by GIT_PS1_SHOWUPSTREAM
+__git_ps1_show_upstream ()
+{
+  local key value
+  local svn_remote svn_url_pattern count n
+  local upstream=git legacy="" verbose=""
+
+  svn_remote=()
+  # get some config options from git-config
+  local output="$(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ')"
+  while read -r key value; do
+    case "$key" in
+    bash.showupstream)
+      GIT_PS1_SHOWUPSTREAM="$value"
+      if [[ -z "${GIT_PS1_SHOWUPSTREAM}" ]]; then
+        p=""
+        return
+      fi
+      ;;
+    svn-remote.*.url)
+      svn_remote[$((${#svn_remote[@]} + 1))]="$value"
+      svn_url_pattern+="\\|$value"
+      upstream=svn+git # default upstream is SVN if available, else git
+      ;;
+    esac
+  done <<< "$output"
+
+  # parse configuration values
+  for option in ${GIT_PS1_SHOWUPSTREAM}; do
+    case "$option" in
+    git|svn) upstream="$option" ;;
+    verbose) verbose=1 ;;
+    legacy)  legacy=1  ;;
+    esac
+  done
+
+  # Find our upstream
+  case "$upstream" in
+  git)    upstream="@{upstream}" ;;
+  svn*)
+    # get the upstream from the "git-svn-id: ..." in a commit message
+    # (git-svn uses essentially the same procedure internally)
+    local -a svn_upstream
+    svn_upstream=($(git log --first-parent -1 \
+          --grep="^git-svn-id: \(${svn_url_pattern#??}\)" 2>/dev/null))
+    if [[ 0 -ne ${#svn_upstream[@]} ]]; then
+      svn_upstream=${svn_upstream[${#svn_upstream[@]} - 2]}
+      svn_upstream=${svn_upstream%@*}
+      local n_stop="${#svn_remote[@]}"
+      for ((n=1; n <= n_stop; n++)); do
+        svn_upstream=${svn_upstream#${svn_remote[$n]}}
+      done
+
+      if [[ -z "$svn_upstream" ]]; then
+        # default branch name for checkouts with no layout:
+        upstream=${GIT_SVN_ID:-git-svn}
+      else
+        upstream=${svn_upstream#/}
+      fi
+    elif [[ "svn+git" = "$upstream" ]]; then
+      upstream="@{upstream}"
+    fi
+    ;;
+  esac
+
+  # Find how many commits we are ahead/behind our upstream
+  if [[ -z "$legacy" ]]; then
+    count="$(git rev-list --count --left-right \
+        "$upstream"...HEAD 2>/dev/null)"
+  else
+    # produce equivalent output to --count for older versions of git
+    local commits
+    if commits="$(git rev-list --left-right "$upstream"...HEAD 2>/dev/null)"
+    then
+      local commit behind=0 ahead=0
+      for commit in $commits
+      do
+        case "$commit" in
+        "<"*) ((behind++)) ;;
+        *)    ((ahead++))  ;;
+        esac
+      done
+      count="$behind  $ahead"
+    else
+      count=""
+    fi
+  fi
+
+  # calculate the result
+  if [[ -z "$verbose" ]]; then
+    case "$count" in
+    "") # no upstream
+      p="" ;;
+    "0  0") # equal to upstream
+      p="=" ;;
+    "0  "*) # ahead of upstream
+      p=">" ;;
+    *"  0") # behind upstream
+      p="<" ;;
+    *)      # diverged from upstream
+      p="<>" ;;
+    esac
+  else
+    case "$count" in
+    "") # no upstream
+      p="" ;;
+    "0  0") # equal to upstream
+      p=" u=" ;;
+    "0  "*) # ahead of upstream
+      p=" u+${count#0 }" ;;
+    *"  0") # behind upstream
+      p=" u-${count%  0}" ;;
+    *)      # diverged from upstream
+      p=" u+${count#* }-${count%  *}" ;;
+    esac
+  fi
+
+}
+
+# Helper function that is meant to be called from __git_ps1.  It
+# injects color codes into the appropriate gitstring variables used
+# to build a gitstring.
+__git_ps1_colorize_gitstring ()
+{
+  if [[ -n ${ZSH_VERSION-} ]]; then
+    local c_red='%F{red}'
+    local c_green='%F{green}'
+    local c_lblue='%F{blue}'
+    local c_clear='%f'
+  else
+    # Using \[ and \] around colors is necessary to prevent
+    # issues with command line editing/browsing/completion!
+    local c_red='\[\e[31m\]'
+    local c_green='\[\e[32m\]'
+    local c_lblue='\[\e[1;34m\]'
+    local c_clear='\[\e[0m\]'
+  fi
+  local bad_color=$c_red
+  local ok_color=$c_green
+  local flags_color="$c_lblue"
+
+  local branch_color=""
+  if [ $detached = no ]; then
+    branch_color="$ok_color"
+  else
+    branch_color="$bad_color"
+  fi
+  c="$branch_color$c"
+
+  z="$c_clear$z"
+  if [ "$w" = "*" ]; then
+    w="$bad_color$w"
+  fi
+  if [ -n "$i" ]; then
+    i="$ok_color$i"
+  fi
+  if [ -n "$s" ]; then
+    s="$flags_color$s"
+  fi
+  if [ -n "$u" ]; then
+    u="$bad_color$u"
+  fi
+  r="$c_clear$r"
+}
+
+# __git_ps1 accepts 0 or 1 arguments (i.e., format string)
+# when called from PS1 using command substitution
+# in this mode it prints text to add to bash PS1 prompt (includes branch name)
+#
+# __git_ps1 requires 2 or 3 arguments when called from PROMPT_COMMAND (pc)
+# in that case it _sets_ PS1. The arguments are parts of a PS1 string.
+# when two arguments are given, the first is prepended and the second appended
+# to the state string when assigned to PS1.
+# The optional third parameter will be used as printf format string to further
+# customize the output of the git-status string.
+# In this mode you can request colored hints using GIT_PS1_SHOWCOLORHINTS=true
+__git_ps1 ()
+{
+  local pcmode=no
+  local detached=no
+  local ps1pc_start='\u@\h:\w '
+  local ps1pc_end='\$ '
+  local printf_format=' (%s)'
+
+  case "$#" in
+    2|3)  pcmode=yes
+      ps1pc_start="$1"
+      ps1pc_end="$2"
+      printf_format="${3:-$printf_format}"
+    ;;
+    0|1)  printf_format="${1:-$printf_format}"
+    ;;
+    *)  return
+    ;;
+  esac
+
+  local repo_info rev_parse_exit_code
+  repo_info="$(git rev-parse --git-dir --is-inside-git-dir \
+    --is-bare-repository --is-inside-work-tree \
+    --short HEAD 2>/dev/null)"
+  rev_parse_exit_code="$?"
+
+  if [ -z "$repo_info" ]; then
+    if [ $pcmode = yes ]; then
+      #In PC mode PS1 always needs to be set
+      PS1="$ps1pc_start$ps1pc_end"
+    fi
+    return
+  fi
+
+  local short_sha
+  if [ "$rev_parse_exit_code" = "0" ]; then
+    short_sha="${repo_info##*$'\n'}"
+    repo_info="${repo_info%$'\n'*}"
+  fi
+  local inside_worktree="${repo_info##*$'\n'}"
+  repo_info="${repo_info%$'\n'*}"
+  local bare_repo="${repo_info##*$'\n'}"
+  repo_info="${repo_info%$'\n'*}"
+  local inside_gitdir="${repo_info##*$'\n'}"
+  local g="${repo_info%$'\n'*}"
+
+  local r=""
+  local b=""
+  local step=""
+  local total=""
+  if [ -d "$g/rebase-merge" ]; then
+    read b 2>/dev/null <"$g/rebase-merge/head-name"
+    read step 2>/dev/null <"$g/rebase-merge/msgnum"
+    read total 2>/dev/null <"$g/rebase-merge/end"
+    if [ -f "$g/rebase-merge/interactive" ]; then
+      r="|REBASE-i"
+    else
+      r="|REBASE-m"
+    fi
+  else
+    if [ -d "$g/rebase-apply" ]; then
+      read step 2>/dev/null <"$g/rebase-apply/next"
+      read total 2>/dev/null <"$g/rebase-apply/last"
+      if [ -f "$g/rebase-apply/rebasing" ]; then
+        read b 2>/dev/null <"$g/rebase-apply/head-name"
+        r="|REBASE"
+      elif [ -f "$g/rebase-apply/applying" ]; then
+        r="|AM"
+      else
+        r="|AM/REBASE"
+      fi
+    elif [ -f "$g/MERGE_HEAD" ]; then
+      r="|MERGING"
+    elif [ -f "$g/CHERRY_PICK_HEAD" ]; then
+      r="|CHERRY-PICKING"
+    elif [ -f "$g/REVERT_HEAD" ]; then
+      r="|REVERTING"
+    elif [ -f "$g/BISECT_LOG" ]; then
+      r="|BISECTING"
+    fi
+
+    if [ -n "$b" ]; then
+      :
+    elif [ -h "$g/HEAD" ]; then
+      # symlink symbolic ref
+      b="$(git symbolic-ref HEAD 2>/dev/null)"
+    else
+      local head=""
+      if ! read head 2>/dev/null <"$g/HEAD"; then
+        if [ $pcmode = yes ]; then
+          PS1="$ps1pc_start$ps1pc_end"
+        fi
+        return
+      fi
+      # is it a symbolic ref?
+      b="${head#ref: }"
+      if [ "$head" = "$b" ]; then
+        detached=yes
+        b="$(
+        case "${GIT_PS1_DESCRIBE_STYLE-}" in
+        (contains)
+          git describe --contains HEAD ;;
+        (branch)
+          git describe --contains --all HEAD ;;
+        (describe)
+          git describe HEAD ;;
+        (* | default)
+          git describe --tags --exact-match HEAD ;;
+        esac 2>/dev/null)" ||
+
+        b="$short_sha..."
+        b="($b)"
+      fi
+    fi
+  fi
+
+  if [ -n "$step" ] && [ -n "$total" ]; then
+    r="$r $step/$total"
+  fi
+
+  local w=""
+  local i=""
+  local s=""
+  local u=""
+  local c=""
+  local p=""
+
+  if [ "true" = "$inside_gitdir" ]; then
+    if [ "true" = "$bare_repo" ]; then
+      c="BARE:"
+    else
+      b="GIT_DIR!"
+    fi
+  elif [ "true" = "$inside_worktree" ]; then
+    if [ -n "${GIT_PS1_SHOWDIRTYSTATE-}" ] &&
+       [ "$(git config --bool bash.showDirtyState)" != "false" ]
+    then
+      git diff --no-ext-diff --quiet --exit-code || w="*"
+      if [ -n "$short_sha" ]; then
+        git diff-index --cached --quiet HEAD -- || i="+"
+      else
+        i="#"
+      fi
+    fi
+    if [ -n "${GIT_PS1_SHOWSTASHSTATE-}" ] &&
+       [ -r "$g/refs/stash" ]; then
+      s="$"
+    fi
+
+    if [ -n "${GIT_PS1_SHOWUNTRACKEDFILES-}" ] &&
+       [ "$(git config --bool bash.showUntrackedFiles)" != "false" ] &&
+       git ls-files --others --exclude-standard --error-unmatch -- '*' >/dev/null 2>/dev/null
+    then
+      u="%${ZSH_VERSION+%}"
+    fi
+
+    if [ -n "${GIT_PS1_SHOWUPSTREAM-}" ]; then
+      __git_ps1_show_upstream
+    fi
+  fi
+
+  local z="${GIT_PS1_STATESEPARATOR-" "}"
+
+  # NO color option unless in PROMPT_COMMAND mode
+  if [ $pcmode = yes ] && [ -n "${GIT_PS1_SHOWCOLORHINTS-}" ]; then
+    __git_ps1_colorize_gitstring
+  fi
+
+  local f="$w$i$s$u"
+  local gitstring="$c${b##refs/heads/}${f:+$z$f}$r$p"
+
+  if [ $pcmode = yes ]; then
+    if [[ -n ${ZSH_VERSION-} ]]; then
+      gitstring=$(printf -- "$printf_format" "$gitstring")
+    else
+      printf -v gitstring -- "$printf_format" "$gitstring"
+    fi
+    PS1="$ps1pc_start$gitstring$ps1pc_end"
+  else
+    printf -- "$printf_format" "$gitstring"
+  fi
+}