From dbf92d499a140d94a295cf0a903c4679554b4774 Mon Sep 17 00:00:00 2001 From: Sing_chan <51314274+betterpig@users.noreply.github.com> Date: Thu, 23 Jun 2022 13:11:28 +0800 Subject: [PATCH] upgrade pre-commit tools in docker (#43534) * upgrade pre-commit tools in docker * python36 support pre-commit<=2.1.1 * upgrade pip; remove old clang-format; remove redundant apt update * remove redundant apt-get update * remove duplicate apt-get update that will cause cinn download fail * remove duplicate pip upgrade * add --fix-missing to avoid cinn docker build fail * remove duplicate apt-get update in make_cinn_docker * make cinn command in the end * solve gpg problem * remove unneccessary modification --- .pre-commit-config.yaml | 1 - tools/dockerfile/Dockerfile.ubuntu | 37 +++++++++++++++--------- tools/dockerfile/Dockerfile.ubuntu18 | 43 +++++++++++++++++++--------- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75ae8be2cb7..5c6bded87ce 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,4 +80,3 @@ repos: hooks: - id: cmakelint args: [--config=./tools/codestyle/.cmakelintrc] - # exclude files which need to be fixed diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu index 000e7098570..7e0c3a62b1d 100644 --- a/tools/dockerfile/Dockerfile.ubuntu +++ b/tools/dockerfile/Dockerfile.ubuntu @@ -28,7 +28,7 @@ RUN apt-get update && \ wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ curl sed grep graphviz libjpeg-dev zlib1g-dev \ python-matplotlib \ - automake locales clang-format swig \ + automake locales swig \ liblapack-dev liblapacke-dev \ net-tools libtool module-init-tools && \ apt-get clean -y @@ -172,24 +172,39 @@ RUN pip3.6 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ pip --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ pip --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark -RUN pip3.6 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ +RUN pip3.6 --no-cache-dir install 'ipython==5.3.0' && \ pip3.6 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.7 --no-cache-dir install 'ipython==5.3.0' && \ pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.8 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.8 --no-cache-dir install 'ipython==5.3.0' && \ pip3.8 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.9 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ + pip3.9 --no-cache-dir install 'ipython==5.3.0' && \ pip3.9 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ pip --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \ pip --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' #For docstring checker -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort && \ - pip3.7 --no-cache-dir install pylint pytest astroid isort && \ - pip3.8 --no-cache-dir install pylint pytest astroid isort && \ - pip3.9 --no-cache-dir install pylint pytest astroid isort && \ +RUN pip3.6 --no-cache-dir install pytest astroid isort && \ + pip3.7 --no-cache-dir install pytest astroid isort && \ + pip3.8 --no-cache-dir install pytest astroid isort && \ + pip3.9 --no-cache-dir install pytest astroid isort && \ pip --no-cache-dir install pylint pytest astroid isort +#For pre-commit +RUN pip3.6 --no-cache-dir install --upgrade pip==20.3.3 && \ + pip3.7 --no-cache-dir install --upgrade pip && \ + pip3.8 --no-cache-dir install --upgrade pip && \ + pip3.9 --no-cache-dir install --upgrade pip + +RUN pip3.6 --no-cache-dir install pre-commit==2.1.1 pylint==2.12.0 && \ + pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.6 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 + RUN pip3.6 --no-cache-dir install coverage && \ pip3.7 --no-cache-dir install coverage && \ pip3.8 --no-cache-dir install coverage && \ @@ -205,10 +220,6 @@ RUN pip3.6 --no-cache-dir install -r /root/requirements.txt && \ # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y && \ - pip3.6 install --upgrade pip==20.3.3 && \ - pip3.7 install --upgrade pip && \ - pip3.8 install --upgrade pip && \ - pip3.9 install --upgrade pip && \ pip3.6 --no-cache-dir install certifi urllib3[secure] && \ pip3.7 --no-cache-dir install certifi urllib3[secure] && \ pip3.8 --no-cache-dir install certifi urllib3[secure] && \ diff --git a/tools/dockerfile/Dockerfile.ubuntu18 b/tools/dockerfile/Dockerfile.ubuntu18 index 57c042f127a..3811004e7a0 100644 --- a/tools/dockerfile/Dockerfile.ubuntu18 +++ b/tools/dockerfile/Dockerfile.ubuntu18 @@ -18,6 +18,11 @@ ENV HOME /root COPY paddle/scripts/docker/root/ /root/ RUN chmod 777 /tmp + +RUN apt-key del 7fa2af80 +RUN rm /etc/apt/sources.list.d/cuda.list && rm /etc/apt/sources.list.d/nvidia-ml.list +RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub + RUN apt-get update --allow-unauthenticated && \ apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ @@ -107,23 +112,38 @@ RUN git config --global credential.helper store # Fix locales to en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8 -RUN pip3.6 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ +RUN pip3.6 --no-cache-dir install ipython==5.3.0 && \ pip3.6 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.7 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ + pip3.7 --no-cache-dir install ipython==5.3.0 && \ pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.8 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ + pip3.8 --no-cache-dir install ipython==5.3.0 && \ pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.9 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ + pip3.9 --no-cache-dir install ipython==5.3.0 && \ pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip2.7 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ pip2.7 --no-cache-dir install ipykernel==4.6.0 wheel #For docstring checker -RUN pip3.6 --no-cache-dir install pylint pytest astroid isort && \ - pip3.7 --no-cache-dir install pylint pytest astroid isort && \ - pip3.8 --no-cache-dir install pylint pytest astroid isort && \ - pip3.9 --no-cache-dir install pylint pytest astroid isort && \ - pip2.7 --no-cache-dir install pylint pytest astroid isort +RUN pip3.6 --no-cache-dir install pytest astroid isort && \ + pip3.7 --no-cache-dir install pytest astroid isort && \ + pip3.8 --no-cache-dir install pytest astroid isort && \ + pip3.9 --no-cache-dir install pytest astroid isort && \ + pip2.7 --no-cache-dir install pytest astroid isort + +#For pre-commit +RUN pip3.6 --no-cache-dir install --upgrade pip==20.3.3 && \ + pip3.7 --no-cache-dir install --upgrade pip && \ + pip3.8 --no-cache-dir install --upgrade pip && \ + pip3.9 --no-cache-dir install --upgrade pip + +RUN pip3.6 --no-cache-dir install pre-commit==2.1.1 pylint==2.12.0 && \ + pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.6 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 COPY ./python/requirements.txt /root/ RUN pip3.6 --no-cache-dir install -r /root/requirements.txt && \ @@ -151,9 +171,4 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache && \ cd ../ && rm -rf ccache-3.7.9 ccache-3.7.9.tar.gz -# clang-form 3.8.0 -RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \ - cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz - EXPOSE 22 -- GitLab