Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2299
Star 20931
Fork 5422
P
Paddle
未验证 提交 97ec1d84 编写于 作者: T tianshuo78520a 提交者： GitHub
浏览文件
操作

Add py3.10 (#55286) 

* Add py3.10;test=py3-ninja

* Add py3.10;test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* Fix test error

* Fix build docker error

* Fix build docker error
上级 ba6dcbc9
显示空白变更内容
内联 并排
Showing with 33 addition and 49 deletion
python/paddle/utils/cpp_extension/extension_utils.py
浏览文件 @ 97ec1d84
...@@ -16,6 +16,7 @@ import atexit ...@@ -16,6 +16,7 @@ import atexit
import collections import collections
import glob import glob
import hashlib import hashlib
import importlib.abc
import importlib.util import importlib.util
import json import json
import logging import logging
...@@ -173,6 +174,7 @@ def custom_write_stub(resource, pyfile): ...@@ -173,6 +174,7 @@ def custom_write_stub(resource, pyfile):
import sys import sys
import types import types
import paddle import paddle
import importlib.abc
import importlib.util import importlib.util
cur_dir = os.path.dirname(os.path.abspath(__file__)) cur_dir = os.path.dirname(os.path.abspath(__file__))
......
tools/dockerfile/Dockerfile.ubuntu20
浏览文件 @ 97ec1d84
...@@ -61,10 +61,11 @@ ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH ...@@ -61,10 +61,11 @@ ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y python3.7 python3.7-dev python3.7-distutils\ apt-get install -y python3.7 python3.7-dev python3.7-distutils\
python3.8 python3.8-dev python3.8-distutils \ python3.8 python3.8-dev python3.8-distutils \
python3.9 python3.9-dev python3.9-distutils && \ python3.9 python3.9-dev python3.9-distutils \
python3.10 python3.10-dev python3.10-distutils && \
apt-get install python-is-python3 && \ apt-get install python-is-python3 && \
rm /usr/bin/python && ln -s /usr/bin/python3.7 /usr/bin/python && \ rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3 rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
WORKDIR /home WORKDIR /home
...@@ -72,7 +73,8 @@ RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b ...@@ -72,7 +73,8 @@ RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b
WORKDIR /home/setuptools-50.3.2 WORKDIR /home/setuptools-50.3.2
RUN python3.9 setup.py build && python3.9 setup.py install && \ RUN python3.9 setup.py build && python3.9 setup.py install && \
python3.8 setup.py build && python3.8 setup.py install && \ python3.8 setup.py build && python3.8 setup.py install && \
python3.7 setup.py build && python3.7 setup.py install python3.7 setup.py build && python3.7 setup.py install && \
python3.10 setup.py build && python3.10 setup.py install
WORKDIR /home WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
...@@ -80,14 +82,12 @@ WORKDIR pip-20.0.1 ...@@ -80,14 +82,12 @@ WORKDIR pip-20.0.1
RUN python3.9 setup.py install && \ RUN python3.9 setup.py install && \
python3.8 setup.py install && \ python3.8 setup.py install && \
python3.7 setup.py install python3.7 setup.py install && \
python3.10 setup.py install
WORKDIR /home WORKDIR /home
RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \ RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
rm -r setuptools-50.3.2 pip-20.0.1 rm -r setuptools-50.3.2 pip-20.0.1
RUN rm /usr/local/bin/pip && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip && \
rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3
# remove them when apt-get support 2.27 and higher version # remove them when apt-get support 2.27 and higher version
RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
...@@ -95,7 +95,6 @@ RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ ...@@ -95,7 +95,6 @@ RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
cd binutils-2.33.1 && \ cd binutils-2.33.1 && \
./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz
# Install Go and glide # Install Go and glide
RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \ RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
tar -xz -C /usr/local && \ tar -xz -C /usr/local && \
...@@ -114,34 +113,44 @@ RUN git config --global credential.helper store ...@@ -114,34 +113,44 @@ RUN git config --global credential.helper store
# Fix locales to en_US.UTF-8 # Fix locales to en_US.UTF-8
RUN localedef -i en_US -f UTF-8 en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
#For pre-commit
RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \
python3.8 -m pip --no-cache-dir install --upgrade pip && \
python3.9 -m pip --no-cache-dir install --upgrade pip && \
python3.10 -m pip --no-cache-dir install --upgrade pip
RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \
rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3
RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \ RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \
pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.8 --no-cache-dir install ipython==5.3.0 && \ pip3.8 --no-cache-dir install ipython==5.3.0 && \
pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.9 --no-cache-dir install ipython==5.3.0 && \ pip3.9 --no-cache-dir install ipython==5.3.0 && \
pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.10 --no-cache-dir install ipython==5.3.0 && \
pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel
#For docstring checker #For docstring checker
RUN pip3.7 --no-cache-dir install pytest astroid isort && \ RUN pip3.7 --no-cache-dir install pytest astroid isort && \
pip3.8 --no-cache-dir install pytest astroid isort && \ pip3.8 --no-cache-dir install pytest astroid isort && \
pip3.9 --no-cache-dir install pytest astroid isort pip3.9 --no-cache-dir install pytest astroid isort && \
pip3.10 --no-cache-dir install pytest astroid isort
#For pre-commit
RUN pip3.7 --no-cache-dir install --upgrade pip && \
pip3.8 --no-cache-dir install --upgrade pip && \
pip3.9 --no-cache-dir install --upgrade pip
RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
COPY ./python/requirements.txt /root/ COPY ./python/requirements.txt /root/
RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \ RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \
pip3.8 --no-cache-dir install -r /root/requirements.txt && \ pip3.8 --no-cache-dir install -r /root/requirements.txt && \
pip3.9 --no-cache-dir install -r /root/requirements.txt pip3.9 --no-cache-dir install -r /root/requirements.txt && \
pip3.10 --no-cache-dir install -r /root/requirements.txt
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config #RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
......
tools/dockerfile/ci_dockerfile.sh
浏览文件 @ 97ec1d84
...@@ -14,32 +14,6 @@ ...@@ -14,32 +14,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
function make_ubuntu_dockerfile(){
dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16"
sed "s/<baseimg>/10.1-cudnn7-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name}
sed -i "s#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g" ${dockerfile_name}
dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext zstd \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
tar -xvf git-2.17.1.tar.gz \&\& \
cd git-2.17.1 \&\& \
./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
make -j8 \&\& make install " ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name}
sed -i "s#<install_gcc>#WORKDIR /usr/bin \\
COPY tools/dockerfile/build_scripts /build_scripts \\
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\
RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\
ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
sed -i "s#bash /build_scripts/install_nccl2.sh#wget -q --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
RUN dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y libsndfile1 libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 zstd pigz ninja-build --allow-change-held-packages #g" ${dockerfile_name}
}
function make_ubuntu_trt7_dockerfile(){ function make_ubuntu_trt7_dockerfile(){
dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16" dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16"
...@@ -88,9 +62,9 @@ function make_centos_dockerfile(){ ...@@ -88,9 +62,9 @@ function make_centos_dockerfile(){
sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name} sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libsndfile1 zstd pigz libcurl4-openssl-dev gettext zstd ninja-build" ${dockerfile_name} sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libsndfile1 zstd pigz ninja-build" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro \&\& pip3.9 install PyGithub wheel \&\& pip3.10 install PyGithub distro" ${dockerfile_name} sed -i "${dockerfile_line}i RUN pip install wheel PyGithub distro" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
tar -xvf git-2.17.1.tar.gz \&\& \ tar -xvf git-2.17.1.tar.gz \&\& \
cd git-2.17.1 \&\& \ cd git-2.17.1 \&\& \
./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
...@@ -158,7 +132,7 @@ function make_unbuntu18_cu117_dockerfile(){ ...@@ -158,7 +132,7 @@ function make_unbuntu18_cu117_dockerfile(){
dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
tar -xvf git-2.17.1.tar.gz \&\& \ tar -xvf git-2.17.1.tar.gz \&\& \
cd git-2.17.1 \&\& \ cd git-2.17.1 \&\& \
./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
...@@ -196,7 +170,6 @@ function make_ubuntu18_cu112_dockerfile(){ ...@@ -196,7 +170,6 @@ function make_ubuntu18_cu112_dockerfile(){
} }
function main() { function main() {
make_ubuntu_dockerfile
make_ubuntu_trt7_dockerfile make_ubuntu_trt7_dockerfile
make_centos_dockerfile make_centos_dockerfile
make_cinn_dockerfile make_cinn_dockerfile
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或