From d9b50f664f31f978222317d1bec38f673893806a Mon Sep 17 00:00:00 2001 From: Qi Li Date: Tue, 16 Mar 2021 16:35:31 +0800 Subject: [PATCH] [ROCM] update ci scripts and dockefile, test=develop (#31551) --- paddle/scripts/paddle_build.sh | 22 +++++++++++++++++++--- tools/dockerfile/Dockerfile.rocm | 16 +++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 3fd93a664d..7a360ac229 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -205,6 +205,13 @@ function cmake_base() { -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.8.0/include/python3.8 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.8.0/lib/libpython3.so" pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt + elif [ "$1" == "conda-python3.7" ]; then + export LD_LIBRARY_PATH=/opt/conda/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/conda/bin/:${PATH} + export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/conda/bin/python + -DPYTHON_INCLUDE_DIR:PATH=/opt/conda/include/python3.7m + -DPYTHON_LIBRARIES:FILEPATH=/opt/conda/lib/libpython3.so" + /opt/conda/bin/pip install -r ${PADDLE_ROOT}/python/requirements.txt fi else pip install -r ${PADDLE_ROOT}/python/requirements.txt @@ -230,7 +237,8 @@ function cmake_base() { ${PYTHON_FLAGS} -DWITH_GPU=${WITH_GPU:-OFF} -DWITH_TENSORRT=${WITH_TENSORRT:-ON} - -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} + -DWITH_ROCM=${WITH_ROCM:-OFF} + -DWITH_RCCL=${WITH_RCCL:-OFF} -DWITH_DISTRIBUTE=${distibuted_flag} -DWITH_MKL=${WITH_MKL:-ON} -DWITH_AVX=${WITH_AVX:-OFF} @@ -267,7 +275,8 @@ EOF ${PYTHON_FLAGS} \ -DWITH_GPU=${WITH_GPU:-OFF} \ -DWITH_TENSORRT=${WITH_TENSORRT:-ON} \ - -DWITH_AMD_GPU=${WITH_AMD_GPU:-OFF} \ + -DWITH_ROCM=${WITH_ROCM:-OFF} \ + -DWITH_RCCL=${WITH_RCCL:-OFF} \ -DWITH_DISTRIBUTE=${distibuted_flag} \ -DWITH_MKL=${WITH_MKL:-ON} \ -DWITH_AVX=${WITH_AVX:-OFF} \ @@ -1028,6 +1037,8 @@ function card_test() { # get the CUDA device count, XPU device count is one if [ "${WITH_XPU}" == "ON" ];then CUDA_DEVICE_COUNT=1 + elif [ "${WITH_ROCM}" == "ON" ];then + CUDA_DEVICE_COUNT=4 else CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l) fi @@ -1423,7 +1434,7 @@ function parallel_test() { mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build pip install ${PADDLE_ROOT}/build/python/dist/*whl - if [ "$WITH_GPU" == "ON" ];then + if [ "$WITH_GPU" == "ON" ] || [ "$WITH_ROCM" == "ON" ];then parallel_test_base_gpu else if [ "$WITH_XPU" == "ON" ];then @@ -1982,6 +1993,11 @@ function main() { parallel_test check_coverage ;; + check_rocm_coverage) + cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} + parallel_test + check_coverage + ;; cmake_gen) cmake_gen ${PYTHON_ABI:-""} ;; diff --git a/tools/dockerfile/Dockerfile.rocm b/tools/dockerfile/Dockerfile.rocm index eab4ef07c8..5df66b9ea6 100644 --- a/tools/dockerfile/Dockerfile.rocm +++ b/tools/dockerfile/Dockerfile.rocm @@ -5,7 +5,6 @@ # Build: ROCM 4.0.1 # cd Paddle/tools/dockerfile # docker build -f Dockerfile.rocm \ -# --build-arg ROCM_VERSION=4.0.1 \ # -t paddlepaddle/paddle-centos-rocm401-dev:latest . # # docker run -it --device=/dev/kfd --device=/dev/dri \ @@ -22,7 +21,7 @@ ENV LANGUAGE en_US.UTF-8 RUN yum install -y epel-release deltarpm sudo openssh-server gettext-devel sqlite-devel \ zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel \ make bzip2 git patch unzip bison yasm diffutils automake which file kernel-headers kernel-devel \ - net-tools numactl-devel chrpath + net-tools numactl-devel chrpath screen initscripts # Install devtoolset-7 RUN yum install -y yum-utils centos-release-scl && \ @@ -45,11 +44,10 @@ RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && \ ENV PATH=/opt/cmake-3.16/bin:${PATH} # ROCM -ARG ROCM_VERSION RUN yum install -y kmod wget openblas-devel epel-release RUN echo "[ROCm]" > /etc/yum.repos.d/rocm.repo && \ echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo && \ - echo "baseurl=http://repo.radeon.com/rocm/yum/${ROCM_VERSION}" >> /etc/yum.repos.d/rocm.repo && \ + echo "baseurl=http://repo.radeon.com/rocm/yum/4.0.1" >> /etc/yum.repos.d/rocm.repo && \ echo "enabled=1" >> /etc/yum.repos.d/rocm.repo && \ echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo RUN yum install -y rocm-dev rocm-utils rocfft miopen-hip rocblas hipsparse rocrand rccl hipcub rocthrust rocprofiler-dev roctracer-dev @@ -89,10 +87,14 @@ RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/p cd .. && rm -f protobuf-cpp-3.6.1.tar.gz && rm -rf protobuf-3.6.1 # conda -RUN cd /opt && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && chmod +x Miniconda3-latest-Linux-x86_64.sh -RUN mkdir /opt/conda && ./Miniconda3-latest-Linux-x86_64.sh -b -f -p "/opt/conda" && rm -rf Miniconda3-latest-Linux-x86_64.sh +ENV CONDA_FILE=Miniconda3-py37_4.9.2-Linux-x86_64.sh +RUN cd /opt && wget https://repo.anaconda.com/miniconda/${CONDA_FILE} && chmod +x ${CONDA_FILE} +RUN mkdir /opt/conda && ./${CONDA_FILE} -b -f -p "/opt/conda" && rm -rf ${CONDA_FILE} ENV PATH=/opt/conda/bin:${PATH} -RUN conda init bash && conda install -n base jupyter +RUN conda init bash && conda install -n base jupyter jupyterlab + +# install pylint and pre-commit +RUN /opt/conda/bin/pip install pre-commit pylint pytest astroid isort protocol PyGithub # install Paddle requirement RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt -- GitLab