From cc443675e946dfc7ebea7ee3dd10bc7f54c2c803 Mon Sep 17 00:00:00 2001 From: zhongpu <2013000149@qq.com> Date: Tue, 3 Sep 2019 19:24:50 +0800 Subject: [PATCH] modify paddle_build.sh for Paddle python3 runtime image generation, test=develop (#19218) * modify paddle_build.sh for Paddle python3 version image generation, test=develop * modify paddle_build.sh for Paddle python3 image generation, test=develop --- paddle/scripts/paddle_build.sh | 140 +++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 40 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 039b827c04..477dfe4c35 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -725,7 +725,7 @@ function gen_dockerfile() { CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1).$(echo $CUDA_VERSION | cut -d '.' -f 2)" CUDNN_MAJOR=$(echo $CUDNN_VERSION | cut -d '.' -f 1) if [[ ${WITH_GPU} == "ON" ]]; then - BASE_IMAGE="nvidia/cuda:${CUDA_MAJOR}-cudnn${CUDNN_MAJOR}-runtime-ubuntu16.04" + BASE_IMAGE="nvidia/cuda:${CUDA_MAJOR}-cudnn${CUDNN_MAJOR}-devel-ubuntu16.04" else BASE_IMAGE="ubuntu:16.04" fi @@ -744,7 +744,84 @@ function gen_dockerfile() { Generate ${PADDLE_ROOT}/build/Dockerfile ... ======================================== EOF + + ref_CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1)" + if [[ ${WITH_GPU} == "ON" ]]; then + ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR} + else + ref_gpu=cpu + fi + if [[ ${WITH_GPU} == "ON" ]]; then + install_gpu="_gpu" + else + install_gpu="" + fi + if [[ ${WITH_MKL} == "ON" ]]; then + ref_mkl=mkl + else + ref_mkl=openblas + fi + ref_web=https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl} + + ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl + ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl + ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl + ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + + ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl + ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + + if [[ ${PADDLE_BRANCH} != "latest" && ${WITH_MKL} == "ON" && ${WITH_GPU} == "ON" ]]; then + ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl + ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl + ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl + ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl + ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl + ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl + ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl + fi + + #ref_paddle2_mv1="" + #ref_paddle2_mv2="" + ref_paddle35_mv1="" + ref_paddle35_mv2="" + ref_paddle36_mv1="" + ref_paddle36_mv2="" + #ref_paddle37_mv1="" + #ref_paddle37_mv2="" + if [[ ${PADDLE_BRANCH} == "latest" && ${WITH_GPU} == "ON" ]]; then + #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl + ref_paddle35_whl=paddlepaddle_gpu-1.5.1-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle_gpu-1.5.1-cp36-cp36m-linux_x86_64.whl + #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl + #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&" + #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2" + ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&" + ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}" + ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&" + ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}" + #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&" + #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37" + fi + if [[ ${PADDLE_BRANCH} == "latest" && ${WITH_GPU} != "ON" ]]; then + #ref_paddle2_whl=paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl + ref_paddle35_whl=paddlepaddle-1.5.1-cp35-cp35m-linux_x86_64.whl + ref_paddle36_whl=paddlepaddle-1.5.1-cp36-cp36m-linux_x86_64.whl + #ref_paddle37_whl=paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl + #ref_paddle2_mv1="mv ref_paddle2 paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl &&" + #ref_paddle2_mv2="&& mv paddlepaddle_gpu-1.5.1-cp27-cp27mu-linux_x86_64.whl ref_paddle2" + ref_paddle35_mv1="mv ${ref_paddle35} ${ref_paddle35_whl} &&" + ref_paddle35_mv2="&& mv ${ref_paddle35_whl} ${ref_paddle35}" + ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&" + ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}" + #ref_paddle37_mv1="mv ref_paddle37 paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl &&" + #ref_paddle37_mv2="&& mv paddlepaddle_gpu-1.5.1-cp37-cp37m-linux_x86_64.whl ref_paddle37" + fi + cat > ${PADDLE_ROOT}/build/Dockerfile < @@ -752,32 +829,31 @@ EOF EOF if [[ ${WITH_GPU} == "ON" ]]; then - NCCL_DEPS="apt-get install -y --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true" + NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.4.7-1+cuda${CUDA_MAJOR} libnccl-dev=2.4.7-1+cuda${CUDA_MAJOR} || true" else NCCL_DEPS="true" fi + if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then + NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0" + fi + PADDLE_VERSION="paddle version" CMD='"paddle", "version"' - - if [ "$1" == "cp35-cp35m" ]; then - cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \ - pip3.6 install opencv-python && pip3.6 install /*.whl; apt-get install -f -y && \ + make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.6.0.tgz + RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ + pip3.6 install opencv-python && wget ${ref_web}/${ref_paddle36} && ${ref_paddle36_mv1} pip3.6 install ${ref_paddle36_whl} ${ref_paddle36_mv2}; apt-get install -f -y && \ apt-get clean -y && \ - rm -f /*.whl && \ - ${PADDLE_VERSION} && \ + rm -f ${ref_paddle36} && \ ldconfig - ${DOCKERFILE_CUDNN_DSO} - ${DOCKERFILE_CUBLAS_DSO} - ${DOCKERFILE_GPU_ENV} EOF - elif [ "$1" == "cp37-cp37m" ]; then - cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null - RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && \ - pip3.7 install opencv-python && pip3.7 install /*.whl; apt-get install -f -y && \ + make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.7.0.tgz + RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ + pip3.7 install opencv-python && wget ${ref_web}/${ref_paddle37} && pip3.7 install ${ref_paddle37_whl}; apt-get install -f -y && \ apt-get clean -y && \ - rm -f /*.whl && \ - ${PADDLE_VERSION} && \ + rm -f ${ref_paddle37} && \ ldconfig - ${DOCKERFILE_CUDNN_DSO} - ${DOCKERFILE_CUBLAS_DSO} - ${DOCKERFILE_GPU_ENV} EOF - else - cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/Dockerfile <