From 23ad0cc48016d7fefa7568a4769f44cafe8f5a5e Mon Sep 17 00:00:00 2001 From: Allen Guo Date: Fri, 29 Jul 2022 11:36:37 +0800 Subject: [PATCH] update to sdk2.6.0 (#44673) --- .../fluid/platform/device/ipu/ipu_compiler.cc | 4 +- .../fluid/platform/device/ipu/ipu_executor.cc | 16 +++----- .../fluid/platform/device/ipu/ipu_executor.h | 2 +- .../fluid/platform/device/ipu/ipu_strategy.cc | 2 - .../device/ipu/supported_ops_autogen.h | 6 +-- .../ipu/custom_ops/leaky_relu_ipu.cc | 1 + tools/dockerfile/Dockerfile.ipu | 38 ++++++++----------- 7 files changed, 28 insertions(+), 41 deletions(-) diff --git a/paddle/fluid/platform/device/ipu/ipu_compiler.cc b/paddle/fluid/platform/device/ipu/ipu_compiler.cc index 2bebdd51280..22b8b67fa64 100644 --- a/paddle/fluid/platform/device/ipu/ipu_compiler.cc +++ b/paddle/fluid/platform/device/ipu/ipu_compiler.cc @@ -18,13 +18,13 @@ #include #include #include - -#include "paddle/utils/blank.h" +#include #include "paddle/fluid/framework/ir/graph_helper.h" #include "paddle/fluid/platform/device/ipu/ipu_names.h" #include "paddle/fluid/platform/device/ipu/ipu_strategy.h" #include "paddle/fluid/platform/device/ipu/ipu_utils.h" +#include "paddle/utils/blank.h" namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/device/ipu/ipu_executor.cc b/paddle/fluid/platform/device/ipu/ipu_executor.cc index cf051f97820..29f82b07473 100644 --- a/paddle/fluid/platform/device/ipu/ipu_executor.cc +++ b/paddle/fluid/platform/device/ipu/ipu_executor.cc @@ -14,6 +14,9 @@ limitations under the License. */ #include "paddle/fluid/platform/device/ipu/ipu_executor.h" +#include +#include + #include "paddle/fluid/framework/operator.h" #include "paddle/fluid/platform/device/ipu/ipu_compiler.h" #include "paddle/fluid/platform/device/ipu/ipu_names.h" @@ -204,9 +207,7 @@ void Executor::Run(const std::vector &inputs, } VLOG(10) << "Prepared inputs/anchors"; - if (ipu_strategy_->is_training && compiler_resources_->with_lr_sched && - !(ipu_strategy_->popart_options.createImplicitPipeliningFwdOnlyProgram && - ipu_strategy_->runtime_options.enable_eval)) { + if (ipu_strategy_->is_training && compiler_resources_->with_lr_sched) { popart::Optimizer *optimizer; if (ipu_strategy_->runtime_options.enable_eval) { VLOG(10) << "Switch optimizer to eval mode"; @@ -229,12 +230,7 @@ void Executor::Run(const std::vector &inputs, popart::StepIO stepio(popart_inputs, popart_anchors); VLOG(10) << "Running..."; - if (ipu_strategy_->popart_options.createImplicitPipeliningFwdOnlyProgram && - ipu_strategy_->runtime_options.enable_eval) { - session_->run("implicitPipeliningFwdOnly", stepio); - } else { - session_->run(stepio); - } + session_->run(stepio); VLOG(10) << "Running...done"; } @@ -286,7 +282,7 @@ void Executor::AcquireDevice() { VLOG(10) << "Create distribution device..."; auto ipus_per_replica = ipu_strategy_->num_ipus / ipu_strategy_->popart_options.replicatedGraphCount; - auto device_id = popdist_get_device(ipus_per_replica); + auto device_id = popdist::getDeviceId(ipus_per_replica); device_ = popart::DeviceManager::createDeviceManager().acquireDeviceById( device_id); PADDLE_ENFORCE_NOT_NULL( diff --git a/paddle/fluid/platform/device/ipu/ipu_executor.h b/paddle/fluid/platform/device/ipu/ipu_executor.h index 1a46ebc69b1..eab1bd9d36e 100644 --- a/paddle/fluid/platform/device/ipu/ipu_executor.h +++ b/paddle/fluid/platform/device/ipu/ipu_executor.h @@ -19,8 +19,8 @@ limitations under the License. */ #include #include #include +#include #include -#include #include "paddle/fluid/platform/device/ipu/ipu_utils.h" diff --git a/paddle/fluid/platform/device/ipu/ipu_strategy.cc b/paddle/fluid/platform/device/ipu/ipu_strategy.cc index d7965010696..56794d851ed 100644 --- a/paddle/fluid/platform/device/ipu/ipu_strategy.cc +++ b/paddle/fluid/platform/device/ipu/ipu_strategy.cc @@ -302,8 +302,6 @@ IpuStrategy::IpuStrategy() { ADD_POPART_BOOL_OPTION_ALIAS( schedule_non_weight_update_gradient_consumers_early, scheduleNonWeightUpdateGradientConsumersEarly); - ADD_POPART_BOOL_OPTION_ALIAS(create_implicit_pipelining_fwd_only_program, - createImplicitPipeliningFwdOnlyProgram); ADD_POPART_DOUBLE_OPTION_ALIAS(outline_sequence_break_cost, outlineSequenceBreakCost); diff --git a/paddle/fluid/platform/device/ipu/supported_ops_autogen.h b/paddle/fluid/platform/device/ipu/supported_ops_autogen.h index 017ed648930..96db5affe38 100644 --- a/paddle/fluid/platform/device/ipu/supported_ops_autogen.h +++ b/paddle/fluid/platform/device/ipu/supported_ops_autogen.h @@ -33,13 +33,12 @@ OP_DECL(popart_dynamicupdate_v2, aiGraphcoreOpset.dynamicupdate, ARG(INT_VEC,axe OP_DECL(popart_dynamiczero_v2, aiGraphcoreOpset.dynamiczero, ARG(INT_VEC,axes) ARG(INT_VEC,sizes) ) // NOLINT OP_DECL(popart_dynamicadd_v2, aiGraphcoreOpset.dynamicadd, ARG(INT_VEC,axes) ARG(INT_VEC,sizes) ) // NOLINT OP_DECL(popart_sequenceslice_v2, aiGraphcoreOpset.sequenceslice, ARG(INT,zeroUnused) ) // NOLINT -OP_DECL(popart_replicatedallreduce_v2, aiGraphcoreOpset.replicatedallreduce, OPT_ARG(INT_VEC,commGroup) ) // NOLINT OP_DECL(popart_l1loss_v2, aiGraphcoreOpset.l1loss, ARG(FLOAT,lambda) SIG_ARG(INT32,popart::ReductionType,reduction) ) // NOLINT OP_DECL(popart_nllloss_v2, aiGraphcoreOpset.nllloss, SIG_ARG(INT32,popart::ReductionType,reduction) OPT_ARG(INT32,ignoreIndex) ARG(BOOL,inputIsLogProbability) ) // NOLINT OP_DECL(popart_identityloss_v2, aiGraphcoreOpset.identityloss, SIG_ARG(INT32,popart::ReductionType,reduction) ) // NOLINT OP_DECL(popart_tensorremap_v2, aiGraphcoreOpset.tensorremap, ARG(INT,remap_type) ) // NOLINT -OP_DECL(popart_ctcloss_v2, aiGraphcoreOpset.ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ) // NOLINT -OP_DECL(popart__ctcloss_v2, aiGraphcoreOpset._ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ) // NOLINT +OP_DECL(popart_ctcloss_v2, aiGraphcoreOpset.ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ARG(BOOL,zeroInfinity) ) // NOLINT +OP_DECL(popart__ctcloss_v2, aiGraphcoreOpset._ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ARG(BOOL,zeroInfinity) ) // NOLINT OP_DECL(popart_ctcbeamsearchdecoder_v2, aiGraphcoreOpset.ctcbeamsearchdecoder, ARG(INT,blank) ARG(INT,beamWidth) ARG(INT,topPaths) ) // NOLINT OP_DECL(popart_shapeddropout_v2, aiGraphcoreOpset.shapeddropout, ARG(INT_VEC,shape) ARG(FLOAT,ratio) ) // NOLINT OP_DECL(popart_atan2_v2, aiGraphcoreOpset.atan2, NONE) // NOLINT @@ -48,6 +47,7 @@ OP_DECL(popart_log1p_v2, aiGraphcoreOpset.log1p, NONE) // NOLINT OP_DECL(popart_fmod_v2, aiGraphcoreOpset.fmod, NONE) // NOLINT OP_DECL(popart_remainder_v2, aiGraphcoreOpset.remainder, NONE) // NOLINT OP_DECL(popart_reverse_v2, aiGraphcoreOpset.reverse, ARG(INT_VEC,dimensions) ) // NOLINT +OP_DECL(popart_slice_v2, aiGraphcoreOpset.slice, ARG(INT_VEC,ends) ARG(INT_VEC,starts) ARG(INT_VEC,axes) ) // NOLINT OP_DECL(popart_bitwisenot_v2, aiGraphcoreOpset.bitwisenot, NONE) // NOLINT OP_DECL(popart_bitwiseand_v2, aiGraphcoreOpset.bitwiseand, NONE) // NOLINT OP_DECL(popart_bitwiseor_v2, aiGraphcoreOpset.bitwiseor, NONE) // NOLINT diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/leaky_relu_ipu.cc b/python/paddle/fluid/tests/unittests/ipu/custom_ops/leaky_relu_ipu.cc index 1fea75b3b5a..c7d2c50acbd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/leaky_relu_ipu.cc +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/leaky_relu_ipu.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include diff --git a/tools/dockerfile/Dockerfile.ipu b/tools/dockerfile/Dockerfile.ipu index 8f1948de8a4..2a3bfd332ba 100644 --- a/tools/dockerfile/Dockerfile.ipu +++ b/tools/dockerfile/Dockerfile.ipu @@ -6,7 +6,7 @@ # run a container # docker run --ulimit memlock=-1:-1 --net=host --cap-add=IPC_LOCK --device=/dev/infiniband/ --ipc=host --rm -it paddlepaddle/paddle:latest-dev-ipu bash -FROM graphcore/poplar:2.5.1 +FROM graphcore/poplar:2.6.0 MAINTAINER PaddlePaddle Authors # ENV variables @@ -14,29 +14,21 @@ ARG WITH_AVX ENV HOME /root ENV WITH_AVX=${WITH_AVX:-ON} -# Add bash enhancements -COPY paddle/scripts/docker/root/ /root/ +# install pkgs RUN apt-get update && apt-get install -y apt-utils RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip make libgcc-s1 sudo openssh-server \ - coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev \ - bison graphviz libjpeg-dev zlib1g zlib1g-dev automake locales swig net-tools libtool module-init-tools numactl libnuma-dev \ - openssl libffi-dev pciutils libblas-dev gfortran libblas3 liblapack-dev liblapack3 default-jre screen tmux gdb lldb gcc g++ + coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev \ + bison graphviz libjpeg-dev zlib1g zlib1g-dev automake locales swig net-tools libtool module-init-tools numactl libnuma-dev \ + openssl libffi-dev pciutils libblas-dev gfortran libblas3 liblapack-dev liblapack3 default-jre screen tmux gdb lldb gcc g++ RUN apt-get update && apt-get install -y rdma-core librdmacm1 -RUN apt-get update && apt-get install libspdlog-dev - -# Downgrade gcc&&g++ -WORKDIR /usr/bin -COPY tools/dockerfile/build_scripts /build_scripts -RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts -RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ -RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc -RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ -RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc -RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ -ENV PATH=/usr/local/gcc-8.2/bin:$PATH + +# install g++-8 +RUN apt install g++-8 gcc-8 -y +RUN ln -sf /usr/bin/gcc-8 /usr/bin/gcc +RUN ln -sf /usr/bin/g++-8 /usr/bin/g++ # install cmake WORKDIR /home @@ -50,6 +42,7 @@ RUN cd /opt && wget -q https://repo.anaconda.com/miniconda/${CONDA_FILE} && chmo RUN mkdir /opt/conda && ./${CONDA_FILE} -b -f -p "/opt/conda" && rm -rf ${CONDA_FILE} ENV PATH=/opt/conda/bin:${PATH} RUN conda init bash && conda install -n base jupyter jupyterlab +RUN conda install -n base spdlog==1.8.0 -y # Install Go and glide RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ @@ -93,10 +86,9 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ make -j8 && make install && \ ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache -# clang-form 3.8.0 -RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \ - cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz +# # clang-form 3.8.0 +# RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ +# tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \ +# cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz RUN apt-get clean -y -EXPOSE 22 -- GitLab