diff --git a/paddle/fluid/platform/device/ipu/ipu_backend.cc b/paddle/fluid/platform/device/ipu/ipu_backend.cc index 0871624a5d749fe5042d4b4239b42ab4f25f7967..9e960a99123c07778c511e2ae4bf78072553bd40 100644 --- a/paddle/fluid/platform/device/ipu/ipu_backend.cc +++ b/paddle/fluid/platform/device/ipu/ipu_backend.cc @@ -74,11 +74,7 @@ void IpuBackend::WeightsToHost() { executor_->WeightsToHost(); } void IpuBackend::Detach() { executor_->Detach(); } -void IpuBackend::Reset() { - executor_->Detach(); - compiler_.reset(); - executor_.reset(); -} +void IpuBackend::Reset() { executor_->Reset(); } void IpuBackend::SetScope(const framework::Scope& scope) { scope_ = &scope; diff --git a/paddle/fluid/platform/device/ipu/ipu_executor.cc b/paddle/fluid/platform/device/ipu/ipu_executor.cc index 4f15ecf3babf289f51b9529b2d767ba54bf2ad0e..d490334ee33f52145b47e1e025ed29d9a0632f68 100644 --- a/paddle/fluid/platform/device/ipu/ipu_executor.cc +++ b/paddle/fluid/platform/device/ipu/ipu_executor.cc @@ -88,11 +88,7 @@ class PdIArray final : public popart::IArray { } // namespace -Executor::~Executor() { - Detach(); - session_.reset(); - executor_resources_.reset(); -} +Executor::~Executor() { Reset(); } void Executor::Prepare(const std::string &proto) { VLOG(10) << "enter Executor::Prepare"; @@ -299,6 +295,12 @@ void Executor::Detach() { } } +void Executor::Reset() { + Detach(); + session_.reset(); + executor_resources_.reset(); +} + void Executor::SetWeightsIO() { auto opt_type = compiler_resources_->optimizer_type; VLOG(10) << "SetWeightsIO for " << opt_type; diff --git a/paddle/fluid/platform/device/ipu/ipu_executor.h b/paddle/fluid/platform/device/ipu/ipu_executor.h index 70c9477e69babe9ef05db388528db1d239eceaf1..1a46ebc69b19754ddf2f44394998cbea11ff6528 100644 --- a/paddle/fluid/platform/device/ipu/ipu_executor.h +++ b/paddle/fluid/platform/device/ipu/ipu_executor.h @@ -63,6 +63,9 @@ class Executor { // Detach IPU void Detach(); + // Reset session + void Reset(); + // Scope void SetScope(const Scope *scope) { scope_ = scope; } diff --git a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py index 2583d9409a0a78d58c67398f2f677c1dad7bfc69..ad11083b677736c6731fdaf919460926b3e9f3e9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py @@ -128,6 +128,11 @@ class IPUOpTest(IPUTest): cls.fetch_list: List[str] = None cls.output_dict: Optional[Dict] = {} + def tearDown(self): + # Manual reset when using ipumodel + if self.use_ipumodel(): + paddle.framework.core.IpuBackend.get_instance().reset() + @property def fp16_enabled(self): return True diff --git a/tools/dockerfile/Dockerfile.ipu b/tools/dockerfile/Dockerfile.ipu index ee2d984035624888e3afa67a0f497926ea247ea2..8f1948de8a4dc50f6baa194ad2a7150191008c99 100644 --- a/tools/dockerfile/Dockerfile.ipu +++ b/tools/dockerfile/Dockerfile.ipu @@ -6,7 +6,7 @@ # run a container # docker run --ulimit memlock=-1:-1 --net=host --cap-add=IPC_LOCK --device=/dev/infiniband/ --ipc=host --rm -it paddlepaddle/paddle:latest-dev-ipu bash -FROM graphcore/poplar-extbaidu:2.5.0-ubuntu-18.04-20220407 +FROM graphcore/poplar:2.5.1 MAINTAINER PaddlePaddle Authors # ENV variables