From 258202169666de15f8ec6cac56d1ed0b77e28d55 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Thu, 31 Aug 2023 11:12:42 +0800 Subject: [PATCH] Throw error for NVCC lazy in 1F1B pipeline (#56725) --- python/paddle/distributed/auto_parallel/static/engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index 0e3c316a547..d1261eca247 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -242,6 +242,14 @@ class Engine: elif self._strategy.pipeline.enable: self._acc_steps = self._strategy.pipeline.accumulate_steps + if ( + self._strategy.pipeline.enable + and self._strategy.pipeline.schedule_mode == "1F1B" + ): + assert ( + os.getenv("CUDA_MODULE_LOADING") != "LAZY" + ), "EXP_CUDA_MODULE_LOADING_LAZY not supported in 1F1B pipeline." + self.history = None paddle.framework.set_flags({'FLAGS_new_executor_sequential_run': 1}) -- GitLab