Created by: lkkkk686

`[2020-04-28 17:40:24,064] [ INFO] - Strategy with warmup, linear decay, slanted triangle learning rate, weight decay regularization, /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py:804: UserWarning: There are no operators in the program to be executed. If you pass Program manually, please use fluid.program_guard to ensure the current Program is being used. warnings.warn(error_info) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py:782: UserWarning: The following exception is not an EOF exception. "The following exception is not an EOF exception.") ---------------------------------------------------------------------------EnforceNotMet Traceback (most recent call last) in ----> 1 run_states = cls_task.finetune_and_eval() /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in finetune_and_eval(self) 861 862 def finetune_and_eval(self): --> 863 return self.finetune(do_eval=True) 864 865 def finetune(self, do_eval=False): /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in finetune(self, do_eval) 876 # Start to finetune 877 with self.phase_guard(phase="train"): --> 878 self.init_if_necessary() 879 self._finetune_start_event() 880 run_states = [] /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in init_if_necessary(self) 364 def init_if_necessary(self): 365 if not self.is_checkpoint_loaded: --> 366 if not self.load_checkpoint(): 367 self.exe.run(self._base_startup_program) 368 self.is_checkpoint_loaded = True /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in load_checkpoint(self) 830 self.config.checkpoint_dir, 831 self.exe, --> 832 main_program=self.main_program) 833 834 return is_load_successful /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in main_program(self) 505 def main_program(self): 506 if not self.env.is_inititalized: --> 507 self._build_env() 508 return self.env.main_program 509 /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlehub/finetune/task/base_task.py in _build_env(self) 441 places=self.places) 442 --> 443 self.exe.run(self.env.startup_program) 444 self._build_env_end_event() 445 /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py in run(self, program, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache) 781 warnings.warn( 782 "The following exception is not an EOF exception.") --> 783 six.reraise(*sys.exc_info()) 784 785 def _run_impl(self, program, feed, fetch_list, feed_var_name, /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py in reraise(tp, value, tb) 691 if value.traceback is not tb: 692 raise value.with_traceback(tb) --> 693 raise value 694 finally: 695 value = None /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py in run(self, program, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache) 776 scope=scope, 777 return_numpy=return_numpy, --> 778 use_program_cache=use_program_cache) 779 except Exception as e: 780 if not isinstance(e, core.EOFException): /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py in _run_impl(self, program, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache) 829 scope=scope, 830 return_numpy=return_numpy, --> 831 use_program_cache=use_program_cache) 832 833 program._compile(scope, self.place) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py in _run_program(self, program, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache) 903 if not use_program_cache: 904 self._default_executor.run(program.desc, scope, 0, True, True, --> 905 fetch_var_name) 906 else: 907 self._default_executor.run_prepared_ctx(ctx, scope, False, False, EnforceNotMet:

C++ Call Stacks (More useful to developers):

0 std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) 2 paddle::platform::CUDADeviceContext::CUDADeviceContext(paddle::platform::CUDAPlace) 3 std::_Function_handler<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > (), std::reference_wrapper<std::_Bind_simple<paddle::platform::EmplaceDeviceContext<paddle::platform::CUDADeviceContext, paddle::platform::CUDAPlace>(std::map<paddle::platform::Place, std::shared_future<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > >, std::lesspaddle::platform::Place, std::allocator<std::pair<paddle::platform::Place const, std::shared_future<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > > > > >, paddle::platform::Place)::{lambda()#1 (closed)} ()> > >::_M_invoke(std::_Any_data const&) 4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > >, std::__future_base::_Result_base::_Deleter>, std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > > >::_M_invoke(std::_Any_data const&) 5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) 6 std::__future_base::_Deferred_state<std::_Bind_simple<paddle::platform::EmplaceDeviceContext<paddle::platform::CUDADeviceContext, paddle::platform::CUDAPlace>(std::map<paddle::platform::Place, std::shared_future<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > >, std::lesspaddle::platform::Place, std::allocator<std::pair<paddle::platform::Place const, std::shared_future<std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > > > > >, paddle::platform::Place)::{lambda()#1 (closed)} ()>, std::unique_ptr<paddle::platform::DeviceContext, std::default_deletepaddle::platform::DeviceContext > >::_M_run_deferred() 7 paddle::platform::DeviceContextPool::Get(paddle::platform::Place const&) 8 paddle::framework::GarbageCollector::GarbageCollector(paddle::platform::Place const&, unsigned long) 9 paddle::framework::UnsafeFastGPUGarbageCollector::UnsafeFastGPUGarbageCollector(paddle::platform::CUDAPlace const&, unsigned long) 10 paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) 11 paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocatorstd::string > const&, bool, bool)

Error Message Summary:

Error: An error occurred here. There is no accurate error hint for this error yet. We are continuously in the process of increasing hint for this kind of error check. It would be helpful if you could inform us of how this conversion went by opening a github issue. And we will resolve it with high priority.

• New issue link: https://github.com/PaddlePaddle/Paddle/issues/new
• Recommended issue content: all error stack information: out of memory at (/paddle/paddle/fluid/platform/device_context.cc:221)`