Created by: shuyiliu

Traceback (most recent call last): File "train.py", line 258, in train() File "train.py", line 112, in train exe.run(fluid.default_startup_program()) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/executor.py", line 651, in run use_program_cache=use_program_cache) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/executor.py", line 749, in run exe.run(program.desc, scope, 0, True, True, fetch_var_name) paddle.fluid.core_avx.EnforceNotMet: Invoke operator fill_constant error. Python Callstacks: File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/framework.py", line 1842, in prepend_op attrs=kwargs.get("attrs", None)) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/initializer.py", line 189, in call stop_gradient=True) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/framework.py", line 1625, in create_var kwargs'initializer' File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/layer_helper_base.py", line 383, in set_variable_initializer initializer=initializer) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 317, in add_accumulator var, initializer=Constant(value=float(fill_value))) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 760, in create_accumulators self.add_accumulator(self.velocity_acc_str, p) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 364, in create_optimization_pass [p[0] for p in parameters_and_grads]) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 532, in apply_gradients optimize_ops = self.create_optimization_pass(params_grads) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 562, in apply_optimize optimize_ops = self.apply_gradients(params_grads) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 601, in minimize loss, startup_program=startup_program, params_grads=params_grads) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/dygraph/base.py", line 87, in impl return func(*args, **kwargs) File "/home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/paddle/fluid/wrapped_decorator.py", line 25, in impl return wrapped_func(args, kwargs) File "</home/gpu-server2/anaconda3/envs/CNdetection/lib/python3.7/site-packages/decorator.py:decorator-gen-20>", line 2, in minimize File "train.py", line 102, in train optimizer.minimize(loss) File "train.py", line 258, in train() C++ Callstacks: Enforce failed. Expected allocating <= available, but received allocating:10725104759 > available:10702159616. Insufficient GPU memory to allocation. at [/paddle/paddle/fluid/platform/gpu_info.cc:262] PaddlePaddle Call Stacks: 0 0x7f6b521d2e78p void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 360 1 0x7f6b521d31c7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const, int) + 87 2 0x7f6b54324c66p paddle::platform::GpuMaxChunkSize() + 630 3 0x7f6b542f8f5ap 4 0x7f6b84277827p 5 0x7f6b542f85fdp paddle::memory::legacy::GetGPUBuddyAllocator(int) + 109 6 0x7f6b542f9445p void paddle::memory::legacy::Allocpaddle::platform::CUDAPlace(paddle::platform::CUDAPlace const&, unsigned long) + 37 7 0x7f6b542f9985p paddle::memory::allocation::LegacyAllocator::AllocateImpl(unsigned long) + 421 8 0x7f6b542edaa5p paddle::memory::allocation::AllocatorFacade::Alloc(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, unsigned long) + 181 9 0x7f6b542edc2ap paddle::memory::allocation::AllocatorFacade::AllocShared(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, unsigned long) + 26 10 0x7f6b53ee676cp paddle::memory::AllocShared(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, unsigned long) + 44 11 0x7f6b542bf7f4p paddle::framework::Tensor::mutable_data(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, paddle::framework::proto::VarType_Type, unsigned long) + 148 12 0x7f6b52fe7a2ep paddle::operators::FillConstantKernel::Compute(paddle::framework::ExecutionContext const&) const + 494 13 0x7f6b52feab43p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernelpaddle::platform::float16 >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 14 0x7f6b54261057p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375 15 0x7f6b54261431p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529 16 0x7f6b5425ea2cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332 17 0x7f6b5235d81ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 606 18 0x7f6b523607dfp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocatorstd::string > const&, bool) + 143 19 0x7f6b521c3fddp 20 0x7f6b52205286p 21 0x55e9977a8744p _PyMethodDef_RawFastCallKeywords + 596 22 0x55e9977a8861p _PyCFunction_FastCallKeywords + 33 23 0x55e9978146e8p _PyEval_EvalFrameDefault + 21240 24 0x55e997758539p _PyEval_EvalCodeWithName + 761 25 0x55e9977a7f57p _PyFunction_FastCallKeywords + 903 26 0x55e9978108ccp _PyEval_EvalFrameDefault + 5340 27 0x55e997758539p _PyEval_EvalCodeWithName + 761 28 0x55e9977a7ef5p _PyFunction_FastCallKeywords + 805 29 0x55e99780fa93p _PyEval_EvalFrameDefault + 1699 30 0x55e997758d09p _PyEval_EvalCodeWithName + 2761 31 0x55e9977a7f57p _PyFunction_FastCallKeywords + 903 32 0x55e99780f806p _PyEval_EvalFrameDefault + 1046 33 0x55e997758539p _PyEval_EvalCodeWithName + 761 34 0x55e997759424p PyEval_EvalCodeEx + 68 35 0x55e99775944cp PyEval_EvalCode + 28 36 0x55e99786eb74p 37 0x55e997878eb1p PyRun_FileExFlags + 161 38 0x55e9978790a3p PyRun_SimpleFileExFlags + 451 39 0x55e99787a195p 40 0x55e99787a2bcp _Py_UnixMain + 60 41 0x7f6b83e98b97p __libc_start_main + 231 42 0x55e99781f062p