image_classification mobilenet.py training halfway throw exception
Created by: seiriosPlus
exception:
Pass 0, batch 150, loss 4.19872951508, acc 0.0500000007451
Pass 0, batch 151, loss 4.3166437149, acc 0.0750000029802
Pass 0, batch 152, loss 4.25785493851, acc 0.10000000149
Pass 0, batch 153, loss 4.23684549332, acc 0.0689655169845
Traceback (most recent call last):
File "mobilenet.py", line 224, in <module>
train(learning_rate=0.005, batch_size=40, num_passes=300)
File "mobilenet.py", line 212, in train
fetch_list=[avg_cost, b_acc_var, b_size_var])
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 349, in run
self.executor.run(program_cache.desc, scope, 0, True, True)
paddle.fluid.core.EnforceNotMet: Cannot find fetch variable in scope, fetch_var_name is mean_0.tmp_0 at [/paddle/paddle/fluid/operators/fetch_op.cc:36]
PaddlePaddle Call Stacks:
0 0x7f67888c8026p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1 0x7f6788d530c9p paddle::operators::FetchOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 2105
2 0x7f67893d5788p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 88
3 0x7f67889582aep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool) + 1086
4 0x7f67889592d0p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 64
5 0x7f67888dd37bp void pybind11::cpp_function::initialize<pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}, void, paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}&&, void (*)(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call) + 555
6 0x7f67888d6dd4p pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 2596
7 0x4c37edp PyEval_EvalFrameEx + 31165
8 0x4b9ab6p PyEval_EvalCodeEx + 774
9 0x4c16e7p PyEval_EvalFrameEx + 22711
10 0x4b9ab6p PyEval_EvalCodeEx + 774
11 0x4c16e7p PyEval_EvalFrameEx + 22711
12 0x4b9ab6p PyEval_EvalCodeEx + 774
13 0x4eb30fp
14 0x4e5422p PyRun_FileExFlags + 130
15 0x4e3cd6p PyRun_SimpleFileExFlags + 390
16 0x493ae2p Py_Main + 1554
17 0x7f67d9988830p __libc_start_main + 240
18 0x4933e9p _start + 41
env: docker paddlepadle version: lastest-gpu