PaddleDetection Retinanet 评估失败 Cannot find fetched variable(im_id)
Created by: ellinyang
软硬件环境:GPU P4, CUDA7.0; CUDNN9.0 代码版本 paddleDetection( commit_id d7fcd54d) paddle 1.6.2 背景:将异步读取的pyreader改为同步feeder读取
`def eval_run_feeder(exe, compile_program, reader, feeder, keys, values, cls):
"""
Run evaluation program, return program outputs.
"""
iter_id = 0
results = []
if len(cls) != 0:
values = []
for i in range(len(cls)):
_, accum_map = cls[i].get_map_var()
cls[i].reset(exe)
values.append(accum_map)
print("@@@@@@values", values)
images_num = 0
start_time = time.time()
has_bbox = 'bbox' in keys
for it, data in enumerate(reader()):
print("data",data)
outs = exe.run(compile_program,
feed=feeder.feed(data),
fetch_list=values,
return_numpy=False)
res = {
k: (np.array(v), v.recursive_sequence_lengths())
for k, v in zip(keys, outs)
}
results.append(res)
if iter_id % 100 == 0:
logger.info('Test iter {}'.format(iter_id))
iter_id += 1
images_num += len(res['bbox'][1][0]) if has_bbox else 1
end_time = time.time()
fps = images_num / (end_time - start_time)
if has_bbox:
logger.info('Total number of images: {}, inference time: {} fps.'.
format(images_num, fps))
else:
logger.info('Total iteration: {}, inference time: {} batch/s.'.format(
images_num, fps))
return results
`
报错如下:
`1225 12:21:15.713791 20964 parallel_executor.cc:421] The number of CUDAPlace, which is used in ParallelExecutor, is 1. And the Program will be copied 1 copies
I1225 12:21:15.724485 20964 build_strategy.cc:363] SeqOnlyAllReduceOps:0, num_trainers:1
I1225 12:21:15.738996 20964 parallel_executor.cc:368] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py:779: UserWarning: The following exception is not an EOF exception.
"The following exception is not an EOF exception.")
Traceback (most recent call last):
File "tools/train_feeder.py", line 374, in <module>
main()
File "tools/train_feeder.py", line 299, in main
eval_keys, eval_values, eval_cls)
File "tools/train_feeder.py", line 79, in eval_run_feeder
return_numpy=False)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 780, in run
six.reraise(*sys.exc_info())
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 775, in run
use_program_cache=use_program_cache)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 834, in _run_impl
return_numpy=return_numpy)
File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 674, in _run_parallel
tensors = exe.run(fetch_var_names)._move_to_list()
paddle.fluid.core_avx.EnforceNotMet:
--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0 std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int)
2 paddle::framework::details::FastThreadedSSAGraphExecutor::InsertFetchOps(std::vector<std::string, std::allocator<std::string> > const&, std::vector<paddle::framework::LoDTensor, std::allocator<paddle::framework::LoDTensor> >*, std::unordered_map<std::string, std::vector<paddle::framework::details::VarHandleBase*, std::allocator<paddle::framework::details::VarHandleBase*> >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<paddle::framework::details::VarHandleBase*, std::allocator<paddle::framework::details::VarHandleBase*> > > > >*, std::unordered_map<paddle::framework::details::OpHandleBase*, std::atomic<int>, std::hash<paddle::framework::details::OpHandleBase*>, std::equal_to<paddle::framework::details::OpHandleBase*>, std::allocator<std::pair<paddle::framework::details::OpHandleBase* const, std::atomic<int> > > >*, std::vector<paddle::framework::details::OpHandleBase*, std::allocator<paddle::framework::details::OpHandleBase*> >*, std::vector<paddle::framework::details::OpHandleBase*, std::allocator<paddle::framework::details::OpHandleBase*> >*)
3 paddle::framework::details::FastThreadedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&)
4 paddle::framework::details::ScopeBufferedMonitor::Apply(std::function<void ()> const&, bool)
5 paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&)
6 paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&)
----------------------
Error Message Summary:
----------------------
Error: Cannot find fetched variable(im_id).(Perhaps the main_program is not set to ParallelExecutor) at (/paddle/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc:145)`