Created by: freekoy

root@63a11f01a6d1:/work/paddlepaddle/icnet# python train.py --batch_size=4 --use_gpu=True --checkpoint_path="./chkpnt/"
-----------  Configuration Arguments -----------
batch_size: 4
checkpoint_path: ./chkpnt/
init_model: None
random_mirror: True
random_scaling: True
use_gpu: 1
------------------------------------------------
W1225 04:23:18.551097    90 device_context.cc:203] Please NOTE: device: 0, CUDA Capability: 75, Driver Version: 10.0, Runtime Version: 9.0
W1225 04:23:18.551133    90 device_context.cc:210] device: 0, cuDNN Version: 7.0.
Iter[100]; train loss: 0.655; sub4_loss: 0.835; sub24_loss: 0.529; sub124_loss: 0.310
kpis	train_cost	0.654622
Saved checkpoint: ./chkpnt//100
kpis	train_duration	189.435656
root@63a11f01a6d1:/work/paddlepaddle/icnet# python eval.py --model_path="./chkpnt/100/" --use_gpu=True
-----------  Configuration Arguments -----------
model_path: ./chkpnt/100/
use_gpu: 1
------------------------------------------------
W1225 04:27:08.380987   130 device_context.cc:203] Please NOTE: device: 0, CUDA Capability: 75, Driver Version: 10.0, Runtime Version: 9.0
W1225 04:27:08.381017   130 device_context.cc:210] device: 0, cuDNN Version: 7.0.
loaded model from: ./chkpnt/100/
Traceback (most recent call last):
  File "eval.py", line 96, in <module>
    main()
  File "eval.py", line 92, in main
    eval(args)
  File "eval.py", line 80, in eval
    fetch_list=fetch_vars)
  File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/executor.py", line 472, in run
    self.executor.run(program.desc, scope, 0, True, True)
paddle.fluid.core.EnforceNotMet: Enforce failed. Expected x_dims[i + axis] == y_dims[i], but received x_dims[i + axis]:128 != y_dims[i]:213.
Broadcast dimension mismatch. at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:63]
PaddlePaddle Call Stacks: 
0       0x7f48636d7e96p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1       0x7f48639b0334p paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*) + 916
2       0x7f4864a52d23p void paddle::operators::ElementwiseComputeEx<paddle::operators::AddFunctor<float>, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor<float>, paddle::framework::Tensor*) + 499
3       0x7f4864a534a3p void paddle::operators::default_elementwise_add<paddle::platform::CUDADeviceContext, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) + 115
4       0x7f4864a53638p paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 344
5       0x7f4864a536b3p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, int>, paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ElementwiseAddKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
6       0x7f4864edcafcp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 492
7       0x7f4864ed8c3fp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 463
8       0x7f48637acb63p paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 227
9       0x7f48637ad590p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 128
10      0x7f48636c3c3bp
11      0x7f48636ff81ep
12            0x4e9ba7p PyCFunction_Call + 119
13            0x53c6d5p PyEval_EvalFrameEx + 23029
14            0x540199p
15            0x53bd92p PyEval_EvalFrameEx + 20658
16            0x53b7e4p PyEval_EvalFrameEx + 19204
17            0x53b7e4p PyEval_EvalFrameEx + 19204
18            0x540199p
19            0x540e4fp PyEval_EvalCode + 31
20            0x60c272p
21            0x60e71ap PyRun_FileExFlags + 154
22            0x60ef0cp PyRun_SimpleFileExFlags + 444
23            0x63fb26p Py_Main + 1110
24            0x4cfeb1p main + 225
25      0x7f48b2fde830p __libc_start_main + 240
26            0x5d6049p _start + 41