Created by: huillll

env CUDA_VISIBLE_DEVICES=0 python train.py --skip_test=True --log_period=1000 --save_model_dir=./models_attention --model=attention
-----------  Configuration Arguments -----------
average_window: 0.15
batch_size: 32
eval_period: 15000
init_model: None
log_period: 1000
max_average_window: 12500
min_average_window: 10000
model: attention
parallel: False
profile: False
save_model_dir: ./models_attention
save_model_period: 15000
skip_batch_num: 0
skip_test: 1
test_images: None
test_list: None
total_step: 720000
train_images: None
train_list: None
use_gpu: True
------------------------------------------------
/usr/local/lib/python2.7/dist-packages/paddle/fluid/evaluator.py:71: Warning: The EditDistance is deprecated, because maintain a modified program inside evaluator cause bug easily, please use fluid.metrics.EditDistance instead.
  % (self.__class__.__name__, self.__class__.__name__), Warning)
finish batch shuffle
W0716 12:37:56.041159  6098 device_context.cc:259] Please NOTE: device: 0, CUDA Capability: 61, Driver API Version: 10.1, Runtime API Version: 10.0
W0716 12:37:56.043552  6098 device_context.cc:267] device: 0, cuDNN Version: 7.5.
Traceback (most recent call last):
  File "train.py", line 218, in <module>
    main()
  File "train.py", line 214, in main
    train(args)
  File "train.py", line 148, in train
    results = train_one_batch(data)
  File "train.py", line 109, in train_one_batch
    fetch_list=fetch_vars)
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 650, in run
    use_program_cache=use_program_cache)
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 748, in _run
    exe.run(program.desc, scope, 0, True, True, fetch_var_name)
paddle.fluid.core_avx.EnforceNotMet: Invoke operator edit_distance error.
Python Callstacks: 
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/framework.py", line 1748, in append_op
    attrs=kwargs.get("attrs", None))
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layer_helper.py", line 43, in append_op
    return self.main_program.current_block().append_op(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layers/nn.py", line 5392, in edit_distance
    attrs={"normalized": normalized})
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/evaluator.py", line 261, in __init__
    input=input, label=label, ignored_tokens=ignored_tokens)
  File "/home/luhui/ai-baidu/models/PaddleCV/ocr_recognition/attention_model.py", line 187, in attention_train_net
    input=maxid, label=label_out, ignored_tokens=[sos, eos])
  File "train.py", line 60, in train
    args, data_shape, num_classes)
  File "train.py", line 214, in main
    train(args)
  File "train.py", line 218, in <module>
    main()
C++ Callstacks: 
Reference string 10 is empty. at [/paddle/paddle/fluid/operators/edit_distance_op.cu:92]
PaddlePaddle Call Stacks: 
0       0x7fa9d28407d0p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 352
1       0x7fa9d2840b49p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 137
2       0x7fa9d2ce17b4p paddle::operators::EditDistanceGPUKernel<paddle::platform::CUDAPlace, float>::Compute(paddle::framework::ExecutionContext const&) const + 5476
3       0x7fa9d2ce1bc3p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::EditDistanceGPUKernel<paddle::platform::CUDAPlace, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
4       0x7fa9d48c5887p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375
5       0x7fa9d48c5c61p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
6       0x7fa9d48c325cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332
7       0x7fa9d29cc4bep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 382
8       0x7fa9d29cf55fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocator<std::string> > const&, bool) + 143
9       0x7fa9d2831c5dp
10      0x7fa9d28736f6p
11      0x55ee22475fe5p PyEval_EvalFrameEx + 32869
12      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
13      0x55ee22473c38p PyEval_EvalFrameEx + 23736
14      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
15      0x55ee224735fep PyEval_EvalFrameEx + 22142
16      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
17      0x55ee22473c38p PyEval_EvalFrameEx + 23736
18      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
19      0x55ee22473c38p PyEval_EvalFrameEx + 23736
20      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
21      0x55ee22473c38p PyEval_EvalFrameEx + 23736
22      0x55ee2246bd0ap PyEval_EvalCodeEx + 1754
23      0x55ee2246b629p PyEval_EvalCode + 25
24      0x55ee2249c61fp
25      0x55ee22497322p PyRun_FileExFlags + 130
26      0x55ee2249667dp PyRun_SimpleFileExFlags + 397
27      0x55ee224451abp Py_Main + 1675
28      0x7faa104bbb97p __libc_start_main + 231
29      0x55ee22444a2ap _start + 42