模型训练报错
Created by: chenmingwei00
Traceback (most recent call last): File "/root/.pycharm_helpers/pydev/pydevd.py", line 1664, in main() File "/root/.pycharm_helpers/pydev/pydevd.py", line 1658, in main globals = debugger.run(setup['file'], None, None, is_module) File "/root/.pycharm_helpers/pydev/pydevd.py", line 1068, in run pydev_imports.execfile(file, globals, locals) # execute the script File "/root/.pycharm_helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile exec(compile(contents+"\n", file, 'exec'), glob, loc) File "/root/comprehension/paddle_seq/src/run_classifier.py", line 447, in main(args) File "/root/comprehension/paddle_seq/src/run_classifier.py", line 134, in main use_fp16=args.use_fp16) File "/root/comprehension/paddle_seq/src/optimization.py", line 151, in optimization _, param_grads = optimizer.minimize(loss) File "</root/anaconda3/lib/python3.7/site-packages/decorator.py:decorator-gen-135>", line 2, in minimize File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/wrapped_decorator.py", line 25, in impl return wrapped_func(*args, kwargs) File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/dygraph/base.py", line 87, in impl return func(args, kwargs) File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 594, in minimize no_grad_set=no_grad_set) File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/optimizer.py", line 493, in backward no_grad_set, callbacks) File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/backward.py", line 578, in append_backward append_backward_vars(root_block, fwd_op_num, grad_to_var, grad_info_map) File "/root/anaconda3/lib/python3.7/site-packages/paddle/fluid/backward.py", line 392, in append_backward_vars op_desc.infer_shape(block.desc) paddle.fluid.core_avx.EnforceNotMet: Input(Out@GRAD) shouldn't be null. at [/paddle/paddle/fluid/operators/reshape_op.cc:370] PaddlePaddle Call Stacks: 0 0x7fda2c8e0950p void paddle::platform::EnforceNotMet::Init<char const>(char const, char const, int) + 352 1 0x7fda2c8e0cc9p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const, int) + 137 2 0x7fda2cf3fb20p paddle::operators::Reshape2GradOp::InferShape(paddle::framework::InferShapeContext) const + 800 3 0x7fda2ca52b5ep paddle::framework::OpDesc::InferShape(paddle::framework::BlockDesc const&) const + 862 4 0x7fda2c98c61cp 5 0x7fda2c913286p 6 0x557cf233a744p _PyMethodDef_RawFastCallKeywords + 596 7 0x557cf233a861p _PyCFunction_FastCallKeywords + 33 8 0x557cf23a66e8p _PyEval_EvalFrameDefault + 21240 9 0x557cf2339ccbp _PyFunction_FastCallKeywords + 251 10 0x557cf23a1806p _PyEval_EvalFrameDefault + 1046 11 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 12 0x557cf2339ef5p _PyFunction_FastCallKeywords + 805 13 0x557cf23a1806p _PyEval_EvalFrameDefault + 1046 14 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 15 0x557cf2339f57p _PyFunction_FastCallKeywords + 903 16 0x557cf23a28ccp _PyEval_EvalFrameDefault + 5340 17 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 18 0x557cf22eb635p _PyFunction_FastCallDict + 469 19 0x557cf23a3232p _PyEval_EvalFrameDefault + 7746 20 0x557cf22ea81ap _PyEval_EvalCodeWithName + 1498 21 0x557cf22eb635p _PyFunction_FastCallDict + 469 22 0x557cf23a3232p _PyEval_EvalFrameDefault + 7746 23 0x557cf22ea81ap _PyEval_EvalCodeWithName + 1498 24 0x557cf2339f57p _PyFunction_FastCallKeywords + 903 25 0x557cf23a1806p _PyEval_EvalFrameDefault + 1046 26 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 27 0x557cf2339ef5p _PyFunction_FastCallKeywords + 805 28 0x557cf23a1a93p _PyEval_EvalFrameDefault + 1699 29 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 30 0x557cf2339f57p _PyFunction_FastCallKeywords + 903 31 0x557cf23a28ccp _PyEval_EvalFrameDefault + 5340 32 0x557cf2339ccbp _PyFunction_FastCallKeywords + 251 33 0x557cf23a1806p _PyEval_EvalFrameDefault + 1046 34 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 35 0x557cf22eb424p PyEval_EvalCodeEx + 68 36 0x557cf22eb44cp PyEval_EvalCode + 28 37 0x557cf23b0f8dp 38 0x557cf233a5d9p _PyMethodDef_RawFastCallKeywords + 233 39 0x557cf233a861p _PyCFunction_FastCallKeywords + 33 40 0x557cf23a7526p _PyEval_EvalFrameDefault + 24886 41 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 42 0x557cf2339ef5p _PyFunction_FastCallKeywords + 805 43 0x557cf23a5f29p _PyEval_EvalFrameDefault + 19257 44 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 45 0x557cf2339ef5p _PyFunction_FastCallKeywords + 805 46 0x557cf23a1a93p _PyEval_EvalFrameDefault + 1699 47 0x557cf2339ccbp _PyFunction_FastCallKeywords + 251 48 0x557cf23a1806p _PyEval_EvalFrameDefault + 1046 49 0x557cf22ea539p _PyEval_EvalCodeWithName + 761 50 0x557cf22eb424p PyEval_EvalCodeEx + 68 51 0x557cf22eb44cp PyEval_EvalCode + 28 52 0x557cf2400b74p 53 0x557cf240aeb1p PyRun_FileExFlags + 161 54 0x557cf240b0a3p PyRun_SimpleFileExFlags + 451 55 0x557cf240c195p 56 0x557cf240c2bcp _Py_UnixMain + 60 57 0x7fda6be5b830p __libc_start_main + 240 58 0x557cf23b1062p