icnet 多卡实现报错
Created by: ccmeteorljh
paddle version:1.3.0 错误如下：
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
/paddle/paddle/fluid/operators/math/cross_entropy.cu:40 Assertion `label[i] >= 0 && label[i] < D || label[i] == ignore_index` failed.
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what(): unspecified launch failure at [/paddle/paddle/fluid/framework/details/op_handle_base.cc:37]
PaddlePaddle Call Stacks:
0 0x7f61bf8398f5p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 357
1 0x7f61bf839c79p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 137
2 0x7f61c10b7db1p paddle::framework::details::OpHandleBase::~OpHandleBase() + 337
3 0x7f61c10a5c61p paddle::framework::details::FetchOpHandle::~FetchOpHandle() + 17
4 0x7f61bf8ea979p paddle::framework::ir::Node::~Node() + 73
5 0x7f61bf8eab21p paddle::framework::ir::Node::~Node() + 17
6 0x7f61c10a779bp paddle::framework::details::ClearFetchOp(paddle::framework::ir::Graph*, std::vector<paddle::framework::details::FetchOpHandle*, std::allocator<paddle::framework::details::FetchOpHandle*> >*) + 1995
7 0x7f61c1053024p paddle::framework::details::ThreadedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&) + 4660
8 0x7f61c104978ap paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&) + 394
9 0x7f61bf977c92p paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&, std::string const&) + 562
10 0x7f61bf82a28ep
11 0x7f61bf8651fep
12 0x4c37edp PyEval_EvalFrameEx + 31165
13 0x4b9ab6p PyEval_EvalCodeEx + 774
14 0x4c16e7p PyEval_EvalFrameEx + 22711
15 0x4b9ab6p PyEval_EvalCodeEx + 774
16 0x4c16e7p PyEval_EvalFrameEx + 22711
17 0x4b9ab6p PyEval_EvalCodeEx + 774
18 0x4d54b9p
19 0x4a577ep PyObject_Call + 62
20 0x4bed3dp PyEval_EvalFrameEx + 12045
21 0x4c136fp PyEval_EvalFrameEx + 21823
22 0x4b9ab6p PyEval_EvalCodeEx + 774
23 0x4d55f3p
24 0x4a577ep PyObject_Call + 62
25 0x4bed3dp PyEval_EvalFrameEx + 12045
26 0x4b9ab6p PyEval_EvalCodeEx + 774
27 0x4d54b9p
28 0x4eebeep
29 0x4a577ep PyObject_Call + 62
30 0x548253p
31 0x4c15bfp PyEval_EvalFrameEx + 22415
32 0x4c136fp PyEval_EvalFrameEx + 21823
33 0x4b9ab6p PyEval_EvalCodeEx + 774
34 0x4d55f3p
35 0x4a577ep PyObject_Call + 62
36 0x4bed3dp PyEval_EvalFrameEx + 12045
37 0x4b9ab6p PyEval_EvalCodeEx + 774
38 0x4d54b9p
39 0x4eebeep
40 0x4a577ep PyObject_Call + 62
41 0x548253p
42 0x4c15bfp PyEval_EvalFrameEx + 22415
43 0x4b9ab6p PyEval_EvalCodeEx + 774
44 0x4d55f3p
45 0x4a577ep PyObject_Call + 62
46 0x4bed3dp PyEval_EvalFrameEx + 12045
47 0x4b9ab6p PyEval_EvalCodeEx + 774
48 0x4d54b9p
49 0x4eebeep
50 0x4a577ep PyObject_Call + 62
51 0x548253p
52 0x4c15bfp PyEval_EvalFrameEx + 22415
53 0x4b9ab6p PyEval_EvalCodeEx + 774
54 0x4d55f3p
55 0x4a577ep PyObject_Call + 62
56 0x4bed3dp PyEval_EvalFrameEx + 12045
57 0x4b9ab6p PyEval_EvalCodeEx + 774
58 0x4d54b9p
59 0x4eebeep
60 0x4a577ep PyObject_Call + 62
61 0x548253p
62 0x4c15bfp PyEval_EvalFrameEx + 22415
63 0x4b9ab6p PyEval_EvalCodeEx + 774
64 0x4d55f3p
65 0x4a577ep PyObject_Call + 62
66 0x4bed3dp PyEval_EvalFrameEx + 12045
67 0x4b9ab6p PyEval_EvalCodeEx + 774
68 0x4d54b9p
69 0x4eebeep
70 0x4a577ep PyObject_Call + 62
71 0x548253p
72 0x4c15bfp PyEval_EvalFrameEx + 22415
73 0x4c136fp PyEval_EvalFrameEx + 21823
74 0x4c136fp PyEval_EvalFrameEx + 21823
75 0x4b9ab6p PyEval_EvalCodeEx + 774
76 0x4d55f3p
77 0x4eebeep
78 0x4a577ep PyObject_Call + 62
79 0x4bed3dp PyEval_EvalFrameEx + 12045
80 0x4b9ab6p PyEval_EvalCodeEx + 774
81 0x4d54b9p
82 0x4eebeep
83 0x4ee7f6p
84 0x4aa9abp
85 0x4c15bfp PyEval_EvalFrameEx + 22415
86 0x4b9ab6p PyEval_EvalCodeEx + 774
87 0x4eb30fp
88 0x4e5422p PyRun_FileExFlags + 130
89 0x4e3cd6p PyRun_SimpleFileExFlags + 390
90 0x493ae2p Py_Main + 1554
91 0x7f6222376830p __libc_start_main + 240
92 0x4933e9p _start + 41
*** Aborted at 1553054642 (unix time) try "date -d @1553054642" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x6ef2) received by PID 28402 (TID 0x7f6222b57700) from PID 28402; stack trace: ***
@ 0x7f6222731390 (unknown)
@ 0x7f622238b428 gsignal
@ 0x7f622238d02a abort
@ 0x7f621972492d __gnu_cxx::__verbose_terminate_handler()
@ 0x7f6219722996 __cxxabiv1::__terminate()
@ 0x7f6219721a49 __cxa_call_terminate
@ 0x7f6219722335 __gxx_personality_v0
@ 0x7f621c10bf83 (unknown)
@ 0x7f621c10c2eb _Unwind_RaiseException
@ 0x7f6219722beb __cxa_throw
@ 0x7f61c10b7dcf paddle::framework::details::OpHandleBase::~OpHandleBase()
@ 0x7f61c10a5c61 paddle::framework::details::FetchOpHandle::~FetchOpHandle()
@ 0x7f61bf8ea979 paddle::framework::ir::Node::~Node()
@ 0x7f61bf8eab21 paddle::framework::ir::Node::~Node()
@ 0x7f61c10a779b paddle::framework::details::ClearFetchOp()
@ 0x7f61c1053024 paddle::framework::details::ThreadedSSAGraphExecutor::Run()
@ 0x7f61c104978a paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run()
@ 0x7f61bf977c92 paddle::framework::ParallelExecutor::Run()
@ 0x7f61bf82a28e _ZZN8pybind1112cpp_function10initializeIZN6paddle6pybindL18pybind11_init_coreERNS_6moduleEEUlRNS2_9framework16ParallelExecutorERKSt6vectorISsSaISsEERKSsE152_vIS8_SD_SF_EINS_4nameENS_9is_methodENS_7siblingEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESX_
@ 0x7f61bf8651fe pybind11::cpp_function::dispatcher()
@ 0x4c37ed PyEval_EvalFrameEx
@ 0x4b9ab6 PyEval_EvalCodeEx
@ 0x4c16e7 PyEval_EvalFrameEx
@ 0x4b9ab6 PyEval_EvalCodeEx
@ 0x4c16e7 PyEval_EvalFrameEx
@ 0x4b9ab6 PyEval_EvalCodeEx
@ 0x4d54b9 (unknown)
@ 0x4a577e PyObject_Call
@ 0x4bed3d PyEval_EvalFrameEx
@ 0x4c136f PyEval_EvalFrameEx
@ 0x4b9ab6 PyEval_EvalCodeEx
@ 0x4d55f3 (unknown)
Abortedprint 具体的label值如下： 发现里面存在负数