分布式模型训练,到第9轮迭代时报错
Created by: hfreedom00
报错部分的模型网络结构代码: q_u_concat = layers.concat(input=[q_fc3, u_fc3], axis=1) q_u_fc = fluid.layers.fc(input=q_u_concat, size=64, act='relu', param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( scale=1 / math.sqrt(q_u_concat.shape[1])), learning_rate=FC_LR)) predict = fluid.layers.fc(input=q_u_fc, size=2, act='softmax', param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( scale=1 / math.sqrt(q_u_concat.shape[1])), learning_rate=1.0))
label = layers.data(name='score', shape=[1], dtype='int64')
cost = fluid.layers.cross_entropy(input=predict, label=label)
avg_cost = fluid.layers.reduce_sum(cost)
accuracy = fluid.layers.accuracy(input=predict, label=label)
auc_var, batch_auc_var, auc_states = \
fluid.layers.auc(input=predict, label=label, num_thresholds=2 ** 12, slide_steps=20)
报错信息: Python Callstacks: File "/usr/local/lib/python2.7/site-packages/paddle/fluid/framework.py", line 1317, in append_op attrs=kwargs.get("attrs", None)) File "/usr/local/lib/python2.7/site-packages/paddle/fluid/layer_helper.py", line 56, in append_op return self.main_program.current_block().append_op(args, kwargs) File "/usr/local/lib/python2.7/site-packages/paddle/fluid/layers/metric_op.py", line 169, in auc "StatNegOut": [batch_stat_neg] File "train_batch_1_yibu.py", line 498, in model fluid.layers.auc(input=predict, label=label, num_thresholds=2 ** 12, slide_steps=20) File "train_batch_1_yibu.py", line 503, in train predict, avg_cost, auc_var, batch_auc_var, q_fc3, u_fc3 = model() File "train_batch_1_yibu.py", line 674, in main train(use_cuda, is_local) File "train_batch_1_yibu.py", line 682, in main(use_cuda, is_local) ++ Callstacks: Enforce failed. Expected predict_data <= 1, but received predict_data:-nan > 1:1. The predict data must less or equal 1. at [/paddle/paddle/fluid/operators/metrics/auc_op.h:80] PaddlePaddle Call Stacks: 0 0x7f40f185959dp void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 365 1 0x7f40f18598e7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const, int) + 87 2 0x7f40f21a6482p paddle::operators::AucKernel<paddle::platform::CPUPlace, float>::statAuc(paddle::framework::Tensor const, paddle::framework::Tensor const*, int, int, int, long*, long*, long**, long**) + 1202 3 0x7f40f21a696ep paddle::operators::AucKernel<paddle::platform::CPUPlace, float>::Compute(paddle::framework::ExecutionContext const&) const + 830 4 0x7f40f21a6c23p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::AucKernel<paddle::platform::CPUPlace, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 5 0x7f40f281dc93p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 659 6 0x7f40f281c7bbp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 267 7 0x7f40f196e042p paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 226 8 0x7f40f196f105p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 261