集群训练报错Check failed: height_ == a.height_ (256 vs. 1390)
Created by: 20092136
以下是网络和 slot 类型的配置
def lstm_dnn_net(seq_dim,
feat_dim,
class_dim,
emb_dim=128,
hid_dim=512,
lstm_dim=128,
is_predict=False):
"""
lstm 和 DNN 网络
"""
bias_attr = ParameterAttribute(initial_std=0., l2_rate=0.0001)
fc_para_attr = ParameterAttribute(learning_rate=2e-3)
lstm_para_attr = ParameterAttribute(initial_std=0., learning_rate=2e-3)#, sparse_update=True)
sparse_up = ParameterAttribute(sparse_update=True)
layer_attr = ExtraLayerAttribute(drop_rate=0.5)
para_attr = [lstm_para_attr, fc_para_attr]
relu = ReluActivation()
linear = LinearActivation()
seq_data = data_layer("seq", seq_dim)
emb = embedding_layer(input=seq_data, size=emb_dim)
bi_lstm = bidirectional_lstm(input=emb, size=lstm_dim, concat_act=relu)
#dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5)
lstm_out = fc_layer(input=bi_lstm, size=lstm_dim, act=TanhActivation(),
bias_attr=bias_attr)
feat_data = data_layer("feat", feat_dim)
emb2 = embedding_layer(input=feat_data, size=emb_dim)
feat_layer_1 = fc_layer(input=emb2, size=hid_dim/4, act=relu,
bias_attr=bias_attr)#, param_attr=sparse_up)
feat_layer_2 = fc_layer(input=feat_layer_1, size=hid_dim/4, act=relu, bias_attr=bias_attr)
output = fc_layer(name='fc_ly_4', input=[lstm_out, feat_layer_2], size=class_dim,
act=SoftmaxActivation(),
bias_attr=bias_attr, param_attr=para_attr)
if is_predict:
outputs(output)
else:
outputs(
classification_cost(input=output, label=data_layer('label', 1),
evaluator=[precision_recall_evaluator, classification_error_evaluator]))
def hook(obj, dictionary, **kwargs):
obj.input_types = [integer_value_sequence(len(dictionary)), integer_value_sequence(6121), integer_value(888)]
训练时报错: F1130 14:12:50.641754 18718 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1390) F1130 14:12:50.643165 18726 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1327) F1130 14:12:50.643165 18726 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1327) F1130 14:12:50.644318 18714 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1408) F1130 14:12:50.643165 18726 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1327) F1130 14:12:50.644318 18714 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1408) F1130 14:12:50.645272 18722 Matrix.cpp:378] Check failed: height_ == a.height_ (256 vs. 1387)
请问可能是什么原因?