提交 313e9f55 编写于 作者: Y Yu Yang

Fix slow parsing a recursive depends topology

* Fix  #2797
* It because trainer_config_helpers' __dfs_travel__ did not record the
  node which travelled, and if the topology has a recursive dependency,
  there are some nodes will be travelled multiple times.
* Add a `travelled` set to record which node is travelled.
* Also add a unittest for this situation.
上级 6398c15c
......@@ -1408,6 +1408,8 @@ def outputs(layers, *args):
:return:
"""
traveled = set()
def __dfs_travel__(layer,
predicate=lambda x: x.layer_type == LayerType.DATA):
"""
......@@ -1419,6 +1421,11 @@ def outputs(layers, *args):
:type layer: LayerOutput
:return:
"""
if layer in traveled:
return []
else:
traveled.add(layer)
assert isinstance(layer, LayerOutput), "layer is %s" % (layer)
retv = []
if layer.parents is not None:
......
......@@ -6,6 +6,7 @@ img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cos
test_rnn_group shared_fc shared_lstm shared_gru test_cost_layers_with_weight
test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops
test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer
test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer)
test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer
test_recursive_topology)
export whole_configs=(test_split_datasource)
......@@ -131,6 +131,7 @@ input_layer_names: "weight"
input_layer_names: "multi_class_label"
output_layer_names: "__cost_0__"
output_layer_names: "__mse_cost_0__"
output_layer_names: "__nce_layer_0__"
evaluators {
name: "classification_error_evaluator"
type: "classification_error"
......@@ -154,6 +155,7 @@ sub_models {
input_layer_names: "multi_class_label"
output_layer_names: "__cost_0__"
output_layer_names: "__mse_cost_0__"
output_layer_names: "__nce_layer_0__"
evaluator_names: "classification_error_evaluator"
is_recurrent_layer_group: false
}
......
type: "nn"
layers {
name: "data"
type: "data"
size: 100
active_type: ""
}
layers {
name: "__addto_0__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "data"
}
inputs {
input_layer_name: "data"
}
}
layers {
name: "__addto_1__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_0__"
}
inputs {
input_layer_name: "__addto_0__"
}
}
layers {
name: "__addto_2__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_1__"
}
inputs {
input_layer_name: "__addto_1__"
}
}
layers {
name: "__addto_3__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_2__"
}
inputs {
input_layer_name: "__addto_2__"
}
}
layers {
name: "__addto_4__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_3__"
}
inputs {
input_layer_name: "__addto_3__"
}
}
layers {
name: "__addto_5__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_4__"
}
inputs {
input_layer_name: "__addto_4__"
}
}
layers {
name: "__addto_6__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_5__"
}
inputs {
input_layer_name: "__addto_5__"
}
}
layers {
name: "__addto_7__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_6__"
}
inputs {
input_layer_name: "__addto_6__"
}
}
layers {
name: "__addto_8__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_7__"
}
inputs {
input_layer_name: "__addto_7__"
}
}
layers {
name: "__addto_9__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_8__"
}
inputs {
input_layer_name: "__addto_8__"
}
}
layers {
name: "__addto_10__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_9__"
}
inputs {
input_layer_name: "__addto_9__"
}
}
layers {
name: "__addto_11__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_10__"
}
inputs {
input_layer_name: "__addto_10__"
}
}
layers {
name: "__addto_12__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_11__"
}
inputs {
input_layer_name: "__addto_11__"
}
}
layers {
name: "__addto_13__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_12__"
}
inputs {
input_layer_name: "__addto_12__"
}
}
layers {
name: "__addto_14__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_13__"
}
inputs {
input_layer_name: "__addto_13__"
}
}
layers {
name: "__addto_15__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_14__"
}
inputs {
input_layer_name: "__addto_14__"
}
}
layers {
name: "__addto_16__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_15__"
}
inputs {
input_layer_name: "__addto_15__"
}
}
layers {
name: "__addto_17__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_16__"
}
inputs {
input_layer_name: "__addto_16__"
}
}
layers {
name: "__addto_18__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_17__"
}
inputs {
input_layer_name: "__addto_17__"
}
}
layers {
name: "__addto_19__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_18__"
}
inputs {
input_layer_name: "__addto_18__"
}
}
layers {
name: "__addto_20__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_19__"
}
inputs {
input_layer_name: "__addto_19__"
}
}
layers {
name: "__addto_21__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_20__"
}
inputs {
input_layer_name: "__addto_20__"
}
}
layers {
name: "__addto_22__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_21__"
}
inputs {
input_layer_name: "__addto_21__"
}
}
layers {
name: "__addto_23__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_22__"
}
inputs {
input_layer_name: "__addto_22__"
}
}
layers {
name: "__addto_24__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_23__"
}
inputs {
input_layer_name: "__addto_23__"
}
}
layers {
name: "__addto_25__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_24__"
}
inputs {
input_layer_name: "__addto_24__"
}
}
layers {
name: "__addto_26__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_25__"
}
inputs {
input_layer_name: "__addto_25__"
}
}
layers {
name: "__addto_27__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_26__"
}
inputs {
input_layer_name: "__addto_26__"
}
}
layers {
name: "__addto_28__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_27__"
}
inputs {
input_layer_name: "__addto_27__"
}
}
layers {
name: "__addto_29__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_28__"
}
inputs {
input_layer_name: "__addto_28__"
}
}
layers {
name: "__addto_30__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_29__"
}
inputs {
input_layer_name: "__addto_29__"
}
}
layers {
name: "__addto_31__"
type: "addto"
size: 100
active_type: ""
inputs {
input_layer_name: "__addto_30__"
}
inputs {
input_layer_name: "__addto_30__"
}
}
layers {
name: "__fc_layer_0__"
type: "fc"
size: 32
active_type: "relu"
inputs {
input_layer_name: "__addto_31__"
input_parameter_name: "___fc_layer_0__.w0"
}
bias_parameter_name: "___fc_layer_0__.wbias"
}
layers {
name: "__fc_layer_1__"
type: "fc"
size: 10
active_type: "softmax"
inputs {
input_layer_name: "__fc_layer_0__"
input_parameter_name: "___fc_layer_1__.w0"
}
bias_parameter_name: "___fc_layer_1__.wbias"
}
parameters {
name: "___fc_layer_0__.w0"
size: 3200
initial_mean: 0.0
initial_std: 0.1
dims: 100
dims: 32
initial_strategy: 0
initial_smart: true
}
parameters {
name: "___fc_layer_0__.wbias"
size: 32
initial_mean: 0.0
initial_std: 0.0
dims: 1
dims: 32
initial_strategy: 0
initial_smart: false
}
parameters {
name: "___fc_layer_1__.w0"
size: 320
initial_mean: 0.0
initial_std: 0.176776695297
dims: 32
dims: 10
initial_strategy: 0
initial_smart: true
}
parameters {
name: "___fc_layer_1__.wbias"
size: 10
initial_mean: 0.0
initial_std: 0.0
dims: 1
dims: 10
initial_strategy: 0
initial_smart: false
}
input_layer_names: "data"
output_layer_names: "__fc_layer_1__"
sub_models {
name: "root"
layer_names: "data"
layer_names: "__addto_0__"
layer_names: "__addto_1__"
layer_names: "__addto_2__"
layer_names: "__addto_3__"
layer_names: "__addto_4__"
layer_names: "__addto_5__"
layer_names: "__addto_6__"
layer_names: "__addto_7__"
layer_names: "__addto_8__"
layer_names: "__addto_9__"
layer_names: "__addto_10__"
layer_names: "__addto_11__"
layer_names: "__addto_12__"
layer_names: "__addto_13__"
layer_names: "__addto_14__"
layer_names: "__addto_15__"
layer_names: "__addto_16__"
layer_names: "__addto_17__"
layer_names: "__addto_18__"
layer_names: "__addto_19__"
layer_names: "__addto_20__"
layer_names: "__addto_21__"
layer_names: "__addto_22__"
layer_names: "__addto_23__"
layer_names: "__addto_24__"
layer_names: "__addto_25__"
layer_names: "__addto_26__"
layer_names: "__addto_27__"
layer_names: "__addto_28__"
layer_names: "__addto_29__"
layer_names: "__addto_30__"
layer_names: "__addto_31__"
layer_names: "__fc_layer_0__"
layer_names: "__fc_layer_1__"
input_layer_names: "data"
output_layer_names: "__fc_layer_1__"
is_recurrent_layer_group: false
}
from paddle.trainer_config_helpers import *
settings(batch_size=1000, learning_rate=1e-5)
din = data_layer(name='data', size=100)
enc = din
for i in range(32):
enc = addto_layer([enc, enc])
pred = fc_layer(
input=fc_layer(
input=enc, size=32, act=ReluActivation()),
size=10,
act=SoftmaxActivation())
outputs(pred)
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册