Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
橘蓝
Paddle
提交
97c4d23f
P
Paddle
项目概览
橘蓝
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
97c4d23f
编写于
5月 26, 2017
作者:
X
xuwei06
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add docs and clean up unused code
上级
0cb8a666
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
91 addition
and
263 deletion
+91
-263
python/paddle/trainer/config_parser.py
python/paddle/trainer/config_parser.py
+2
-2
python/paddle/trainer_config_helpers/config_parser_utils.py
python/paddle/trainer_config_helpers/config_parser_utils.py
+1
-2
python/paddle/v2/config_base.py
python/paddle/v2/config_base.py
+48
-199
python/paddle/v2/evaluator.py
python/paddle/v2/evaluator.py
+3
-2
python/paddle/v2/inference.py
python/paddle/v2/inference.py
+12
-12
python/paddle/v2/layer.py
python/paddle/v2/layer.py
+19
-19
python/paddle/v2/tests/test_rnn_layer.py
python/paddle/v2/tests/test_rnn_layer.py
+3
-1
python/paddle/v2/topology.py
python/paddle/v2/topology.py
+3
-26
未找到文件。
python/paddle/trainer/config_parser.py
浏览文件 @
97c4d23f
...
...
@@ -3371,7 +3371,7 @@ def make_importer(config_dir, config_args):
return
Import
default_settings
=
dict
(
DEFAULT_SETTING
=
dict
(
batch_size
=
None
,
mini_batch_size
=
None
,
algorithm
=
'async_sgd'
,
...
...
@@ -3404,7 +3404,7 @@ default_settings = dict(
adam_beta2
=
0.999
,
adam_epsilon
=
1e-8
,
)
settings
=
copy
.
deepcopy
(
default_settings
)
settings
=
copy
.
deepcopy
(
DEFAULT_SETTING
)
settings_deprecated
=
dict
(
usage_ratio
=
1.
,
)
...
...
python/paddle/trainer_config_helpers/config_parser_utils.py
浏览文件 @
97c4d23f
...
...
@@ -15,7 +15,6 @@
import
copy
import
paddle.trainer.config_parser
as
config_parser
from
paddle.proto.TrainerConfig_pb2
import
OptimizationConfig
'''
This file is a wrapper of formal config_parser. The main idea of this file is to
separete different config logic into different function, such as network configuration
...
...
@@ -38,7 +37,7 @@ def parse_network_config(network_conf, config_arg_str=''):
def
parse_optimizer_config
(
optimizer_conf
,
config_arg_str
=
''
):
config_parser
.
settings
=
copy
.
deepcopy
(
config_parser
.
default_settings
)
config_parser
.
settings
=
copy
.
deepcopy
(
config_parser
.
DEFAULT_SETTING
)
optimizer_conf
()
opt_config
=
OptimizationConfig
()
for
k
,
v
in
config_parser
.
settings
.
iteritems
():
...
...
python/paddle/v2/config_base.py
浏览文件 @
97c4d23f
...
...
@@ -14,206 +14,55 @@
import
collections
import
re
from
paddle.trainer_config_helpers.default_decorators
import
wrap_name_default
import
paddle.trainer_config_helpers
as
conf_helps
from
topology
import
Topology
class
LayerType
(
type
):
def
__new__
(
cls
,
name
,
bases
,
attrs
):
method_name
=
attrs
.
get
(
'METHOD_NAME'
,
None
)
if
method_name
is
not
None
:
method
=
getattr
(
conf_helps
,
method_name
)
if
method
.
__doc__
is
not
None
:
mapper
=
attrs
.
get
(
"__map_docstr__"
,
None
)
if
mapper
is
not
None
:
attrs
[
'__doc__'
]
=
LayerType
.
__map_docstr__
(
mapper
(
method
.
__doc__
),
method_name
=
method_name
,
name
=
name
)
else
:
attrs
[
'__doc__'
]
=
LayerType
.
__map_docstr__
(
method
.
__doc__
,
method_name
=
method_name
,
name
=
name
)
return
super
(
LayerType
,
cls
).
__new__
(
cls
,
name
,
bases
,
attrs
)
@
staticmethod
def
__map_docstr__
(
doc
,
name
,
method_name
):
assert
isinstance
(
doc
,
basestring
)
# replace LayerOutput to paddle.v2.config_base.Layer
doc
=
doc
.
replace
(
"LayerOutput"
,
"paddle.v2.config_base.Layer"
)
doc
=
doc
.
replace
(
'ParameterAttribute'
,
'paddle.v2.attr.ParameterAttribute'
)
doc
=
re
.
sub
(
r
'ExtraLayerAttribute[^\s]?'
,
'paddle.v2.attr.ExtraAttribute'
,
doc
)
# xxx_layer to xxx
doc
=
re
.
sub
(
r
"(?P<name>[a-z]+)_layer"
,
r
"\g<name>"
,
doc
)
# XxxxActivation to paddle.v2.Activation.Xxxx
doc
=
re
.
sub
(
r
"(?P<name>[A-Z][a-zA-Z]+)Activation"
,
r
"paddle.v2.Activation.\g<name>"
,
doc
)
# TODO(yuyang18): Add more rules if needed.
__layer_map__
=
{}
def
__map_docstr__
(
doc
,
name
):
if
doc
is
None
:
return
doc
assert
isinstance
(
doc
,
basestring
)
# replace LayerOutput to paddle.v2.config_base.Layer
doc
=
doc
.
replace
(
"LayerOutput"
,
"paddle.v2.config_base.Layer"
)
doc
=
doc
.
replace
(
'ParameterAttribute'
,
'paddle.v2.attr.ParameterAttribute'
)
doc
=
re
.
sub
(
r
'ExtraLayerAttribute[^\s]?'
,
'paddle.v2.attr.ExtraAttribute'
,
doc
)
# xxx_layer to xxx
doc
=
re
.
sub
(
r
"(?P<name>[a-z]+)_layer"
,
r
"\g<name>"
,
doc
)
# XxxxActivation to paddle.v2.Activation.Xxxx
doc
=
re
.
sub
(
r
"(?P<name>[A-Z][a-zA-Z]+)Activation"
,
r
"paddle.v2.Activation.\g<name>"
,
doc
)
# xxx_evaluator to paddle.v2.evaluator.xxx
doc
=
re
.
sub
(
r
"(?P<name>[a-z]+)_evaluator"
,
r
"evaluator.\g<name>"
,
doc
)
# TODO(yuyang18): Add more rules if needed.
return
doc
def
__convert_to_v2__
(
f
,
name
,
module
):
def
wrapped
(
*
args
,
**
xargs
):
out
=
f
(
*
args
,
**
xargs
)
outs
=
out
if
not
isinstance
(
out
,
collections
.
Sequence
):
outs
=
[
out
]
for
l
in
outs
:
if
isinstance
(
l
,
conf_helps
.
LayerOutput
):
__layer_map__
[
l
.
full_name
]
=
l
return
out
wrapped
.
__doc__
=
__map_docstr__
(
f
.
__doc__
,
name
)
wrapped
.
__name__
=
name
wrapped
.
__module__
=
module
return
wrapped
class
Layer
(
object
):
__metaclass__
=
LayerType
def
__init__
(
self
,
name
=
None
,
parent_layers
=
None
):
assert
isinstance
(
parent_layers
,
dict
)
self
.
name
=
name
self
.
__context__
=
{}
self
.
__parent_layers__
=
parent_layers
# some layer may have some extra parent layer
self
.
__extra_parent__
=
[]
# used for evaluator.
self
.
__children_layers__
=
[]
def
extra_parent
(
self
):
return
self
.
__extra_parent__
def
append_extra_parent
(
self
,
parent
):
self
.
__extra_parent__
.
append
(
parent
)
def
append_child
(
self
,
layer
,
parent_names
):
self
.
__children_layers__
.
append
((
layer
,
parent_names
))
def
to_proto
(
self
,
context
):
"""
function to set proto attribute
"""
self
.
__context__
=
context
# STEP: short cut if this layer is parsed before.
if
self
.
context_name
()
in
context
:
if
self
.
use_context_name
():
return
context
[
self
.
context_name
()]
else
:
return
context
[
self
.
name
]
# STEP: parse extra_parent that is not used by this layer but must
# be parsed before this layer.
for
p
in
self
.
__extra_parent__
:
p
.
to_proto
(
context
=
context
)
# STEP: parse parent that is used by this layer, get the result and
# insert into kwargs of the next layer's to_proto_impl method.
kwargs
=
dict
()
for
layer_name
in
self
.
__parent_layers__
:
if
not
isinstance
(
self
.
__parent_layers__
[
layer_name
],
collections
.
Sequence
):
v1_layer
=
self
.
__parent_layers__
[
layer_name
].
to_proto
(
context
=
context
)
else
:
v1_layer
=
map
(
lambda
x
:
x
.
to_proto
(
context
=
context
),
self
.
__parent_layers__
[
layer_name
])
kwargs
[
layer_name
]
=
v1_layer
# STEP: parse myself and add myself into context.
ret_val
=
self
.
to_proto_impl
(
**
kwargs
)
if
self
.
context_name
()
is
not
None
\
and
self
.
context_name
()
not
in
context
:
context
[
self
.
context_name
()]
=
ret_val
# STEP: parse children that should be pased after this layer.
for
layer
,
pnames
in
self
.
__children_layers__
:
drop
=
False
# child will only be parsed if all parents are in context.
for
pname
in
pnames
:
if
pname
not
in
context
:
drop
=
True
break
if
drop
:
continue
layer
.
to_proto
(
context
=
context
)
# STEP: return v1 layer result
if
self
.
context_name
()
is
None
:
return
ret_val
elif
self
.
use_context_name
():
return
context
[
self
.
context_name
()]
else
:
return
context
[
self
.
name
]
def
to_proto_impl
(
self
,
**
kwargs
):
raise
NotImplementedError
()
def
context_name
(
self
):
"""
Context name means the context which stores `to_proto_impl` result.
If multiple layer share same context_name, the `to_proto_impl` of them
will be invoked only once.
"""
return
self
.
name
def
use_context_name
(
self
):
return
False
def
calculate_size
(
self
):
"""
lazy calculate size of the layer, should be called when to_proto_impl of
this layer is called.
:return:
"""
return
self
.
__context__
[
self
.
context_name
()].
size
def
attr
(
self
):
topo
=
Topology
(
self
)
return
topo
.
get_layer_proto
(
self
.
name
)
def
__convert_to_v2__
(
method_name
,
parent_names
,
is_default_name
=
True
,
attach_parent
=
False
):
if
is_default_name
:
wrapper
=
wrap_name_default
(
name_prefix
=
method_name
)
else
:
wrapper
=
None
class
V2LayerImpl
(
Layer
):
METHOD_NAME
=
method_name
def
__init__
(
self
,
**
kwargs
):
parent_layers
=
dict
()
other_kwargs
=
dict
()
for
pname
in
parent_names
:
if
pname
in
kwargs
:
parent_layers
[
pname
]
=
kwargs
[
pname
]
if
attach_parent
:
pnames
=
[
x
.
context_name
()
for
x
in
parent_layers
.
values
()]
for
pname
in
parent_layers
:
layers
=
kwargs
[
pname
]
if
not
isinstance
(
layers
,
collections
.
Sequence
):
layers
=
[
layers
]
for
layer
in
layers
:
layer
.
append_child
(
self
,
pnames
)
for
key
in
kwargs
.
keys
():
if
key
not
in
parent_names
:
other_kwargs
[
key
]
=
kwargs
[
key
]
name
=
kwargs
.
get
(
'name'
,
None
)
super
(
V2LayerImpl
,
self
).
__init__
(
name
,
parent_layers
)
self
.
__other_kwargs__
=
other_kwargs
if
wrapper
is
not
None
:
__init__
=
wrapper
(
__init__
)
def
to_proto_impl
(
self
,
**
kwargs
):
args
=
dict
()
for
each
in
kwargs
:
args
[
each
]
=
kwargs
[
each
]
for
each
in
self
.
__other_kwargs__
:
args
[
each
]
=
self
.
__other_kwargs__
[
each
]
return
getattr
(
conf_helps
,
method_name
)(
**
args
)
return
V2LayerImpl
Layer
=
conf_helps
.
LayerOutput
python/paddle/v2/evaluator.py
浏览文件 @
97c4d23f
...
...
@@ -13,8 +13,8 @@
# limitations under the License.
import
paddle.trainer_config_helpers.evaluators
as
evs
import
inspect
from
config_base
import
__convert_to_v2__
import
inspect
__all__
=
[]
...
...
@@ -27,7 +27,8 @@ def initialize():
__ev__
=
getattr
(
evs
,
__ev_name__
)
__new_name__
=
convert_to_new_name
(
__ev_name__
)
globals
()[
__new_name__
]
=
__ev__
globals
()[
__new_name__
]
=
__convert_to_v2__
(
__ev__
,
__new_name__
,
__name__
)
globals
()[
__new_name__
].
__name__
=
__new_name__
__all__
.
append
(
__new_name__
)
...
...
python/paddle/v2/inference.py
浏览文件 @
97c4d23f
...
...
@@ -12,9 +12,9 @@ class Inference(object):
"""
Inference combines neural network output and parameters together
to do inference.
.. code-block:: python
inferer = Inference(output_layer=prediction, parameters=parameters)
for data_batch in batches:
print inferer.infer(data_batch)
...
...
@@ -92,8 +92,8 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
.. code-block:: python
result = paddle.infer(output_layer=prediction,
parameters=parameters,
result = paddle.infer(output_layer=prediction,
parameters=parameters,
input=SomeData)
print result
...
...
@@ -101,14 +101,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
.. code-block:: python
result = paddle.infer(output_layer=[prediction1, prediction2],
parameters=parameters,
result = paddle.infer(output_layer=[prediction1, prediction2],
parameters=parameters,
input=SomeData,
field=[id, value]])
print result
:param output_layer: output of the neural network that would be inferred
:type output_layer: paddle.v2.config_base.Layer or a list of
:type output_layer: paddle.v2.config_base.Layer or a list of
paddle.v2.config_base.Layer
:param parameters: parameters of the neural network.
:type parameters: paddle.v2.parameters.Parameters
...
...
@@ -117,14 +117,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
:type input: collections.Iterable
:param feeding: Reader dictionary. Default could generate from input
value.
:param field: The prediction field. It should in [`value`, `id`, `prob`].
`value` and `prob` mean return the prediction probabilities,
:param field: The prediction field. It should in [`value`, `id`, `prob`].
`value` and `prob` mean return the prediction probabilities,
`id` means return the prediction labels. Default is `value`.
Note that `prob` only used when output_layer is beam_search
Note that `prob` only used when output_layer is beam_search
or max_id.
:type field: str
:return: The prediction result. If there are multiple outout_layers and fields,
the return order is outout_layer1.field1, outout_layer2.field1, ...,
:return: The prediction result. If there are multiple outout_layers and fields,
the return order is outout_layer1.field1, outout_layer2.field1, ...,
outout_layer1.field2, outout_layer2.field2 ...
:rtype: numpy.ndarray
"""
...
...
python/paddle/v2/layer.py
浏览文件 @
97c4d23f
...
...
@@ -33,26 +33,14 @@ The primary usage shows below.
import
collections
import
copy
import
re
import
paddle.trainer_config_helpers.layers
as
v1_layers
import
paddle.trainer.config_parser
as
cp
from
paddle.proto.ModelConfig_pb2
import
ModelConfig
,
SubModelConfig
from
config_base
import
__convert_to_v2__
import
config_base
__all__
=
[
'data'
,
'parse_network'
]
__layer_map__
=
{}
def
__wrap__
(
f
):
def
wrapped
(
*
args
,
**
xargs
):
out
=
f
(
*
args
,
**
xargs
)
outs
=
out
if
not
isinstance
(
out
,
collections
.
Sequence
):
outs
=
[
out
]
for
l
in
outs
:
if
isinstance
(
l
,
v1_layers
.
LayerOutput
):
__layer_map__
[
l
.
full_name
]
=
l
return
out
return
wrapped
def
__need_to_keep__
(
name
):
...
...
@@ -90,7 +78,7 @@ for name in v1_layers.__all__:
continue
new_name
=
__convert_name__
(
name
)
if
callable
(
obj
)
and
__need_to_wrap__
(
name
):
globals
()[
new_name
]
=
__
wrap__
(
obj
)
globals
()[
new_name
]
=
__
convert_to_v2__
(
obj
,
new_name
,
__name__
)
else
:
globals
()[
new_name
]
=
obj
__all__
.
append
(
new_name
)
...
...
@@ -102,9 +90,21 @@ def __data_layer__(name, type, **kwargs):
return
l
data
=
__wrap__
(
__data_layer__
)
def
__map_data_docstr__
(
doc
):
doc
=
re
.
sub
(
r
'(data = [^\)]+)\).*'
,
"data = paddle.layer.data(name=
\"
input
\"
, "
"type=paddle.data_type.dense_vector(1000))"
,
doc
)
doc
=
re
.
sub
(
r
':param size:.*'
,
':param type: Data type of this data layer'
,
doc
)
doc
=
re
.
sub
(
r
':type size:.*'
,
":type size: paddle.v2.data_type.InputType"
,
doc
)
return
doc
__data_layer__
.
__doc__
=
__map_data_docstr__
(
v1_layers
.
data_layer
.
__doc__
)
LayerV2
=
v1_layers
.
LayerOutput
data
=
__convert_to_v2__
(
__data_layer__
,
'name'
,
__name__
)
def
__get_used_layers__
(
output_layers
,
extra_layers
=
None
):
...
...
@@ -273,7 +273,7 @@ def parse_network(output_layers, extra_layers=None):
def
get_layer
(
name
):
return
__layer_map__
.
get
(
name
)
return
config_base
.
__layer_map__
.
get
(
name
)
cp
.
begin_parse
()
python/paddle/v2/tests/test_rnn_layer.py
浏览文件 @
97c4d23f
...
...
@@ -32,6 +32,7 @@ class RNNTest(unittest.TestCase):
def
parse_old_rnn
():
reset_parser
()
def
step
(
y
):
mem
=
conf_helps
.
memory
(
name
=
"rnn_state"
,
size
=
hidden_dim
)
out
=
conf_helps
.
fc_layer
(
...
...
@@ -52,6 +53,7 @@ class RNNTest(unittest.TestCase):
def
parse_new_rnn
():
reset_parser
()
def
new_step
(
y
):
mem
=
layer
.
memory
(
name
=
"rnn_state"
,
size
=
hidden_dim
)
out
=
layer
.
fc
(
input
=
[
y
,
mem
],
...
...
@@ -72,7 +74,6 @@ class RNNTest(unittest.TestCase):
parse_new_rnn
().
splitlines
(
1
))
print
''
.
join
(
diff
)
def
test_sequence_rnn_multi_input
(
self
):
dict_dim
=
10
word_dim
=
8
...
...
@@ -81,6 +82,7 @@ class RNNTest(unittest.TestCase):
def
parse_old_rnn
():
reset_parser
()
def
test
():
data
=
conf_helps
.
data_layer
(
name
=
"word"
,
size
=
dict_dim
)
label
=
conf_helps
.
data_layer
(
name
=
"label"
,
size
=
label_dim
)
...
...
python/paddle/v2/topology.py
浏览文件 @
97c4d23f
...
...
@@ -17,34 +17,11 @@ import collections
from
paddle.proto.ModelConfig_pb2
import
ModelConfig
import
paddle.trainer_config_helpers
as
conf_helps
import
layer
as
v2_layer
import
config_base
__all__
=
[
'Topology'
]
def
__flatten__
(
lis
):
"""
Given a list, possibly nested to any level, return it flattened.
"""
new_lis
=
[]
for
item
in
lis
:
if
isinstance
(
item
,
collections
.
Sequence
):
new_lis
.
extend
(
__flatten__
(
item
))
else
:
new_lis
.
append
(
item
)
return
new_lis
def
__bfs_travel__
(
callback
,
*
layers
):
layers
=
__flatten__
(
layers
)
for
each_layer
in
layers
:
__break__
=
callback
(
each_layer
)
if
__break__
:
return
__layers__
=
each_layer
.
__parent_layers__
.
values
()
+
\
each_layer
.
extra_parent
()
__bfs_travel__
(
callback
,
*
__layers__
)
class
Topology
(
object
):
"""
Topology is used to store the information about all layers
...
...
@@ -125,5 +102,5 @@ class Topology(object):
def
__check_layer_type__
(
layer
):
if
not
isinstance
(
layer
,
v2_layer
.
LayerV2
):
raise
ValueError
(
'layer should have type paddle.
layer
.Layer'
)
if
not
isinstance
(
layer
,
config_base
.
Layer
):
raise
ValueError
(
'layer should have type paddle.
v2.config_base
.Layer'
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录