Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
ujs_wantz
Paddle
提交
e55ec44b
P
Paddle
项目概览
ujs_wantz
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e55ec44b
编写于
2月 28, 2017
作者:
Y
Yu Yang
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'develop' of github.com:baidu/Paddle into feature/clean_mnist_v2
上级
0432e907
c444708a
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
217 addition
and
38 deletion
+217
-38
demo/mnist/api_train_v2.py
demo/mnist/api_train_v2.py
+8
-10
python/paddle/v2/__init__.py
python/paddle/v2/__init__.py
+3
-1
python/paddle/v2/data_feeder.py
python/paddle/v2/data_feeder.py
+1
-1
python/paddle/v2/data_type.py
python/paddle/v2/data_type.py
+3
-3
python/paddle/v2/layer.py
python/paddle/v2/layer.py
+1
-0
python/paddle/v2/parameters.py
python/paddle/v2/parameters.py
+8
-12
python/paddle/v2/tests/CMakeLists.txt
python/paddle/v2/tests/CMakeLists.txt
+7
-1
python/paddle/v2/tests/test_topology.py
python/paddle/v2/tests/test_topology.py
+83
-0
python/paddle/v2/topology.py
python/paddle/v2/topology.py
+95
-0
python/paddle/v2/trainer.py
python/paddle/v2/trainer.py
+8
-10
未找到文件。
demo/mnist/api_train_v2.py
浏览文件 @
e55ec44b
import
numpy
import
paddle.v2
as
paddle
...
...
@@ -30,19 +29,18 @@ def main():
pass
trainer
=
paddle
.
trainer
.
SGD
(
update_equation
=
adam_optimizer
)
trainer
.
train
(
reader
=
paddle
.
reader
.
batched
(
paddle
.
reader
.
shuffle
(
paddle
.
dataset
.
mnist
.
train_creator
(),
buf_size
=
8192
),
batch_size
=
32
),
topology
=
cost
,
paddle
.
reader
.
shuffle
(
paddle
.
dataset
.
mnist
.
train_creator
(),
buf_size
=
8192
),
batch_size
=
32
),
cost
=
cost
,
parameters
=
parameters
,
event_handler
=
event_handler
,
data_types
=
[
# data_types will be removed, It should be in
# network topology
(
'pixel'
,
images
.
type
),
(
'label'
,
label
.
type
)],
reader_dict
=
{
'pixel'
:
0
,
'label'
:
1
}
)
batch_size
=
32
,
# batch size should be refactor in Data reader
reader_dict
=
{
images
.
name
:
0
,
label
.
name
:
1
})
if
__name__
==
'__main__'
:
...
...
python/paddle/v2/__init__.py
浏览文件 @
e55ec44b
...
...
@@ -18,6 +18,7 @@ import parameters
import
trainer
import
event
import
data_type
import
topology
import
data_feeder
from
.
import
dataset
from
.
import
reader
...
...
@@ -27,7 +28,8 @@ import py_paddle.swig_paddle as api
__all__
=
[
'optimizer'
,
'layer'
,
'activation'
,
'parameters'
,
'init'
,
'trainer'
,
'event'
,
'data_type'
,
'attr'
,
'pooling'
,
'data_feeder'
,
'dataset'
,
'reader'
'event'
,
'data_type'
,
'attr'
,
'pooling'
,
'data_feeder'
,
'dataset'
,
'reader'
,
'topology'
]
...
...
python/paddle/v2/data_feeder.py
浏览文件 @
e55ec44b
...
...
@@ -23,7 +23,7 @@ class DataFeeder(DataProviderConverter):
"""
DataFeeder converts the data returned by paddle.reader into a data structure
of Arguments which is defined in the API. The paddle.reader usually returns
a list of mini-batch data entries. Each data entry in the list is one sampe.
a list of mini-batch data entries. Each data entry in the list is one samp
l
e.
Each sample is a list or a tuple with one feature or multiple features.
DataFeeder converts this mini-batch data entries into Arguments in order
to feed it to C++ interface.
...
...
python/paddle/v2/data_type.py
浏览文件 @
e55ec44b
...
...
@@ -13,10 +13,10 @@
# limitations under the License.
from
paddle.trainer.PyDataProvider2
import
\
InputType
,
dense_vector
,
sparse_binary_vector
,
\
InputType
,
DataType
,
dense_vector
,
sparse_binary_vector
,
\
sparse_vector
,
integer_value
,
integer_value_sequence
__all__
=
[
'InputType'
,
'
dense_vector'
,
'sparse_binary_vector'
,
'sparse
_vector'
,
'integer_value'
,
'integer_value_sequence'
'InputType'
,
'
DataType'
,
'dense_vector'
,
'sparse_binary
_vector'
,
'
sparse_vector'
,
'
integer_value'
,
'integer_value_sequence'
]
python/paddle/v2/layer.py
浏览文件 @
e55ec44b
...
...
@@ -284,6 +284,7 @@ def mixed(size=0,
return
MixedLayerV2
(
size
,
input
,
name
,
act
,
bias_attr
,
layer_attr
)
LayerV2
=
Layer
data
=
DataLayerV2
AggregateLevel
=
conf_helps
.
layers
.
AggregateLevel
ExpandLevel
=
conf_helps
.
layers
.
ExpandLevel
...
...
python/paddle/v2/parameters.py
浏览文件 @
e55ec44b
import
numpy
as
np
from
.
import
layer
as
v2_layer
import
py_paddle.swig_paddle
as
api
from
paddle.proto.ParameterConfig_pb2
import
ParameterConfig
from
topology
import
Topology
__all__
=
[
'Parameters'
,
'create'
]
def
create
(
*
layers
):
def
create
(
layers
):
"""
Create parameter pool by layers. In paddle, layer can be represent a
model config.
Create parameter pool by topology.
:param layers:
:return:
"""
for
layer
in
layers
:
if
not
isinstance
(
layer
,
v2_layer
.
Layer
):
raise
ValueError
(
'create must pass a topologies which type is paddle.layer.Layer'
)
model_config
=
v2_layer
.
parse_network
(
*
layers
)
topology
=
Topology
(
layers
)
pool
=
Parameters
()
for
param
in
model_config
.
parameters
:
for
param
in
topology
.
proto
()
.
parameters
:
pool
.
__append_config__
(
param
)
return
pool
...
...
@@ -224,7 +219,8 @@ class Parameters(object):
except
ValueError
:
# If no such parameter in gradient machine, then don't copy
pass
self
.
__gradient_machines__
.
append
(
gradient_machine
)
self
.
__gradient_machines__
.
append
(
gradient_machine
)
def
__get_parameter_in_gradient_machine__
(
gradient_machine
,
name
):
...
...
python/paddle/v2/tests/CMakeLists.txt
浏览文件 @
e55ec44b
...
...
@@ -2,5 +2,11 @@ add_test(NAME test_v2_layer
COMMAND
${
PROJ_ROOT
}
/paddle/.set_python_path.sh -d
${
PROJ_ROOT
}
/python/
${
PYTHON_EXECUTABLE
}
${
PROJ_ROOT
}
/python/paddle/v2/tests/test_layer.py
WORKING_DIRECTORY
${
PROJ_ROOT
}
/python/paddle
)
add_test
(
NAME test_v2_api
COMMAND bash
${
PROJ_ROOT
}
/python/paddle/v2/tests/run_tests.sh
${
PYTHON_EXECUTABLE
}
)
COMMAND bash
${
PROJ_ROOT
}
/python/paddle/v2/tests/run_tests.sh
${
PYTHON_EXECUTABLE
}
)
add_test
(
NAME topology_test
COMMAND
${
PROJ_ROOT
}
/paddle/.set_python_path.sh -d
${
PROJ_ROOT
}
/python/
${
PYTHON_EXECUTABLE
}
${
PROJ_ROOT
}
/python/paddle/v2/tests/test_topology.py
WORKING_DIRECTORY
${
PROJ_ROOT
}
/python/paddle
)
python/paddle/v2/tests/test_topology.py
0 → 100644
浏览文件 @
e55ec44b
# Copyright PaddlePaddle contributors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
unittest
import
paddle.v2.layer
as
layer
import
paddle.v2.topology
as
topology
import
paddle.v2.data_type
as
data_type
import
paddle.trainer_config_helpers
as
conf_helps
class
TestTopology
(
unittest
.
TestCase
):
def
test_data_type
(
self
):
pixel
=
layer
.
data
(
name
=
'pixel'
,
type
=
data_type
.
dense_vector
(
784
))
label
=
layer
.
data
(
name
=
'label'
,
type
=
data_type
.
integer_value
(
10
))
hidden
=
layer
.
fc
(
input
=
pixel
,
size
=
100
,
act
=
conf_helps
.
SigmoidActivation
())
inference
=
layer
.
fc
(
input
=
hidden
,
size
=
10
,
act
=
conf_helps
.
SoftmaxActivation
())
cost
=
layer
.
classification_cost
(
input
=
inference
,
label
=
label
)
topo
=
topology
.
Topology
(
cost
)
data_types
=
topo
.
data_type
()
self
.
assertEqual
(
len
(
data_types
),
2
)
pixel_data_type
=
filter
(
lambda
type
:
type
[
0
]
==
"pixel"
,
data_types
)
self
.
assertEqual
(
len
(
pixel_data_type
),
1
)
pixel_data_type
=
pixel_data_type
[
0
]
self
.
assertEqual
(
pixel_data_type
[
1
].
type
,
data_type
.
DataType
.
Dense
)
self
.
assertEqual
(
pixel_data_type
[
1
].
dim
,
784
)
label_data_type
=
filter
(
lambda
type
:
type
[
0
]
==
"label"
,
data_types
)
self
.
assertEqual
(
len
(
label_data_type
),
1
)
label_data_type
=
label_data_type
[
0
]
self
.
assertEqual
(
label_data_type
[
1
].
type
,
data_type
.
DataType
.
Index
)
self
.
assertEqual
(
label_data_type
[
1
].
dim
,
10
)
def
test_get_layer
(
self
):
pixel
=
layer
.
data
(
name
=
'pixel'
,
type
=
data_type
.
dense_vector
(
784
))
label
=
layer
.
data
(
name
=
'label'
,
type
=
data_type
.
integer_value
(
10
))
hidden
=
layer
.
fc
(
input
=
pixel
,
size
=
100
,
act
=
conf_helps
.
SigmoidActivation
())
inference
=
layer
.
fc
(
input
=
hidden
,
size
=
10
,
act
=
conf_helps
.
SoftmaxActivation
())
cost
=
layer
.
classification_cost
(
input
=
inference
,
label
=
label
)
topo
=
topology
.
Topology
(
cost
)
pixel_layer
=
topo
.
get_layer
(
"pixel"
)
label_layer
=
topo
.
get_layer
(
"label"
)
self
.
assertEqual
(
pixel_layer
,
pixel
)
self
.
assertEqual
(
label_layer
,
label
)
def
test_parse
(
self
):
pixel
=
layer
.
data
(
name
=
'pixel'
,
type
=
data_type
.
dense_vector
(
784
))
label
=
layer
.
data
(
name
=
'label'
,
type
=
data_type
.
integer_value
(
10
))
hidden
=
layer
.
fc
(
input
=
pixel
,
size
=
100
,
act
=
conf_helps
.
SigmoidActivation
())
inference
=
layer
.
fc
(
input
=
hidden
,
size
=
10
,
act
=
conf_helps
.
SoftmaxActivation
())
maxid
=
layer
.
max_id
(
input
=
inference
)
cost1
=
layer
.
classification_cost
(
input
=
inference
,
label
=
label
)
cost2
=
layer
.
cross_entropy_cost
(
input
=
inference
,
label
=
label
)
topology
.
Topology
(
cost2
).
proto
()
topology
.
Topology
([
cost1
]).
proto
()
topology
.
Topology
([
cost1
,
cost2
]).
proto
()
topology
.
Topology
([
inference
,
maxid
]).
proto
()
if
__name__
==
'__main__'
:
unittest
.
main
()
python/paddle/v2/topology.py
0 → 100644
浏览文件 @
e55ec44b
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
collections
from
paddle.proto.ModelConfig_pb2
import
ModelConfig
import
layer
as
v2_layer
__all__
=
[
'Topology'
]
class
Topology
(
object
):
"""
Topology is used to store the information about all layers
and network configs.
"""
def
__init__
(
self
,
layers
):
if
not
isinstance
(
layers
,
collections
.
Sequence
):
__check_layer_type__
(
layers
)
layers
=
[
layers
]
for
layer
in
layers
:
__check_layer_type__
(
layer
)
self
.
layers
=
layers
self
.
__model_config__
=
v2_layer
.
parse_network
(
*
layers
)
assert
isinstance
(
self
.
__model_config__
,
ModelConfig
)
def
proto
(
self
):
return
self
.
__model_config__
def
get_layer
(
self
,
name
):
"""
get v2.Layer Class instance by layer name
:param name:
:return:
"""
result_layer
=
[]
def
find_layer_by_name
(
layer
,
layer_name
):
if
len
(
result_layer
)
==
1
:
return
elif
layer
.
name
==
layer_name
:
result_layer
.
append
(
layer
)
else
:
for
parent_layer
in
layer
.
__parent_layers__
.
values
():
find_layer_by_name
(
parent_layer
,
layer_name
)
for
layer
in
self
.
layers
:
find_layer_by_name
(
layer
,
name
)
assert
len
(
result_layer
)
==
1
return
result_layer
[
0
]
def
data_layers
(
self
):
"""
get all data layer
:return:
"""
data_layers
=
set
()
def
find_data_layer
(
layer
):
if
isinstance
(
layer
,
v2_layer
.
DataLayerV2
):
data_layers
.
add
(
layer
)
for
parent_layer
in
layer
.
__parent_layers__
.
values
():
find_data_layer
(
parent_layer
)
for
layer
in
self
.
layers
:
find_data_layer
(
layer
)
return
data_layers
def
data_type
(
self
):
"""
get data_type from proto, such as:
[('image', dense_vector(768)), ('label', integer_value(10))]
"""
return
[(
data_layer
.
name
,
data_layer
.
type
)
for
data_layer
in
self
.
data_layers
()]
def
__check_layer_type__
(
layer
):
if
not
isinstance
(
layer
,
v2_layer
.
LayerV2
):
raise
ValueError
(
'layer should have type paddle.layer.Layer'
)
python/paddle/v2/trainer.py
浏览文件 @
e55ec44b
import
collections
import
py_paddle.swig_paddle
as
api
from
paddle.proto.ModelConfig_pb2
import
ModelConfig
from
data_feeder
import
DataFeeder
from
data_feeder
import
DataFeeder
from
topology
import
Topology
from
.
import
event
as
v2_event
from
.
import
layer
as
v2_layer
from
.
import
optimizer
as
v2_optimizer
from
.
import
parameters
as
v2_parameters
...
...
@@ -57,11 +56,10 @@ class SGD(ITrainer):
def
train
(
self
,
reader
,
topology
,
cost
,
parameters
,
num_passes
=
1
,
event_handler
=
None
,
data_types
=
None
,
reader_dict
=
None
):
"""
Training method. Will train num_passes of input data.
...
...
@@ -73,18 +71,18 @@ class SGD(ITrainer):
:param event_handler: Event handler. A method will be invoked when event
occurred.
:type event_handler: (BaseEvent) => None
:param data_types: Not important, will be removed after data refactor.
:return:
"""
if
event_handler
is
None
:
event_handler
=
default_event_handler
topology
=
v2_layer
.
parse_network
(
topology
)
topology
=
Topology
(
cost
)
__check_train_args__
(
**
locals
())
gm
=
api
.
GradientMachine
.
createFromConfigProto
(
topology
,
api
.
CREATE_MODE_NORMAL
,
self
.
__optimizer__
.
enable_types
())
topology
.
proto
(),
api
.
CREATE_MODE_NORMAL
,
self
.
__optimizer__
.
enable_types
())
assert
isinstance
(
gm
,
api
.
GradientMachine
)
parameters
.
append_gradient_machine
(
gm
)
gm
.
randParameters
()
...
...
@@ -98,7 +96,7 @@ class SGD(ITrainer):
assert
isinstance
(
pass_evaluator
,
api
.
Evaluator
)
out_args
=
api
.
Arguments
.
createArguments
(
0
)
feeder
=
DataFeeder
(
data_types
,
reader_dict
)
feeder
=
DataFeeder
(
topology
.
data_type
()
,
reader_dict
)
for
pass_id
in
xrange
(
num_passes
):
event_handler
(
v2_event
.
BeginPass
(
pass_id
))
...
...
@@ -144,7 +142,7 @@ def __check_train_args__(reader, topology, parameters, event_handler, **kwargs):
raise
TypeError
(
'train_data_reader should be a function, '
'which can return a iterator'
)
if
not
isinstance
(
topology
,
ModelConfig
):
if
not
isinstance
(
topology
,
Topology
):
raise
TypeError
(
'topology should be a model config'
)
if
not
isinstance
(
parameters
,
v2_parameters
.
Parameters
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录