From 40427979a7f621257d7171dfaefeb3afe9a0b45e Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 27 Feb 2017 23:36:28 +0800 Subject: [PATCH] refine code --- demo/mnist/api_train_v2.py | 6 ++-- python/paddle/v2/data_feeder.py | 2 +- python/paddle/v2/data_type.py | 6 ++-- python/paddle/v2/layer.py | 1 + python/paddle/v2/parameters.py | 4 +-- python/paddle/v2/tests/CMakeLists.txt | 2 +- .../{topology_test.py => test_topology.py} | 21 +++++++++----- python/paddle/v2/topology.py | 29 ++++++++----------- python/paddle/v2/trainer.py | 14 +++------ 9 files changed, 41 insertions(+), 44 deletions(-) rename python/paddle/v2/tests/{topology_test.py => test_topology.py} (81%) diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index cb1191eddd..a23ddfaca0 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -41,10 +41,12 @@ def main(): trainer.train( train_data_reader=train_reader, - topology=[cost], + topology=cost, parameters=parameters, event_handler=event_handler, - batch_size=32) # batch size should be refactor in Data reader + batch_size=32, # batch size should be refactor in Data reader + reader_dict={images.name: 0, + label.name: 1}) if __name__ == '__main__': diff --git a/python/paddle/v2/data_feeder.py b/python/paddle/v2/data_feeder.py index 2a16d46dda..3b106e100c 100644 --- a/python/paddle/v2/data_feeder.py +++ b/python/paddle/v2/data_feeder.py @@ -23,7 +23,7 @@ class DataFeeder(DataProviderConverter): """ DataFeeder converts the data returned by paddle.reader into a data structure of Arguments which is defined in the API. The paddle.reader usually returns - a list of mini-batch data entries. Each data entry in the list is one sampe. + a list of mini-batch data entries. Each data entry in the list is one sample. Each sample is a list or a tuple with one feature or multiple features. DataFeeder converts this mini-batch data entries into Arguments in order to feed it to C++ interface. diff --git a/python/paddle/v2/data_type.py b/python/paddle/v2/data_type.py index dd3ebfcb42..522ddfdaac 100644 --- a/python/paddle/v2/data_type.py +++ b/python/paddle/v2/data_type.py @@ -13,10 +13,10 @@ # limitations under the License. from paddle.trainer.PyDataProvider2 import \ - InputType, dense_vector, sparse_binary_vector,\ + InputType, DataType, dense_vector, sparse_binary_vector,\ sparse_vector, integer_value, integer_value_sequence __all__ = [ - 'InputType', 'dense_vector', 'sparse_binary_vector', 'sparse_vector', - 'integer_value', 'integer_value_sequence' + 'InputType', 'DataType', 'dense_vector', 'sparse_binary_vector', + 'sparse_vector', 'integer_value', 'integer_value_sequence' ] diff --git a/python/paddle/v2/layer.py b/python/paddle/v2/layer.py index d15e6398f5..faf5b8bd87 100644 --- a/python/paddle/v2/layer.py +++ b/python/paddle/v2/layer.py @@ -284,6 +284,7 @@ def mixed(size=0, return MixedLayerV2(size, input, name, act, bias_attr, layer_attr) +LayerV2 = Layer data = DataLayerV2 AggregateLevel = conf_helps.layers.AggregateLevel ExpandLevel = conf_helps.layers.ExpandLevel diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index b8d4b28703..7c3cde7727 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -2,7 +2,7 @@ import numpy as np import py_paddle.swig_paddle as api from paddle.proto.ParameterConfig_pb2 import ParameterConfig -import topology as v2_topology +from topology import Topology __all__ = ['Parameters', 'create'] @@ -13,7 +13,7 @@ def create(layers): :param layers: :return: """ - topology = v2_topology.Topology(layers) + topology = Topology(layers) pool = Parameters() for param in topology.proto().parameters: pool.__append_config__(param) diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index c77df827d9..46b5d08b87 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -8,5 +8,5 @@ add_test(NAME test_v2_api add_test(NAME topology_test COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/topology_test.py + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) diff --git a/python/paddle/v2/tests/topology_test.py b/python/paddle/v2/tests/test_topology.py similarity index 81% rename from python/paddle/v2/tests/topology_test.py rename to python/paddle/v2/tests/test_topology.py index be60a577be..1bf55a5bc6 100644 --- a/python/paddle/v2/tests/topology_test.py +++ b/python/paddle/v2/tests/test_topology.py @@ -30,14 +30,19 @@ class TestTopology(unittest.TestCase): act=conf_helps.SoftmaxActivation()) cost = layer.classification_cost(input=inference, label=label) topo = topology.Topology(cost) - type = topo.data_type() - self.assertEqual(len(type), 2) - self.assertEqual(type[0][0], "pixel") - self.assertEqual(type[0][1].type, data_type.DataType.Dense) - self.assertEqual(type[0][1].dim, 784) - self.assertEqual(type[1][0], "label") - self.assertEqual(type[1][1].type, data_type.DataType.Index) - self.assertEqual(type[1][1].dim, 10) + data_types = topo.data_type() + self.assertEqual(len(data_types), 2) + pixel_data_type = filter(lambda type: type[0] == "pixel", data_types) + self.assertEqual(len(pixel_data_type), 1) + pixel_data_type = pixel_data_type[0] + self.assertEqual(pixel_data_type[1].type, data_type.DataType.Dense) + self.assertEqual(pixel_data_type[1].dim, 784) + + label_data_type = filter(lambda type: type[0] == "label", data_types) + self.assertEqual(len(label_data_type), 1) + label_data_type = label_data_type[0] + self.assertEqual(label_data_type[1].type, data_type.DataType.Index) + self.assertEqual(label_data_type[1].dim, 10) def test_get_layer(self): pixel = layer.data(name='pixel', type=data_type.dense_vector(784)) diff --git a/python/paddle/v2/topology.py b/python/paddle/v2/topology.py index 9c57f1f8e6..a51b1073b4 100644 --- a/python/paddle/v2/topology.py +++ b/python/paddle/v2/topology.py @@ -49,30 +49,30 @@ class Topology(object): result_layer = [] def find_layer_by_name(layer, layer_name): - if layer.name == layer_name and len(result_layer) == 0: + if len(result_layer) == 1: + return + elif layer.name == layer_name: result_layer.append(layer) - for parent_layer in layer.__parent_layers__.values(): - find_layer_by_name(parent_layer, layer_name) + else: + for parent_layer in layer.__parent_layers__.values(): + find_layer_by_name(parent_layer, layer_name) for layer in self.layers: find_layer_by_name(layer, name) + assert len(result_layer) == 1 return result_layer[0] - def data_layer(self): + def data_layers(self): """ get all data layer :return: """ - data_layers = [] + data_layers = set() def find_data_layer(layer): - assert isinstance(layer, layer.LayerV2) if isinstance(layer, v2_layer.DataLayerV2): - if len( - filter(lambda data_layer: data_layer.name == layer.name, - data_layers)) == 0: - data_layers.append(layer) + data_layers.add(layer) for parent_layer in layer.__parent_layers__.values(): find_data_layer(parent_layer) @@ -85,14 +85,9 @@ class Topology(object): """ get data_type from proto, such as: [('image', dense_vector(768)), ('label', integer_value(10))] - the order is the same with __model_config__.input_layer_names """ - data_types_lists = [] - for layer_name in self.__model_config__.input_layer_names: - data_types_lists.append( - (layer_name, self.get_layer(layer_name).type)) - - return data_types_lists + return [(data_layer.name, data_layer.type) + for data_layer in self.data_layers()] def __check_layer_type__(layer): diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 96a3ee4fd4..3bf2128e16 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -1,13 +1,12 @@ import collections import py_paddle.swig_paddle as api -from py_paddle import DataProviderConverter from data_feeder import DataFeeder +from topology import Topology from . import event as v2_event from . import optimizer as v2_optimizer from . import parameters as v2_parameters -from . import topology as v2_topology __all__ = ['ITrainer', 'SGD'] @@ -69,7 +68,6 @@ class SGD(ITrainer): test_data_reader=None, event_handler=None, batch_size=32, - data_types=None, reader_dict=None): """ Training method. Will train num_passes of input data. @@ -83,13 +81,12 @@ class SGD(ITrainer): occurred. :type event_handler: (BaseEvent) => None :param batch_size: Not important, will be removed after data refactor. - :param data_types: Not important, will be removed after data refactor. :return: """ if event_handler is None: event_handler = default_event_handler - topology = v2_topology.Topology(topology) + topology = Topology(topology) __check_train_args__(**locals()) @@ -109,10 +106,7 @@ class SGD(ITrainer): assert isinstance(pass_evaluator, api.Evaluator) out_args = api.Arguments.createArguments(0) - data_types_lists = [data_type[1] for data_type in topology.data_type()] - converter = DataProviderConverter(input_types=data_types_lists) - - feeder = DataFeeder(data_types, reader_dict) + feeder = DataFeeder(topology.data_type(), reader_dict) for pass_id in xrange(num_passes): event_handler(v2_event.BeginPass(pass_id)) @@ -195,7 +189,7 @@ def __check_train_args__(train_data_reader, topology, parameters, raise ValueError('test_data_reader should be a function, which can ' 'return a iterator') - if not isinstance(topology, v2_topology.Topology): + if not isinstance(topology, Topology): raise ValueError('topology should be a model config') if not isinstance(parameters, v2_parameters.Parameters): -- GitLab