Merge pull request #1386 from jacquesqiao/data-type

add type to layer.data

demo/mnist/api_train.py

@@ -9,7 +9,6 @@ The user api could be simpler and carefully designed.
 import random
 
 import numpy as np
-import paddle.trainer.PyDataProvider2 as dp
 import paddle.v2 as paddle_v2
 import py_paddle.swig_paddle as api
 from paddle.trainer_config_helpers import *
@@ -71,8 +70,10 @@ def main():
     assert isinstance(updater, api.ParameterUpdater)
 
     # define network
-    images = paddle_v2.layer.data(name='pixel', size=784)
-    label = paddle_v2.layer.data(name='label', size=10)
+    images = paddle_v2.layer.data(
+        name='pixel', type=paddle_v2.data_type.dense_vector(784))
+    label = paddle_v2.layer.data(
+        name='label', type=paddle_v2.data_type.integer_value(10))
     hidden1 = paddle_v2.layer.fc(input=images, size=200)
     hidden2 = paddle_v2.layer.fc(input=hidden1, size=200)
     inference = paddle_v2.layer.fc(input=hidden2,
@@ -98,8 +99,7 @@ def main():
 
     # DataProvider Converter is a utility convert Python Object to Paddle C++
     # Input. The input format is as same as Paddle's DataProvider.
-    converter = DataProviderConverter(
-        input_types=[dp.dense_vector(784), dp.integer_value(10)])
+    converter = DataProviderConverter(input_types=[images.type, label.type])
 
     train_file = './data/raw_data/train'
     test_file = './data/raw_data/t10k'

demo/mnist/api_train_v2.py

 import numpy
 import paddle.v2 as paddle
-from paddle.trainer.PyDataProvider2 import dense_vector, integer_value
 
 import mnist_util
 
@@ -16,8 +15,10 @@ def main():
     paddle.init(use_gpu=False, trainer_count=1)
 
     # define network topology
-    images = paddle.layer.data(name='pixel', size=784)
-    label = paddle.layer.data(name='label', size=10)
+    images = paddle.layer.data(
+        name='pixel', type=paddle.data_type.dense_vector(784))
+    label = paddle.layer.data(
+        name='label', type=paddle.data_type.integer_value(10))
     hidden1 = paddle.layer.fc(input=images, size=200)
     hidden2 = paddle.layer.fc(input=hidden1, size=200)
     inference = paddle.layer.fc(input=hidden2,
@@ -51,8 +52,8 @@ def main():
                   batch_size=32,  # batch size should be refactor in Data reader
                   data_types={  # data_types will be removed, It should be in
                       # network topology
-                      'pixel': dense_vector(784),
-                      'label': integer_value(10)
+                      'pixel': images.type,
+                      'label': label.type
                   })
 
 

python/paddle/v2/__init__.py

@@ -17,10 +17,12 @@ import activation
 import parameters
 import trainer
 import event
+import data_type
 import py_paddle.swig_paddle as api
 
 __all__ = [
-    'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', 'event'
+    'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer',
+    'event', 'data_type.py'
 ]
 
 

python/paddle/v2/data_type.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle.trainer.PyDataProvider2 import \
+    InputType, dense_vector, sparse_binary_vector,\
+    sparse_vector, integer_value
+
+__all__ = [
+    'InputType', 'dense_vector', 'sparse_binary_vector', 'sparse_vector',
+    'integer_value'
+]

python/paddle/v2/layer.py

@@ -67,6 +67,7 @@ paddle.v2.parameters.create, no longer exposed to users.
 """
 
 import paddle.trainer_config_helpers as conf_helps
+from . import data_type as v2_data
 from paddle.trainer_config_helpers.config_parser_utils import \
     parse_network_config as __parse__
 from paddle.trainer_config_helpers.default_decorators import wrap_name_default
@@ -157,7 +158,33 @@ def __convert_to_v2__(method_name, name_prefix, parent_names):
     return V2LayerImpl
 
 
-data = __convert_to_v2__('data_layer', None, [])
+"""
+Some layer may need some special config, and can not use __convert_to_v2__ to convert.
+So we also need to implement some special LayerV2.
+"""
+
+
+class DataLayerV2(Layer):
+    def __init__(self, name, type, **kwargs):
+        assert isinstance(type, v2_data.InputType)
+
+        self.type = type
+        self.__method_name__ = 'data_layer'
+        self.__kwargs__ = kwargs
+
+        super(DataLayerV2, self).__init__(name=name, parent_layers=dict())
+
+    def to_proto_impl(self, **kwargs):
+        args = dict()
+        args['size'] = self.type.dim
+        for each in kwargs:
+            args[each] = kwargs[each]
+        for each in self.__kwargs__:
+            args[each] = self.__kwargs__[each]
+        return getattr(conf_helps, self.__method_name__)(name=self.name, **args)
+
+
+data = DataLayerV2
 fc = __convert_to_v2__('fc_layer', name_prefix='fc', parent_names=['input'])
 max_id = __convert_to_v2__(
     'maxid_layer', name_prefix='maxid_layer', parent_names=['input'])
@@ -171,8 +198,8 @@ cross_entropy_cost = __convert_to_v2__(
     parent_names=['input', 'label'])
 
 if __name__ == '__main__':
-    pixel = data(name='pixel', size=784)
-    label = data(name='label', size=10)
+    pixel = data(name='pixel', type=v2_data.dense_vector(784))
+    label = data(name='label', type=v2_data.integer_value(10))
     hidden = fc(input=pixel, size=100, act=conf_helps.SigmoidActivation())
     inference = fc(input=hidden, size=10, act=conf_helps.SoftmaxActivation())
     maxid = max_id(input=inference)