layer.py 11.7 KB
Newer Older
Q
qiaolongfei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
"""
Before this new package paddle.v2.layer, users would need to use functions
in paddle.trainer_config_helpers.layers to configure networks.

The Old Way:
=========
This old way requires that the creation of a network be defined in a Python
function, say network_config, and that this Python function being passed to
paddle.trainer_config_helpers.parse_network_config for the creation of
protobuf message description of this network.

```python
def network_config():
  img = paddle.trainer_config_helpers.data_layer(name="pixel", size=784)
  inference = paddle.trainer_config_helpers.fc_layer(
    input=img,
    size=10,
    act=paddle.trainer_config_helpers.SoftmaxActivation())
  cost = paddle.trainer_config_helpers.classification_cost(
    input=inference,
    label=paddle.trainer_config_helpers.data_layer(name="label", size=10))

proto_desc = parse_network_config(network_config)
```

When parse_network_config executes network_config, those layer definition
functions like data_layer and fc_layer would change some Python global variables,
so that after the execution, parse_network_config could collect information from
these global variables and generates the protobuf message.



The New Way:
=========
In this PR, we define a function in paddle.v2.layer which creates a Python
class for each layer creation function in paddle.trainer_config_helpers.layers.
Users can use create a network as follows:

```python
img = paddle.v2.layer.data(name="pixel", size=784)
inference = paddle.v2.layer.fc(input=img, size=10, act=paddle.v2.layer.Softmax())
cost = paddle.v2.layer.classification(
  input=inference,
  label=paddle.v2.layer.data(name="label", size=10))

parameters = paddle.v2.parameters.create(cost)
```

This new way doesn't require those invocations to layer definition functions
to be in a Python function but could be anywhere.

Also, the creation of a protobuf message is hidden in the invocation of
paddle.v2.parameters.create, no longer exposed to users.
"""
Q
qiaolongfei 已提交
68

Q
qiaolongfei 已提交
69 70
import collections

Q
qiaolongfei 已提交
71 72 73 74
import paddle.trainer_config_helpers as conf_helps
from paddle.trainer_config_helpers.config_parser_utils import \
    parse_network_config as __parse__
from paddle.trainer_config_helpers.default_decorators import wrap_name_default
Q
qiaolongfei 已提交
75

Q
qiaolongfei 已提交
76
import activation
Q
qiaolongfei 已提交
77
import data_type
L
Luo Tao 已提交
78 79
import activation
import attr
Q
qiaolongfei 已提交
80

Q
qiaolongfei 已提交
81 82
__all__ = [
    'parse_network', 'data', 'fc', 'max_id', 'classification_cost',
L
Luo Tao 已提交
83 84 85
    'cross_entropy_cost', 'cross_entropy_with_selfnorm_cost', 'regression_cost',
    'multi_binary_label_cross_entropy_cost', 'rank_cost', 'lambda_cost',
    'sum_cost', 'huber_cost'
Q
qiaolongfei 已提交
86 87
]

Q
qiaolongfei 已提交
88

Q
qiaolongfei 已提交
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
def parse_network(*outputs):
    """
    parse all output layers and then generate a model config proto.
    :param outputs:
    :return:
    """

    def __real_func__():
        context = dict()
        real_output = [each.to_proto(context=context) for each in outputs]
        conf_helps.outputs(real_output)

    return __parse__(__real_func__)


Q
qiaolongfei 已提交
104
class Layer(object):
Q
qiaolongfei 已提交
105
    def __init__(self, name, parent_layers, step_input=None):
Q
qiaolongfei 已提交
106
        assert isinstance(parent_layers, dict)
Q
qiaolongfei 已提交
107 108
        assert isinstance(name, basestring)
        self.name = name
Q
qiaolongfei 已提交
109
        self.step_input = step_input
Q
qiaolongfei 已提交
110
        self.__parent_layers__ = parent_layers
Q
qiaolongfei 已提交
111 112 113 114 115 116

    def to_proto(self, context):
        """
        function to set proto attribute
        """
        kwargs = dict()
Q
qiaolongfei 已提交
117 118
        for layer_name in self.__parent_layers__:
            if not isinstance(self.__parent_layers__[layer_name],
Q
qiaolongfei 已提交
119
                              collections.Sequence):
Q
qiaolongfei 已提交
120
                v1_layer = self.__parent_layers__[layer_name].to_proto(
Q
qiaolongfei 已提交
121 122
                    context=context)
            else:
Q
qiaolongfei 已提交
123 124
                v1_layer = map(lambda x: x.to_proto(context=context),
                               self.__parent_layers__[layer_name])
Q
qiaolongfei 已提交
125 126
                if layer_name == "input" and self.step_input is not None:
                    v1_layer.insert(0, self.step_input)
Q
qiaolongfei 已提交
127
            kwargs[layer_name] = v1_layer
Q
qiaolongfei 已提交
128

Q
qiaolongfei 已提交
129 130 131 132
        # memory may have the same name with some layer
        if isinstance(self, MemoryV2):
            return self.to_proto_impl(**kwargs)

Q
qiaolongfei 已提交
133 134 135 136 137 138 139 140
        if self.name not in context:
            context[self.name] = self.to_proto_impl(**kwargs)
        return context[self.name]

    def to_proto_impl(self, **kwargs):
        raise NotImplementedError()


Q
qiaolongfei 已提交
141
def __convert_to_v2__(method_name, name_prefix, parent_names):
Q
qiaolongfei 已提交
142 143 144 145 146
    if name_prefix is not None:
        wrapper = wrap_name_default(name_prefix=name_prefix)
    else:
        wrapper = None

Q
qiaolongfei 已提交
147
    class V2LayerImpl(Layer):
Q
qiaolongfei 已提交
148
        def __init__(self, name=None, step_input=None, **kwargs):
Q
qiaolongfei 已提交
149 150 151
            parent_layers = dict()
            other_kwargs = dict()
            for pname in parent_names:
L
Luo Tao 已提交
152 153
                if kwargs.has_key(pname):
                    parent_layers[pname] = kwargs[pname]
Q
qiaolongfei 已提交
154 155 156 157 158

            for key in kwargs.keys():
                if key not in parent_names:
                    other_kwargs[key] = kwargs[key]

Q
qiaolongfei 已提交
159
            super(V2LayerImpl, self).__init__(name, parent_layers, step_input)
Q
qiaolongfei 已提交
160 161 162 163 164 165 166 167 168 169 170 171 172
            self.__other_kwargs__ = other_kwargs

        if wrapper is not None:
            __init__ = wrapper(__init__)

        def to_proto_impl(self, **kwargs):
            args = dict()
            for each in kwargs:
                args[each] = kwargs[each]
            for each in self.__other_kwargs__:
                args[each] = self.__other_kwargs__[each]
            return getattr(conf_helps, method_name)(name=self.name, **args)

Q
qiaolongfei 已提交
173
    return V2LayerImpl
Q
qiaolongfei 已提交
174 175


Q
qiaolongfei 已提交
176 177 178 179 180 181 182
"""
Some layer may need some special config, and can not use __convert_to_v2__ to convert.
So we also need to implement some special LayerV2.
"""


class DataLayerV2(Layer):
Q
qiaolongfei 已提交
183
    def __init__(self, name, type, **kwargs):
184
        assert isinstance(type, data_type.InputType)
Q
qiaolongfei 已提交
185

Q
qiaolongfei 已提交
186
        self.type = type
Q
qiaolongfei 已提交
187 188
        self.__method_name__ = 'data_layer'
        self.__kwargs__ = kwargs
Q
qiaolongfei 已提交
189 190 191 192 193

        super(DataLayerV2, self).__init__(name=name, parent_layers=dict())

    def to_proto_impl(self, **kwargs):
        args = dict()
Q
qiaolongfei 已提交
194
        args['size'] = self.type.dim
Q
qiaolongfei 已提交
195 196
        for each in kwargs:
            args[each] = kwargs[each]
Q
qiaolongfei 已提交
197 198
        for each in self.__kwargs__:
            args[each] = self.__kwargs__[each]
Q
qiaolongfei 已提交
199 200 201
        return getattr(conf_helps, self.__method_name__)(name=self.name, **args)


Q
qiaolongfei 已提交
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
class MemoryV2(Layer):
    def __init__(self, name, size, **kwargs):
        self.name = name
        self.size = size
        self.__kwargs__ = kwargs
        super(MemoryV2, self).__init__(name=name, parent_layers=dict())

    def to_proto_impl(self, **kwargs):
        args = dict()
        for each in kwargs:
            args[each] = kwargs[each]
        for each in self.__kwargs__:
            args[each] = self.__kwargs__[each]
        return conf_helps.memory(name=self.name, size=self.size, **args)


Q
qiaolongfei 已提交
218
data = DataLayerV2
Q
qiaolongfei 已提交
219 220
fc = __convert_to_v2__('fc_layer', name_prefix='fc', parent_names=['input'])
max_id = __convert_to_v2__(
L
Luo Tao 已提交
221
    'maxid_layer', name_prefix='maxid', parent_names=['input'])
Q
qiaolongfei 已提交
222
classification_cost = __convert_to_v2__(
Q
qiaolongfei 已提交
223 224
    'classification_cost',
    name_prefix='classification_cost',
L
Luo Tao 已提交
225 226 227 228 229
    parent_names=['input', 'label', 'weight'])
regression_cost = __convert_to_v2__(
    'regression_cost',
    name_prefix='regression_cost',
    parent_names=['input', 'label', 'weight'])
Q
qiaolongfei 已提交
230 231 232 233
cross_entropy_cost = __convert_to_v2__(
    'cross_entropy',
    name_prefix='cross_entropy',
    parent_names=['input', 'label'])
Q
qiaolongfei 已提交
234 235 236 237 238 239 240
embedding = __convert_to_v2__(
    'embedding_layer', name_prefix='embedding', parent_names=['input'])
last_seq = __convert_to_v2__(
    'last_seq', name_prefix='last_seq', parent_names=['input'])
recurrent_group = __convert_to_v2__(
    'recurrent_group', name_prefix='recurrent_layer', parent_names=['input'])
memory = MemoryV2
Q
qiaolongfei 已提交
241

L
Luo Tao 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
cross_entropy_with_selfnorm_cost = __convert_to_v2__(
    'cross_entropy_with_selfnorm',
    name_prefix='cross_entropy_with_selfnorm',
    parent_names=['input', 'label'])
multi_binary_label_cross_entropy_cost = __convert_to_v2__(
    'multi_binary_label_cross_entropy',
    name_prefix='multi_binary_label_cross_entropy',
    parent_names=['input', 'label'])
rank_cost = __convert_to_v2__(
    'rank_cost',
    name_prefix='rank_cost',
    parent_names=['left', 'right', 'label', 'weight'])
lambda_cost = __convert_to_v2__(
    'lambda_cost', name_prefix='lambda_cost', parent_names=['input', 'score'])
sum_cost = __convert_to_v2__(
    'sum_cost', name_prefix='sum_cost', parent_names=['input'])
huber_cost = __convert_to_v2__(
    'huber_cost', name_prefix='huber_cost', parent_names=['input', 'label'])
Q
qiaolongfei 已提交
260 261

if __name__ == '__main__':
262 263
    pixel = data(name='pixel', type=data_type.dense_vector(784))
    label = data(name='label', type=data_type.integer_value(10))
L
Luo Tao 已提交
264 265 266
    weight = data(name='weight', type=data_type.dense_vector(10))
    score = data(name='score', type=data_type.dense_vector(1))

L
Luo Tao 已提交
267 268 269 270 271
    hidden = fc(input=pixel,
                size=100,
                act=activation.Sigmoid(),
                param_attr=attr.Param(name='hidden'))
    inference = fc(input=hidden, size=10, act=activation.Softmax())
Q
qiaolongfei 已提交
272 273
    maxid = max_id(input=inference)
    cost1 = classification_cost(input=inference, label=label)
L
Luo Tao 已提交
274 275 276 277 278 279 280 281 282 283
    cost2 = classification_cost(input=inference, label=label, weight=weight)
    cost3 = cross_entropy_cost(input=inference, label=label)
    cost4 = cross_entropy_with_selfnorm_cost(input=inference, label=label)
    cost5 = regression_cost(input=inference, label=label)
    cost6 = regression_cost(input=inference, label=label, weight=weight)
    cost7 = multi_binary_label_cross_entropy_cost(input=inference, label=label)
    cost8 = rank_cost(left=score, right=score, label=score)
    cost9 = lambda_cost(input=inference, score=score)
    cost10 = sum_cost(input=inference)
    cost11 = huber_cost(input=score, label=label)
Q
qiaolongfei 已提交
284

Q
qiaolongfei 已提交
285 286 287 288 289 290 291
    mem = memory(name="rnn_state", size=10)

    # print parse_network(cost1)
    # print parse_network(cost2)
    # print parse_network(cost1, cost2)
    # print parse_network(cost2)
    # print parse_network(inference, maxid)
Q
qiaolongfei 已提交
292
    print parse_network(cost1, cost2)
L
Luo Tao 已提交
293 294 295
    print parse_network(cost3, cost4)
    print parse_network(cost5, cost6)
    print parse_network(cost7, cost8, cost9, cost10, cost11)
Q
qiaolongfei 已提交
296
    print parse_network(inference, maxid)
Q
qiaolongfei 已提交
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334

    dict_dim = 10
    word_dim = 8
    hidden_dim = 8
    label_dim = 3

    def step(y):
        mem = conf_helps.memory(name="rnn_state", size=hidden_dim)
        out = conf_helps.fc_layer(
            input=[y, mem],
            size=hidden_dim,
            act=activation.Tanh(),
            bias_attr=True,
            name="rnn_state")
        return out

    def test():
        data1 = conf_helps.data_layer(name="word", size=dict_dim)
        embd = conf_helps.embedding_layer(input=data1, size=word_dim)
        conf_helps.recurrent_group(name="rnn", step=step, input=embd)

    # print __parse__(test)

    # yyyyyyyy
    def new_step(y):
        mem = memory(name="rnn_state", size=hidden_dim)
        out = fc(input=[mem],
                 step_input=y,
                 size=hidden_dim,
                 act=activation.Tanh(),
                 bias_attr=True,
                 name="rnn_state")
        return out.to_proto(dict())

    data1 = data(name="word", type=data_type.integer_value(dict_dim))
    embd = embedding(input=data1, size=word_dim)
    aaa = recurrent_group(name="rnn", step=new_step, input=embd)
    print parse_network(aaa)