attrs.py 11.3 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.config_parser import *
Q
qijun 已提交
16
__all__ = [
X
xzl 已提交
17 18
    'HookAttr', 'ParamAttr', 'ExtraAttr', 'ParameterAttribute',
    'ExtraLayerAttribute'
Q
qijun 已提交
19
]
Z
zhangjinchao01 已提交
20 21


22
def convert_and_compare(x, Type):
W
wangyanfei01 已提交
23 24 25 26 27 28
    """
    Convert x to be the same type as Type and then convert back to
    check whether there is a loss of information
    :param x: object to be checked
    :param Type: target type to check x over

29
    """
Q
qijun 已提交
30 31
    return type(x)(Type(x)) == x

32 33

def is_compatible_with(x, Type):
W
wangyanfei01 已提交
34 35 36 37 38
    """
    Check if x has a type compatible with Type
    :param x: object to be checked
    :param Type: target type to check x over

39 40 41 42 43
    """
    if type(x) == Type:
        return True
    try:
        if float == Type or int == Type:
W
wangyanfei01 已提交
44 45 46
            # avoid those types that can be converted to float/int but not very
            # meaningful and  could potentially lead to error
            # i.e., str and bool typed value should not be used for initializing float/int variable
47 48 49
            if not isinstance(x, str) and not isinstance(x, bool):
                return convert_and_compare(x, Type)
        elif bool == Type:
W
wangyanfei01 已提交
50
            # should not use string type to initialize bool variable
51 52 53 54 55 56 57 58
            if not isinstance(x, str):
                return convert_and_compare(x, Type)
        else:
            return False
    except:
        return False


X
xzl 已提交
59 60
class HookAttribute(object):
    """
61 62 63 64 65 66 67 68 69 70 71
    Hook Attribute object. As a member of ParameterAttribute class, the hook is an auxiliary operation that occurs 
    during training process of a layer with parameters, such as img_conv layer, fc layer.

    :param  type: Hook type, currently supported types: 
                        'pruning' :  user specify a sparsity_ratio before training started, and the
                            network will prune the parameters based on the sparsity_ratio. 
                            eg: The definition of Hook object can be hk = HookAttribute('pruning', 0.6)
                            The specific usage can be paddle.layer.img_conv(input=img, filter_size=3,
                                                                       num_channels=3, num_filters=64,
                                                                       param_attr=ParameterAttribute(update_hooks=hk) )
                            The pruning deatils can be found https://arxiv.org/pdf/1506.02626.pdf
X
xzl 已提交
72 73
    :type type: string

Z
zlx 已提交
74
    :param sparsity_ratio: Must be specified if hook type is 'pruning', 
75
                        it represents the ratio of the zero elements to be set by the Parameter.
X
xzl 已提交
76
    :type sparsity_ratio: float or None
X
xzl 已提交
77 78 79
	
    """

80
    def __init__(self, type, sparsity_ratio=None):
X
xzl 已提交
81 82
        self.type = type
        self.sparsity_ratio = sparsity_ratio
X
xzl 已提交
83 84 85 86
        if self.sparsity_ratio is not None:
            assert is_compatible_with(
                self.sparsity_ratio,
                float), 'sparisity_ratio must be float type'
Z
zlx 已提交
87
            assert self.sparsity_ratio <= 1 and self.sparsity_ratio >= 0, 'sparsity_ratio must be a float between [0, 1] '
X
xzl 已提交
88 89

    def __call__(self):
90
        return ParameterHook(self.type, sparsity_ratio=self.sparsity_ratio)
X
xzl 已提交
91 92


Z
zhangjinchao01 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
class ParameterAttribute(object):
    """
    Parameter Attributes object. To fine-tuning network training process, user
    can set attribute to control training details, such as l1,l2 rate / learning
    rate / how to init param.

    NOTE: IT IS A HIGH LEVEL USER INTERFACE.

    :param is_static: True if this parameter will be fixed while training.
    :type is_static: bool

    :param initial_std: Gauss Random initialization standard deviation.
                        None if not using Gauss Random initialize parameter.
    :type initial_std: float or None
    :param initial_mean:  Gauss Random initialization mean.
                         None if not using Gauss Random initialize parameter.
    :type initial_mean: float or None
    :param initial_max: Uniform initialization max value.
    :type initial_max: float or None
    :param initial_min: Uniform initialization min value.
    :type initial_min: float or None
    :param l1_rate: the l1 regularization factor
    :type l1_rate: float or None
    :param l2_rate: the l2 regularization factor
    :type l2_rate: float or None
    :param learning_rate: The parameter learning rate. None means 1.
                          The learning rate when optimize is LEARNING_RATE =
                          GLOBAL_LEARNING_RATE * PARAMETER_LEARNING_RATE
                          * SCHEDULER_FACTOR.

    :type learning_rate: float or None
    :param momentum: The parameter momentum. None means use global value.
    :type momentum: float or None
W
wangyanfei01 已提交
126 127 128 129
    :param gradient_clipping_threshold: gradient clipping threshold. If gradient
                                        value larger than some value, will be
                                        clipped.
    :type gradient_clipping_threshold: float
Z
zhangjinchao01 已提交
130 131 132
    :param sparse_update: Enable sparse update for this parameter. It will
                          enable both local and remote sparse update.
    :type sparse_update: bool
X
xuwei06 已提交
133 134 135 136
    :param initializer: If not None, it should be a callable object which accepts
                        a parameter name and returns numpy array for the initial
                        value of the parameter
    :param initializer: callable object
Z
zhangjinchao01 已提交
137 138
    """

Q
qijun 已提交
139 140 141 142 143 144 145 146 147 148 149
    def __init__(self,
                 name=None,
                 is_static=False,
                 initial_std=None,
                 initial_mean=None,
                 initial_max=None,
                 initial_min=None,
                 l1_rate=None,
                 l2_rate=None,
                 learning_rate=None,
                 momentum=None,
W
wangyanfei01 已提交
150
                 gradient_clipping_threshold=None,
X
xzl 已提交
151
                 sparse_update=False,
Z
zlx 已提交
152
                 update_hooks=None,
X
xuwei06 已提交
153
                 initializer=None):
154 155
        self.attr = {}

Z
zhangjinchao01 已提交
156
        if is_static:
157 158 159
            self.attr['is_static'] = True

        if initial_std is None and initial_mean is None and initial_max \
Z
zhangjinchao01 已提交
160
                is None and initial_min is None:
161
            self.attr['initial_smart'] = True
162 163
        elif is_compatible_with(initial_std, float) or \
             is_compatible_with(initial_mean, float):
Z
zhangjinchao01 已提交
164 165 166 167 168
            if initial_std is not None:
                self.attr['initial_std'] = initial_std
            if initial_mean is not None:
                self.attr['initial_mean'] = initial_mean
            self.attr['initial_strategy'] = 0  # Gauss Random
169 170 171 172
        elif is_compatible_with(initial_max, float) and \
             is_compatible_with(initial_min, float):
            initial_max = initial_max
            initial_min = initial_min
Z
zhangjinchao01 已提交
173 174 175 176 177 178 179 180 181
            assert initial_min < initial_max
            initial_mean = (initial_max + initial_min) / 2
            initial_std = initial_mean - initial_min
            self.attr['initial_mean'] = initial_mean
            self.attr['initial_std'] = initial_std
            self.attr['initial_strategy'] = 1  # Uniform Random
        else:
            raise RuntimeError("Unexpected branch.")

182
        if not is_static and is_compatible_with(l1_rate, float):
Z
zhangjinchao01 已提交
183 184
            self.attr['decay_rate_l1'] = l1_rate

185
        if not is_static and is_compatible_with(l2_rate, float):
Z
zhangjinchao01 已提交
186 187
            self.attr['decay_rate'] = l2_rate

188
        if not is_static and is_compatible_with(learning_rate, float):
Z
zhangjinchao01 已提交
189 190
            self.attr['learning_rate'] = learning_rate

191
        if not is_static and is_compatible_with(momentum, float):
Z
zhangjinchao01 已提交
192 193 194 195 196 197 198 199 200
            self.attr['momentum'] = momentum

        if name is not None:
            self.attr['parameter_name'] = name

        if sparse_update:
            self.attr['sparse_update'] = True
            self.attr['sparse_remote_update'] = True

W
wangyanfei01 已提交
201 202 203 204
        if gradient_clipping_threshold is not None and \
                is_compatible_with(gradient_clipping_threshold, float):
            self.attr['gradient_clipping_threshold'] = \
                gradient_clipping_threshold
X
xuwei06 已提交
205 206
        if initializer is not None:
            self.attr['initializer'] = initializer
W
wangyanfei01 已提交
207

X
xzl 已提交
208 209 210
        if update_hooks:
            self.attr['update_hooks'] = update_hooks

Z
zhangjinchao01 已提交
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
    def set_default_parameter_name(self, name):
        """
        Set default parameter name. If parameter not set, then will use default
        parameter name.


        :param name: default parameter name.
        :type name: basestring
        """
        if 'parameter_name' not in self.attr:
            self.attr['parameter_name'] = name

    @staticmethod
    def to_bias(bias_attr):
        if isinstance(bias_attr, ParameterAttribute):
            return Bias(**bias_attr.attr)
        else:
            return False


class ExtraLayerAttribute(object):
    """
    Some high level layer attributes config. You can set all attributes here,
    but some layer doesn't support all attributes. If you set an attribute to a
    layer that not support this attribute, paddle will print an error and core.

    :param error_clipping_threshold: Error clipping threshold.
    :type error_clipping_threshold: float
    :param drop_rate: Dropout rate. Dropout will create a mask on layer output.
                      The dropout rate is the zero rate of this mask. The
                      details of what dropout is please refer to `here
                      <https://www.cs.toronto.edu/~hinton/absps/
243
                      JMLRdropout.pdf>`_.
Z
zhangjinchao01 已提交
244
    :type drop_rate: float
P
Peng Li 已提交
245
    :param device: device ID of layer. device=-1, use CPU. device>=0, use GPU.
246 247 248 249
                   The details allocation in parallel_nn please refer to `here
                   <http://www.paddlepaddle.org/doc/ui/cmd_argument/
                   use_case.html#case-2-specify-layers-in-different-devices>`_.
    :type device: int
Z
zhangjinchao01 已提交
250 251
    """

Q
qijun 已提交
252 253 254 255
    def __init__(self,
                 error_clipping_threshold=None,
                 drop_rate=None,
                 device=None):
Z
zhangjinchao01 已提交
256
        self.attr = dict()
Y
Yu Yang 已提交
257 258 259 260 261 262 263 264 265
        if error_clipping_threshold is not None:
            error_clipping_threshold = float(error_clipping_threshold)
            if error_clipping_threshold < 0:
                raise ValueError("Error clipping must > 0")
            self.attr['error_clipping_threshold'] = error_clipping_threshold
        if drop_rate is not None:
            drop_rate = float(drop_rate)
            if drop_rate < 0:
                raise ValueError("Dropout rate must > 0")
Z
zhangjinchao01 已提交
266 267
            self.attr["drop_rate"] = drop_rate

268 269 270
        if isinstance(device, int):
            self.attr["device"] = device

Z
zhangjinchao01 已提交
271 272 273 274
    def check(self, layer_name):
        for key in self.attr:
            if not hasattr(self, 'can_%s' % key) or \
                    not getattr(self, 'can_%s' % key):
Q
qijun 已提交
275 276
                raise NotImplementedError("Layer %s cannot support %s" %
                                          (layer_name, key))
Z
zhangjinchao01 已提交
277 278 279 280 281 282 283 284 285

    @staticmethod
    def to_kwargs(attr):
        if attr is None:
            return dict()
        else:
            return attr.attr


X
xzl 已提交
286
HookAttr = HookAttribute
Z
zhangjinchao01 已提交
287 288
ParamAttr = ParameterAttribute
ExtraAttr = ExtraLayerAttribute