attrs.py 10.7 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
2 3 4 5 6 7 8 9 10 11 12 13 14 15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.config_parser import *
16
__all__ = [
X
xzl 已提交
17 18
    'HookAttr', 'ParamAttr', 'ExtraAttr', 'ParameterAttribute',
    'ExtraLayerAttribute'
19
]
20 21


22
def convert_and_compare(x, Type):
W
wangyanfei01 已提交
23 24 25 26 27 28
    """
    Convert x to be the same type as Type and then convert back to
    check whether there is a loss of information
    :param x: object to be checked
    :param Type: target type to check x over

29
    """
30 31
    return type(x)(Type(x)) == x

32 33

def is_compatible_with(x, Type):
W
wangyanfei01 已提交
34 35 36 37 38
    """
    Check if x has a type compatible with Type
    :param x: object to be checked
    :param Type: target type to check x over

39 40 41 42 43
    """
    if type(x) == Type:
        return True
    try:
        if float == Type or int == Type:
W
wangyanfei01 已提交
44 45 46
            # avoid those types that can be converted to float/int but not very
            # meaningful and  could potentially lead to error
            # i.e., str and bool typed value should not be used for initializing float/int variable
47 48 49
            if not isinstance(x, str) and not isinstance(x, bool):
                return convert_and_compare(x, Type)
        elif bool == Type:
W
wangyanfei01 已提交
50
            # should not use string type to initialize bool variable
51 52 53 54 55 56 57 58
            if not isinstance(x, str):
                return convert_and_compare(x, Type)
        else:
            return False
    except:
        return False


X
xzl 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
class HookAttribute(object):
    """
    Hook Attribute object. The hook is an auxiliary operation that occurs 
    during network propagation. Such as pruning operation, It will cut off 
    redundant parameters in the network before training. More detail can see 
    here paddle/parameter/ParameterUpdaterHook.cpp
    NOTE: IT IS A HIGH LEVEL USER INTERFACE.
		
    :param  type: Hook type, eg: 'pruning', 'pruning_static'
    :type type: string

    :param mask_file: Must be specified if hook type is 'pruning_static', 
					  the network reads the mask from the file to determine which parameters should be cut off
    :type mask_file: string

    :param sparsity_ratio: Must be specified if hook type is 'pruning',
	                       the network will hold the sparsity_ratio maximum parameters, and cut off the rest. 
    :type sparsity_ratio: float number between 0 and 1
	
    """

    def __init__(self, type, mask_filename=None, sparsity_ratio=None):
        self.type = type
        self.mask_filename = mask_filename
        self.sparsity_ratio = sparsity_ratio
        assert is_compatible_with(self.sparsity_ratio,
                                  float), 'sparisity_ratio must be float type'
        assert self.sparsity_ratio <= 1 and self.sparsity_ratio >= 0, 'sparisity must be a flaot between [0, 1] '

    def __call__(self):
        return ParameterHook(
            self.type,
            mask_filename=self.mask_filename,
            sparsity_ratio=self.sparsity_ratio)


95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
class ParameterAttribute(object):
    """
    Parameter Attributes object. To fine-tuning network training process, user
    can set attribute to control training details, such as l1,l2 rate / learning
    rate / how to init param.

    NOTE: IT IS A HIGH LEVEL USER INTERFACE.

    :param is_static: True if this parameter will be fixed while training.
    :type is_static: bool

    :param initial_std: Gauss Random initialization standard deviation.
                        None if not using Gauss Random initialize parameter.
    :type initial_std: float or None
    :param initial_mean:  Gauss Random initialization mean.
                         None if not using Gauss Random initialize parameter.
    :type initial_mean: float or None
    :param initial_max: Uniform initialization max value.
    :type initial_max: float or None
    :param initial_min: Uniform initialization min value.
    :type initial_min: float or None
    :param l1_rate: the l1 regularization factor
    :type l1_rate: float or None
    :param l2_rate: the l2 regularization factor
    :type l2_rate: float or None
    :param learning_rate: The parameter learning rate. None means 1.
                          The learning rate when optimize is LEARNING_RATE =
                          GLOBAL_LEARNING_RATE * PARAMETER_LEARNING_RATE
                          * SCHEDULER_FACTOR.

    :type learning_rate: float or None
    :param momentum: The parameter momentum. None means use global value.
    :type momentum: float or None
W
wangyanfei01 已提交
128 129 130 131
    :param gradient_clipping_threshold: gradient clipping threshold. If gradient
                                        value larger than some value, will be
                                        clipped.
    :type gradient_clipping_threshold: float
132 133 134 135 136
    :param sparse_update: Enable sparse update for this parameter. It will
                          enable both local and remote sparse update.
    :type sparse_update: bool
    """

137 138 139 140 141 142 143 144 145 146 147
    def __init__(self,
                 name=None,
                 is_static=False,
                 initial_std=None,
                 initial_mean=None,
                 initial_max=None,
                 initial_min=None,
                 l1_rate=None,
                 l2_rate=None,
                 learning_rate=None,
                 momentum=None,
W
wangyanfei01 已提交
148
                 gradient_clipping_threshold=None,
X
xzl 已提交
149 150
                 sparse_update=False,
                 update_hooks=None):
151 152
        self.attr = {}

153
        if is_static:
154 155 156
            self.attr['is_static'] = True

        if initial_std is None and initial_mean is None and initial_max \
157
                is None and initial_min is None:
158
            self.attr['initial_smart'] = True
159 160
        elif is_compatible_with(initial_std, float) or \
             is_compatible_with(initial_mean, float):
161 162 163 164 165
            if initial_std is not None:
                self.attr['initial_std'] = initial_std
            if initial_mean is not None:
                self.attr['initial_mean'] = initial_mean
            self.attr['initial_strategy'] = 0  # Gauss Random
166 167 168 169
        elif is_compatible_with(initial_max, float) and \
             is_compatible_with(initial_min, float):
            initial_max = initial_max
            initial_min = initial_min
170 171 172 173 174 175 176 177 178
            assert initial_min < initial_max
            initial_mean = (initial_max + initial_min) / 2
            initial_std = initial_mean - initial_min
            self.attr['initial_mean'] = initial_mean
            self.attr['initial_std'] = initial_std
            self.attr['initial_strategy'] = 1  # Uniform Random
        else:
            raise RuntimeError("Unexpected branch.")

179
        if not is_static and is_compatible_with(l1_rate, float):
180 181
            self.attr['decay_rate_l1'] = l1_rate

182
        if not is_static and is_compatible_with(l2_rate, float):
183 184
            self.attr['decay_rate'] = l2_rate

185
        if not is_static and is_compatible_with(learning_rate, float):
186 187
            self.attr['learning_rate'] = learning_rate

188
        if not is_static and is_compatible_with(momentum, float):
189 190 191 192 193 194 195 196 197
            self.attr['momentum'] = momentum

        if name is not None:
            self.attr['parameter_name'] = name

        if sparse_update:
            self.attr['sparse_update'] = True
            self.attr['sparse_remote_update'] = True

W
wangyanfei01 已提交
198 199 200 201 202
        if gradient_clipping_threshold is not None and \
                is_compatible_with(gradient_clipping_threshold, float):
            self.attr['gradient_clipping_threshold'] = \
                gradient_clipping_threshold

X
xzl 已提交
203 204 205
        if update_hooks:
            self.attr['update_hooks'] = update_hooks

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
    def set_default_parameter_name(self, name):
        """
        Set default parameter name. If parameter not set, then will use default
        parameter name.


        :param name: default parameter name.
        :type name: basestring
        """
        if 'parameter_name' not in self.attr:
            self.attr['parameter_name'] = name

    @staticmethod
    def to_bias(bias_attr):
        if isinstance(bias_attr, ParameterAttribute):
            return Bias(**bias_attr.attr)
        else:
            return False


class ExtraLayerAttribute(object):
    """
    Some high level layer attributes config. You can set all attributes here,
    but some layer doesn't support all attributes. If you set an attribute to a
    layer that not support this attribute, paddle will print an error and core.

    :param error_clipping_threshold: Error clipping threshold.
    :type error_clipping_threshold: float
    :param drop_rate: Dropout rate. Dropout will create a mask on layer output.
                      The dropout rate is the zero rate of this mask. The
                      details of what dropout is please refer to `here
                      <https://www.cs.toronto.edu/~hinton/absps/
238
                      JMLRdropout.pdf>`_.
239
    :type drop_rate: float
P
Peng Li 已提交
240
    :param device: device ID of layer. device=-1, use CPU. device>=0, use GPU.
241 242 243 244
                   The details allocation in parallel_nn please refer to `here
                   <http://www.paddlepaddle.org/doc/ui/cmd_argument/
                   use_case.html#case-2-specify-layers-in-different-devices>`_.
    :type device: int
245 246
    """

247 248 249 250
    def __init__(self,
                 error_clipping_threshold=None,
                 drop_rate=None,
                 device=None):
251
        self.attr = dict()
252 253 254 255 256 257 258 259 260
        if error_clipping_threshold is not None:
            error_clipping_threshold = float(error_clipping_threshold)
            if error_clipping_threshold < 0:
                raise ValueError("Error clipping must > 0")
            self.attr['error_clipping_threshold'] = error_clipping_threshold
        if drop_rate is not None:
            drop_rate = float(drop_rate)
            if drop_rate < 0:
                raise ValueError("Dropout rate must > 0")
261 262
            self.attr["drop_rate"] = drop_rate

263 264 265
        if isinstance(device, int):
            self.attr["device"] = device

266 267 268 269
    def check(self, layer_name):
        for key in self.attr:
            if not hasattr(self, 'can_%s' % key) or \
                    not getattr(self, 'can_%s' % key):
270 271
                raise NotImplementedError("Layer %s cannot support %s" %
                                          (layer_name, key))
272 273 274 275 276 277 278 279 280

    @staticmethod
    def to_kwargs(attr):
        if attr is None:
            return dict()
        else:
            return attr.attr


X
xzl 已提交
281
HookAttr = HookAttribute
282 283
ParamAttr = ParameterAttribute
ExtraAttr = ExtraLayerAttribute
反馈
建议
客服 返回
顶部