param_attr.py 3.7 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
F
fengjiayi 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
F
fengjiayi 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
F
update  
fengjiayi 已提交
14

Y
Yu Yang 已提交
15 16 17
from initializer import Initializer, Xavier, Constant
from regularizer import WeightDecayRegularizer

18 19 20 21
__all__ = [
    'ParamAttr',
    'WeightNormParamAttr',
]
22

Y
Yu Yang 已提交
23 24 25 26 27 28 29

class ParamAttr(object):
    def __init__(self,
                 name=None,
                 initializer=None,
                 learning_rate=1.0,
                 regularizer=None,
30
                 trainable=True,
31
                 gradient_clip=None,
W
wanghaoshuang 已提交
32
                 do_model_average=None):
Y
Yu Yang 已提交
33 34 35 36 37
        self.name = name
        self.initializer = initializer
        self.learning_rate = learning_rate
        self.regularizer = regularizer
        self.trainable = trainable
F
fengjiayi 已提交
38
        self.gradient_clip = gradient_clip
W
wanghaoshuang 已提交
39
        self.model_average = do_model_average
Y
Yu Yang 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

    def set_default_initializer(self, initializer):
        if initializer is None:
            if self.initializer is None:
                raise ValueError("ParamAttr.initializer is not set")
            return

        if self.initializer is not None:
            return

        self.initializer = initializer

    def set_default_param_initializer(self):
        self.set_default_initializer(Xavier())

    def set_default_bias_initializer(self):
        self.set_default_initializer(Constant(0.0))

    @staticmethod
    def to_attr(arg):
        if arg is None:
            return ParamAttr()
62 63
        elif isinstance(arg, list) or isinstance(arg, tuple):
            return [ParamAttr.to_attr(a) for a in arg]
Y
Yu Yang 已提交
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
        elif isinstance(arg, ParamAttr):
            return arg
        elif isinstance(arg, str) or isinstance(arg, unicode):
            return ParamAttr(name=arg)
        elif isinstance(arg, Initializer):
            return ParamAttr(initializer=arg)
        elif isinstance(arg, WeightDecayRegularizer):
            return ParamAttr(regularizer=arg)
        elif isinstance(arg, bool):
            return ParamAttr.to_attr(None) if arg else False
        else:
            raise TypeError("{0} cast to ParamAttr".format(type(arg)))

    def to_kwargs(self, with_initializer=False):
        kwargs = {
            'name': self.name,
80 81 82
            'optimize_attr': {
                'learning_rate': self.learning_rate
            },
Y
Yu Yang 已提交
83
            'regularizer': self.regularizer,
84
            'trainable': self.trainable,
85
            'gradient_clip_attr': self.gradient_clip,
W
wanghaoshuang 已提交
86
            'model_average': self.model_average
Y
Yu Yang 已提交
87 88 89 90
        }
        if with_initializer:
            kwargs['initializer'] = self.initializer
        return kwargs
G
guosheng 已提交
91 92 93 94 95


class WeightNormParamAttr(ParamAttr):
    """
    Used for weight normalization. Any field in ParamAttr can also be set here.
96
    Besides, an extra field dim can be set to indicate the dimension except
G
guosheng 已提交
97 98 99 100
    which to normalize.
    """
    # List to record the parameters reparameterized by weight normalization.
    # If these parameters are treated as Variable rather than Parameter,
101
    # it can be used to discriminate these parameters and help to serialize
G
guosheng 已提交
102 103 104 105 106 107
    # these paramters for inference.
    params_with_weight_norm = []

    def __init__(self, dim=None, **kwargs):
        super(WeightNormParamAttr, self).__init__(**kwargs)
        self.dim = dim
反馈
建议
客服 返回
顶部