diff --git a/paddle/fluid/operators/spectral_norm_op.cc b/paddle/fluid/operators/spectral_norm_op.cc index 087d97fde68a06127bafec0dcd5d13d615ec9d01..d4ff660a963465146dbcb9b9d05e8d2653f08dee 100644 --- a/paddle/fluid/operators/spectral_norm_op.cc +++ b/paddle/fluid/operators/spectral_norm_op.cc @@ -78,7 +78,7 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddInput("Weight", "The input weight tensor of spectral_norm operator, " - "This can be a 2-D, 3-D, 4-D, 5-D tensor which is the" + "This can be a 2-D, 3-D, 4-D, 5-D tensor which is the " "weights of fc, conv1d, conv2d, conv3d layer."); AddInput("U", "The weight_u tensor of spectral_norm operator, " @@ -90,29 +90,29 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { "be in shape [C, 1]."); AddInput("V", "The weight_v tensor of spectral_norm operator, " - "This can be a 1-D tensor in shape [W, 1]," - "W is the 2nd dimentions of Weight after reshape" - "corresponding by Attr(dim). As for Attr(dim) = 1" - "in conv2d layer with weight shape [M, C, K1, K2]" - "Weight will be reshape to [C, M*K1*K2], V will" + "This can be a 1-D tensor in shape [W, 1], " + "W is the 2nd dimentions of Weight after reshape " + "corresponding by Attr(dim). As for Attr(dim) = 1 " + "in conv2d layer with weight shape [M, C, K1, K2] " + "Weight will be reshape to [C, M*K1*K2], V will " "be in shape [M*K1*K2, 1]."); AddOutput("Out", "The output weight tensor of spectral_norm operator, " "This tensor is in same shape with Input(Weight)."); AddAttr("dim", - "dimension corresponding to number of outputs," - "it should be set as 0 if Input(Weight) is the" - "weight of fc layer, and should be set as 1 if" - "Input(Weight) is the weight of conv layer," - "default is 0.") + "dimension corresponding to number of outputs, " + "it should be set as 0 if Input(Weight) is the " + "weight of fc layer, and should be set as 1 if " + "Input(Weight) is the weight of conv layer, " + "default 0.") .SetDefault(0); AddAttr("power_iters", - "number of power iterations to calculate" - "spectral norm, default is 1.") + "number of power iterations to calculate " + "spectral norm, default 1.") .SetDefault(1); AddAttr("eps", - "epsilob for numerical stability in" + "epsilob for numerical stability in " "calculating norms") .SetDefault(1e-12); @@ -126,20 +126,28 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { with spectral normalize value. For spectral normalization calculations, we rescaling weight - tensor with \sigma, while \sigma{\mathbf{W}} is + tensor with :math:`\sigma`, while :math:`\sigma{\mathbf{W}}` is - \sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2} + $$\sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \\frac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2}$$ - We calculate \sigma{\mathbf{W}} through power iterations as + We calculate :math:`\sigma{\mathbf{W}}` through power iterations as + $$ \mathbf{v} = \mathbf{W}^{T} \mathbf{u} - \mathbf{v} = \frac{\mathbf{v}}{\|\mathbf{v}\|_2} + $$ + $$ + \mathbf{v} = \\frac{\mathbf{v}}{\|\mathbf{v}\|_2} + $$ + $$ \mathbf{u} = \mathbf{W}^{T} \mathbf{v} - \mathbf{u} = \frac{\mathbf{u}}{\|\mathbf{u}\|_2} + $$ + $$ + \mathbf{u} = \\frac{\mathbf{u}}{\|\mathbf{u}\|_2} + $$ - And \sigma should be + And :math:`\sigma` should be - \sigma{\mathbf{W}} = \mathbf{u}^{T} \mathbf{W} \mathbf{v} + $$\sigma{\mathbf{W}} = \mathbf{u}^{T} \mathbf{W} \mathbf{v}$$ For details of spectral normalization, please refer to paper: `Spectral Normalization `_ . diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 4862733e74e77c3e67fe4156cc73e9c7bc44e6e7..a3d22499fefb674d034881dede980450ce91d6a4 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3357,34 +3357,38 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D Parameters. Calculations are showed as followings. - .. code-block:: text + Step 1: + Generate vector U in shape of [H], and V in shape of [W]. + While H is the :attr:`dim` th dimension of the input weights, + and W is the product result of remain dimensions. - Step 1: - Generate vector u in shape of [h], and v in shape of [w]. - While h is the attr:`dim`th dimension of the input weights, - and w is the product result of remain dimensions. + Step 2: + :attr:`power_iters` shoule be a positive interger, do following + calculations with U and V for :attr:`power_iters` rounds. - Step 2: - While attr:`power_iters` is a positive interger, do following - iteration calculations with u and v for attr:`power_iters` - round. - \mathbf{v} = \mathbf{W}^{T} \mathbf{u} - \mathbf{v} = \frac{\mathbf{v}}{\|\mathbf{v}\|_2} - \mathbf{u} = \mathbf{W}^{T} \mathbf{v} - \mathbf{u} = \frac{\mathbf{u}}{\|\mathbf{u}\|_2} - - Step 3: - Calculate \sigma{W} and scale weight values. - \sigma{\mathbf{W}} = \mathbf{u}^{T} \mathbf{W} \mathbf{v} - \mathbf{W} := \frac{\mathbf{W}}{\sigma{\mathbf{W}}} + .. math:: + + \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2} + + \mathbf{u} := \\frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2} + + Step 3: + Calculate :math:`\sigma(\mathbf{W})` and scale weight values. + + .. math:: + + \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v} + + \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})} Refer to `Spectral Normalization `_ . Args: weight(${weight_type}): ${weight_comment} - dim(${dim_type}): ${dim_comment} - eps(${eps_type}): ${eps_comment} + dim(int): ${dim_comment} + power_iters(int): ${power_iters_comment} + eps(float): ${eps_comment} name (str): The name of this layer. It is optional. Returns: