提交 12416a24 编写于 作者: D dengkaipeng 提交者: ceci3

add doc and test_layers. test=develop

上级 63d322f0
...@@ -109,10 +109,32 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { ...@@ -109,10 +109,32 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker {
.SetDefault(1e-12); .SetDefault(1e-12);
AddComment(R"DOC( AddComment(R"DOC(
This operator samples input X to given output shape by using specified This layer calculate the spectral normalize value of weight of
fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
tensor.
Spectral normalization stabilizes the training of critis in GANs
(Generative Adversarial Networks). This layers rescaling weight tensor
wiht spectral normalize value.
For spectral normalization calculations, we rescaling weight
tensor with \sigma, while \sigma{\mathbf{W}} is
\sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2}
We calculate \sigma{\mathbf{W}} through power iterations as
\mathbf{v} = \mathbf{W}^{T} \mathbf{u}
\mathbf{v} = \frac{\mathbf{v}}{\|\mathbf{v}\|_2}
\mathbf{u} = \mathbf{W}^{T} \mathbf{v}
\mathbf{u} = \frac{\mathbf{u}}{\|\mathbf{u}\|_2}
And \sigma should be
\sigma{\mathbf{W}} = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
For details of spectral normalization, please refer to paper:
`Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .
)DOC"); )DOC");
} }
}; };
......
...@@ -3349,28 +3349,42 @@ def group_norm(input, ...@@ -3349,28 +3349,42 @@ def group_norm(input,
@templatedoc() @templatedoc()
def spectral_norm(weight, def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None):
dim=0,
power_iters=1,
eps=1e-12,
u_attr=None,
v_attr=None,
name=None):
""" """
**Spectral Normalization Layer** **Spectral Normalization Layer**
This layer calculate the spectral normalize value of weight parameters of
fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
Parameters. Calculations are showed as followings.
.. code-block:: text
Step 1:
Generate vector u in shape of [h], and v in shape of [w].
While h is the attr:`dim`th dimension of the input weights,
and w is the product result of remain dimensions.
Step 2:
While attr:`power_iters` is a positive interger, do following
iteration calculations with u and v for attr:`power_iters`
round.
\mathbf{v} = \mathbf{W}^{T} \mathbf{u}
\mathbf{v} = \frac{\mathbf{v}}{\|\mathbf{v}\|_2}
\mathbf{u} = \mathbf{W}^{T} \mathbf{v}
\mathbf{u} = \frac{\mathbf{u}}{\|\mathbf{u}\|_2}
Step 3:
Calculate \sigma{W} and scale weight values.
\sigma{\mathbf{W}} = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
\mathbf{W} := \frac{\mathbf{W}}{\sigma{\mathbf{W}}}
Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ . Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .
Args: Args:
weight(${weight_type}): ${weight_comment} weight(${weight_type}): ${weight_comment}
dim(${dim_type}): ${dim_comment} dim(${dim_type}): ${dim_comment}
eps(${eps_type}): ${eps_comment} eps(${eps_type}): ${eps_comment}
u_attr(ParamAttr|None): The parameter attribute for vector u in
spectral calculatings, set None to use default attribute, which
generates random values in normal distribution N(0, 1). Default: None.
v_attr(ParamAttr|None): The parameter attribute for vector v in
spectral calculatings, set None to use default attribute, which
generates random values in normal distribution N(0, 1). Default: None.
name (str): The name of this layer. It is optional. name (str): The name of this layer. It is optional.
Returns: Returns:
...@@ -3383,43 +3397,43 @@ def spectral_norm(weight, ...@@ -3383,43 +3397,43 @@ def spectral_norm(weight,
>>> x = fluid.layers.spectral_norm(weight=data, dim=1, power_iters=2) >>> x = fluid.layers.spectral_norm(weight=data, dim=1, power_iters=2)
""" """
helper = LayerHelper('spectral_norm', **locals()) helper = LayerHelper('spectral_norm', **locals())
dtype = helper.input_dtype() dtype = weight.dtype
# create intput and parameters # create intput and parameters
inputs = {'Weight': weight} inputs = {'Weight': weight}
input_shape = input.shape input_shape = weight.shape
if data_layout != 'NCHW': h = input_shape[dim]
raise ValueError("unsupported data layout:" + data_layout) w = np.prod(input_shape) // h
param_shape = [input_shape[1]]
if param_attr: u = helper.create_parameter(
scale = helper.create_parameter( attr=ParamAttr(),
attr=helper.param_attr, shape=[h],
shape=param_shape, dtype=dtype,
dtype=dtype, default_initializer=Normal(0., 1.))
default_initializer=Constant(1.0)) u.stop_gradient = True
inputs['Scale'] = scale inputs['U'] = u
if bias_attr: v = helper.create_parameter(
bias = helper.create_parameter( attr=ParamAttr(),
attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True) shape=[w],
inputs['Bias'] = bias dtype=dtype,
default_initializer=Normal(0., 1.))
inputs['V'] = v
v.stop_gradient = True
# create output # create output
mean_out = helper.create_variable(dtype=dtype, stop_gradient=True) out = helper.create_variable(dtype=dtype)
variance_out = helper.create_variable(dtype=dtype, stop_gradient=True)
group_norm_out = helper.create_variable(dtype=dtype)
helper.append_op( helper.append_op(
type="group_norm", type="spectral_norm",
inputs=inputs, inputs=inputs,
outputs={ outputs={"Out": out, },
"Y": group_norm_out, attrs={
"Mean": mean_out, "dim": dim,
"Variance": variance_out, "power_iters": power_iters,
}, "eps": eps,
attrs={"epsilon": epsilon, })
"groups": groups})
return helper.append_activation(group_norm_out) return out
def conv2d_transpose(input, def conv2d_transpose(input,
......
...@@ -1035,6 +1035,19 @@ class TestBook(unittest.TestCase): ...@@ -1035,6 +1035,19 @@ class TestBook(unittest.TestCase):
print(str(program)) print(str(program))
def test_spectral_norm(self):
program = Program()
with program_guard(program):
weight = layers.data(
name='weight',
shape=[2, 3, 32, 32],
dtype="float32",
append_batch_size=False)
out = layers.spectral_norm(weight, dim=1, power_iters=1)
self.assertIsNotNone(out)
print(str(program))
def test_shuffle_channel(self): def test_shuffle_channel(self):
program = Program() program = Program()
with program_guard(program): with program_guard(program):
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册