diff --git a/paddle/fluid/operators/spectral_norm_op.cc b/paddle/fluid/operators/spectral_norm_op.cc index 0d43e65c86cdfaded7754655305269417c0ce211..32b8a41ca88ff029791c2b7576784161e1cccc2b 100644 --- a/paddle/fluid/operators/spectral_norm_op.cc +++ b/paddle/fluid/operators/spectral_norm_op.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -84,20 +84,28 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { "The weight_u tensor of spectral_norm operator, " "This can be a 1-D tensor in shape [H, 1]," "H is the 1st dimentions of Weight after reshape" - "corresponding by Attr(dim)."); + "corresponding by Attr(dim). As for Attr(dim) = 1" + "in conv2d layer with weight shape [M, C, K1, K2]" + "Weight will be reshape to [C, M*K1*Kw], U will" + "be in shape [C, 1]."); AddInput("V", - "The weight_u tensor of spectral_norm operator, " + "The weight_v tensor of spectral_norm operator, " "This can be a 1-D tensor in shape [W, 1]," "W is the 2nd dimentions of Weight after reshape" - "corresponding by Attr(dim)."); + "corresponding by Attr(dim). As for Attr(dim) = 1" + "in conv2d layer with weight shape [M, C, K1, K2]" + "Weight will be reshape to [C, M*K1*Kw], V will" + "be in shape [M*K1*K2, 1]."); AddOutput("Out", "The output weight tensor of spectral_norm operator, " "This tensor is in same shape with Input(Weight)."); AddAttr("dim", "dimension corresponding to number of outputs," - "default 0 for fc layer, and 1 for conv1d, conv2d, conv3d" - "layers") + "it should be set as 0 if Input(Weight) is the" + "weight of fc layer, and should be set as 1 if" + "Input(Weight) is the weight of conv layer," + "default is 0." .SetDefault(0); AddAttr("power_iters", "number of power iterations to calculate" @@ -109,13 +117,13 @@ class SpectralNormOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(1e-12); AddComment(R"DOC( - This layer calculate the spectral normalize value of weight of + This layer calculates the spectral normalize value of weight of fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D tensor. - Spectral normalization stabilizes the training of critis in GANs - (Generative Adversarial Networks). This layers rescaling weight tensor - wiht spectral normalize value. + Spectral normalization stabilizes the training of critic in GANs + (Generative Adversarial Networks). This layer rescaling weight tensor + with spectral normalize value. For spectral normalization calculations, we rescaling weight tensor with \sigma, while \sigma{\mathbf{W}} is diff --git a/paddle/fluid/operators/spectral_norm_op.cu b/paddle/fluid/operators/spectral_norm_op.cu index 634d5b310baa383e5cd764305402efca7ca76017..ea90e3b4c122b00d5bfe13617e48a9bbe0ee8395 100644 --- a/paddle/fluid/operators/spectral_norm_op.cu +++ b/paddle/fluid/operators/spectral_norm_op.cu @@ -1,4 +1,4 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/paddle/fluid/operators/spectral_norm_op.h b/paddle/fluid/operators/spectral_norm_op.h index 45a3ad8d532a176a39e226cd3ea371699eaa87e8..de6e894c1ce1a82f6461a14c0355f24fef0387e7 100644 --- a/paddle/fluid/operators/spectral_norm_op.h +++ b/paddle/fluid/operators/spectral_norm_op.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -73,11 +73,13 @@ static inline void CalcMatrixSigmaAndNormWeight( const int w = weight->dims()[1]; for (int i = 0; i < power_iters; i++) { + // V = W^T * U / ||W^T * U||_2 blas.MatMul(*weight, true, *u, false, T(1), v, T(0)); auto v_t_norm = v_t.square().sum().sqrt().eval().reshape(Array1(1)).broadcast( Array1(w)); v_t.device(place) = v_t / (v_t_norm + v_t_norm.constant(eps)); + // U = W^T * V / ||W^T * V||_2 blas.MatMul(*weight, false, *v, false, T(1), u, T(0)); auto u_t_norm = u_t.square().sum().sqrt().eval().reshape(Array1(1)).broadcast( diff --git a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py index 81cc38a13184797f3b33de58bc4356267cbad9bc..e4e431bcce571798893ccc96c74fd9972b657f3e 100644 --- a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.