batch_norm_op.cc 52.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

Y
Yi Wang 已提交
15
#include "paddle/fluid/operators/batch_norm_op.h"
Q
qingqing01 已提交
16
#include <memory>
S
Siddharth Goyal 已提交
17
#include <string>
Q
qingqing01 已提交
18
#include <unordered_map>
Y
Yi Wang 已提交
19
#include "paddle/fluid/framework/data_layout.h"
20 21 22
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/platform/mkldnn_helper.h"
#endif
Q
Qiao Longfei 已提交
23 24 25 26

namespace paddle {
namespace operators {

Q
qingqing01 已提交
27
void BatchNormOp::InferShape(framework::InferShapeContext *ctx) const {
28 29 30 31 32 33 34
  OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "BatchNorm");
  OP_INOUT_CHECK(ctx->HasInput("Scale"), "Input", "Scale", "BatchNorm");
  OP_INOUT_CHECK(ctx->HasInput("Bias"), "Input", "Bias", "BatchNorm");
  OP_INOUT_CHECK(ctx->HasInput("Mean"), "Input", "Mean", "BatchNorm");
  OP_INOUT_CHECK(ctx->HasInput("Variance"), "Input", "Variance", "BatchNorm");
  OP_INOUT_CHECK(ctx->HasOutput("Y"), "Output", "Y", "BatchNorm");

Q
qingqing01 已提交
35
  bool is_test = ctx->Attrs().Get<bool>("is_test");
36 37 38
  bool trainable_stats = ctx->Attrs().Get<bool>("trainable_statistics");
  bool test_mode = is_test && (!trainable_stats);
  if (!test_mode) {
39 40 41 42 43 44 45
    OP_INOUT_CHECK(ctx->HasOutput("MeanOut"), "Output", "MeanOut", "BatchNorm");
    OP_INOUT_CHECK(ctx->HasOutput("VarianceOut"), "Output", "VarianceOut",
                   "BatchNorm");
    OP_INOUT_CHECK(ctx->HasOutput("SavedMean"), "Output", "SavedMean",
                   "BatchNorm");
    OP_INOUT_CHECK(ctx->HasOutput("SavedVariance"), "Output", "SavedVariance",
                   "BatchNorm");
Q
Qiao Longfei 已提交
46
  }
K
Kexin Zhao 已提交
47

Q
qingqing01 已提交
48 49
  // make sure Mean/MeanOut and Variance/VarianceOut share memory in Python
  PADDLE_ENFORCE_EQ(ctx->Inputs("Mean")[0], ctx->Outputs("MeanOut")[0],
50 51 52 53 54 55
                    platform::errors::InvalidArgument(
                        "Mean and MeanOut should share the same memory"));
  PADDLE_ENFORCE_EQ(
      ctx->Inputs("Variance")[0], ctx->Outputs("VarianceOut")[0],
      platform::errors::InvalidArgument(
          "Variance and VarianceOut should share the same memory"));
Q
qingqing01 已提交
56 57

  const auto x_dims = ctx->GetInputDim("X");
58 59 60 61 62 63 64 65 66 67

  for (int i = 0; i < x_dims.size(); i++) {
    PADDLE_ENFORCE_EQ(
        (x_dims[i] == -1) || (x_dims[i] > 0), true,
        platform::errors::InvalidArgument(
            "Each dimension of input tensor is expected to be -1 or a "
            "positive number, but recieved %d. Input's shape is [%s].",
            x_dims[i], x_dims));
  }

Q
qingqing01 已提交
68 69 70
  const DataLayout data_layout = framework::StringToDataLayout(
      ctx->Attrs().Get<std::string>("data_layout"));

71 72 73 74
  if (ctx->IsRuntime() && ctx->HasInput("MomentumTensor")) {
    auto mom = ctx->Inputs("MomentumTensor");
    PADDLE_ENFORCE_EQ(mom.size(), 1,
                      platform::errors::InvalidArgument(
C
ceci3 已提交
75 76 77
                          "The input tensor MomentumTensor's size must be 1"
                          "But received: MomentumTensor's size is [%d]",
                          mom.size()));
78 79
  }

80 81
  PADDLE_ENFORCE_GE(
      x_dims.size(), 2,
K
Kaipeng Deng 已提交
82 83 84 85 86
      platform::errors::InvalidArgument(
          "ShapeError: the dimension of input "
          "X must greater than or equal to 2. But received: the shape of input "
          "X = [%s], the dimension of input X =[%d]",
          x_dims, x_dims.size()));
87 88
  PADDLE_ENFORCE_LE(
      x_dims.size(), 5,
K
Kaipeng Deng 已提交
89 90 91 92 93
      platform::errors::InvalidArgument(
          "ShapeError: the dimension of input X "
          "must smaller than or equal to 5. But received: the shape of input X "
          "= [%s], the dimension of input X = [%d]",
          x_dims, x_dims.size()));
Q
qingqing01 已提交
94 95

  const int64_t C =
96
      ((ctx->IsRunMKLDNNKernel() == true) || (data_layout == DataLayout::kNCHW)
97 98
           ? x_dims[1]
           : x_dims[x_dims.size() - 1]);
Q
qingqing01 已提交
99

100 101
  auto scale_dim = ctx->GetInputDim("Scale");
  auto bias_dim = ctx->GetInputDim("Bias");
Q
qingqing01 已提交
102

103
  PADDLE_ENFORCE_EQ(
104 105 106 107 108 109 110 111 112 113 114 115
      scale_dim.size(), 1UL,
      platform::errors::InvalidArgument(
          "ShapeError: the dimension of scale must equal to 1."
          "But received: the shape of scale is [%s], the dimension "
          "of scale is [%d]",
          scale_dim, scale_dim.size()));
  PADDLE_ENFORCE_EQ(bias_dim.size(), 1UL,
                    platform::errors::InvalidArgument(
                        "ShapeError: the dimension of bias must equal to 1."
                        "But received: the shape of bias is [%s],the dimension "
                        "of bias is [%d]",
                        bias_dim, bias_dim.size()));
C
ceci3 已提交
116

117
  bool check = true;
118
  if ((!ctx->IsRuntime()) &&
119
      (phi::product(scale_dim) <= 0 || phi::product(bias_dim) <= 0)) {
120 121 122 123
    check = false;
  }

  if (check) {
124
    PADDLE_ENFORCE_EQ(scale_dim[0], C,
125 126 127 128
                      platform::errors::InvalidArgument(
                          "ShapeError: the shape of scale must equal to [%d]"
                          "But received: the shape of scale is [%d]",
                          C, scale_dim[0]));
129
    PADDLE_ENFORCE_EQ(bias_dim[0], C,
130 131 132 133
                      platform::errors::InvalidArgument(
                          "ShapeError: the shape of bias must equal to [%d]"
                          "But received: the shape of bias is [%d]",
                          C, bias_dim[0]));
134
  }
Q
qingqing01 已提交
135 136 137 138 139 140 141 142 143 144
  ctx->SetOutputDim("Y", x_dims);
  ctx->SetOutputDim("MeanOut", {C});
  ctx->SetOutputDim("VarianceOut", {C});
  ctx->SetOutputDim("SavedMean", {C});
  ctx->SetOutputDim("SavedVariance", {C});
  ctx->ShareLoD("X", "Y");
}

framework::OpKernelType BatchNormOp::GetExpectedKernelType(
    const framework::ExecutionContext &ctx) const {
145
  auto input_data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X");
Q
qingqing01 已提交
146 147 148 149 150 151 152
  // By default, the type of the scale, bias, mean,
  // and var tensors should both be float. (For float or float16 input tensor)
  // or double (For double input tensor).
  auto bn_param_type = framework::proto::VarType::FP32;
  if (input_data_type == framework::proto::VarType::FP64) {
    bn_param_type = framework::proto::VarType::FP64;
  }
K
Kaipeng Deng 已提交
153
  PADDLE_ENFORCE_EQ(
154 155
      bn_param_type,
      framework::TransToProtoVarType(ctx.Input<Tensor>("Scale")->dtype()),
K
Kaipeng Deng 已提交
156 157
      platform::errors::InvalidArgument("Scale input should be of float type"));
  PADDLE_ENFORCE_EQ(
158 159
      bn_param_type,
      framework::TransToProtoVarType(ctx.Input<Tensor>("Bias")->dtype()),
K
Kaipeng Deng 已提交
160 161
      platform::errors::InvalidArgument("Bias input should be of float type"));
  PADDLE_ENFORCE_EQ(
162 163
      bn_param_type,
      framework::TransToProtoVarType(ctx.Input<Tensor>("Mean")->dtype()),
K
Kaipeng Deng 已提交
164
      platform::errors::InvalidArgument("Mean input should be of float type"));
165 166
  PADDLE_ENFORCE_EQ(bn_param_type, framework::TransToProtoVarType(
                                       ctx.Input<Tensor>("Variance")->dtype()),
K
Kaipeng Deng 已提交
167 168
                    platform::errors::InvalidArgument(
                        "Variance input should be of float type"));
Q
qingqing01 已提交
169 170 171 172

  // TODO(pzelazko-intel): enable MKLDNN layout when it's ready
  framework::LibraryType library = framework::LibraryType::kPlain;
  framework::DataLayout layout = framework::DataLayout::kAnyLayout;
173
#ifdef PADDLE_WITH_MKLDNN
174 175
  if (library == framework::LibraryType::kPlain &&
      this->CanMKLDNNBeUsed(ctx, input_data_type)) {
Q
qingqing01 已提交
176 177
    library = framework::LibraryType::kMKLDNN;
    layout = framework::DataLayout::kMKLDNN;
K
Kexin Zhao 已提交
178
  }
Q
qingqing01 已提交
179
#endif
Q
Qiao Longfei 已提交
180

Q
qingqing01 已提交
181 182 183 184
  return framework::OpKernelType(input_data_type, ctx.GetPlace(), layout,
                                 library);
}

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
framework::OpKernelType BatchNormOp::GetKernelTypeForVar(
    const std::string &var_name, const Tensor &tensor,
    const framework::OpKernelType &expected_kernel_type) const {
#ifdef PADDLE_WITH_MKLDNN
  // Only input require reshaping, weights and
  // bias are having shape in NCHW order
  if ((var_name == "X") &&
      (expected_kernel_type.data_layout_ == framework::DataLayout::kMKLDNN) &&
      (tensor.layout() != framework::DataLayout::kMKLDNN)) {
    auto attrs = Attrs();
    auto ar = paddle::framework::AttrReader(attrs);
    const std::string data_layout = ar.Get<std::string>("data_layout");
    auto dl = framework::StringToDataLayout(data_layout);
    // Some models may have intentionally set "AnyLayout" for pool
    // op. Treat this as NCHW (default data_format value)
    if (dl != framework::DataLayout::kAnyLayout) {
201 202
      return framework::OpKernelType(expected_kernel_type.data_type_,
                                     tensor.place(), dl);
203 204 205 206 207 208 209
    }
  }
#endif
  return framework::OpKernelType(expected_kernel_type.data_type_,
                                 tensor.place(), tensor.layout());
}

Q
qingqing01 已提交
210 211 212 213 214 215 216 217 218
void BatchNormOpMaker::Make() {
  AddAttr<bool>("is_test",
                "(bool, default false) Set to true for inference only, false "
                "for training. Some layers may run faster when this is true.")
      .SetDefault(false);
  AddAttr<float>("momentum", "").SetDefault(0.9);
  AddAttr<float>("epsilon", "")
      .SetDefault(1e-5)
      .AddCustomChecker([](const float &epsilon) {
K
Kaipeng Deng 已提交
219 220 221 222 223 224 225
        PADDLE_ENFORCE_GE(
            epsilon, 0.0f,
            platform::errors::InvalidArgument(
                "'epsilon' should be greater or equal than 0.0."));
        PADDLE_ENFORCE_LE(epsilon, 0.001f,
                          platform::errors::InvalidArgument(
                              "'epsilon' should be less or equal than 0.001."));
Q
qingqing01 已提交
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
      });
  AddAttr<std::string>("data_layout", "").SetDefault("NCHW");
  AddInput("X", "The input tensor");
  AddInput("Scale",
           "Scale is a 1-dimensional tensor of size C "
           "that is applied to the output");
  AddInput("Bias",
           "Bias is a 1-dimensional tensor of size C "
           "that is applied to the output");
  AddInput("Mean",
           "The global mean (for training) or "
           "estimated mean (for testing)");
  AddInput("Variance",
           "The global variance (for training) "
           "or estimated Variance (for testing)");
241 242 243 244 245
  AddInput("MomentumTensor",
           "(Tensor<float32>, optional) If provided, batch_norm will "
           "use this as momentum, this has a higher priority than "
           "attr(momentum), the shape of this tensor MUST BE [1].")
      .AsDispensable();
Q
qingqing01 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
  AddOutput("Y", "result after normalization");
  AddOutput("MeanOut",
            "Share memory with Mean. "
            "Store the global mean when training");
  AddOutput("VarianceOut",
            "Share memory with Variance. "
            "Store the global Variance when training");
  AddOutput("SavedMean",
            "Mean of the current mini batch, "
            "will apply to output when training")
      .AsIntermediate();
  AddOutput("SavedVariance",
            "Variance of the current mini batch, "
            "will apply to output when training")
      .AsIntermediate();
261 262 263
  AddOutput("ReserveSpace",
            "Reserve GPU space for triggering the new semi-persistent "
            "NHWC kernel")
C
ceci3 已提交
264 265
      .AsDispensable()
      .AsExtra();
Q
qingqing01 已提交
266 267
  AddAttr<bool>("use_mkldnn",
                "(bool, default false) Only used in mkldnn kernel")
C
ceci3 已提交
268 269
      .SetDefault(false)
      .AsExtra();
Q
qingqing01 已提交
270 271
  AddAttr<bool>("fuse_with_relu",
                "(bool, default false) Only used in mkldnn kernel")
C
ceci3 已提交
272 273
      .SetDefault(false)
      .AsExtra();
Q
qingqing01 已提交
274 275 276 277 278 279 280 281
  AddAttr<bool>("use_global_stats",
                "(bool, default false) Whether to use global mean and "
                "variance. In inference or test mode, set use_global_stats "
                "to true or is_test true. the behavior is equivalent. "
                "In train mode, when setting use_global_stats True, the "
                "global mean and variance are also used during train time, "
                "the BN acts as scaling and shiffting.")
      .SetDefault(false);
282 283 284 285 286
  AddAttr<bool>("trainable_statistics",
                "(bool, default false) Whether to calculate mean and variance "
                "in test mode. If setting true in test mode, mean and variace "
                "will be calculated by current batch statistics.")
      .SetDefault(false);
Q
qingqing01 已提交
287
  AddComment(R"DOC(
288
Batch Normalization.
Q
Qiao Longfei 已提交
289

290 291 292 293 294 295
Batch Norm has been implemented as discussed in the paper:
https://arxiv.org/pdf/1502.03167.pdf
Can be used as a normalizer function for conv2d and fully_connected operations.
The required data format for this layer is one of the following:
1. NHWC `[batch, in_height, in_width, in_channels]`
2. NCHW `[batch, in_channels, in_height, in_width]`
Q
Qiao Longfei 已提交
296 297

)DOC");
Q
qingqing01 已提交
298
}
C
chengduo 已提交
299

Q
Qiao Longfei 已提交
300
template <typename T>
Q
QI JUN 已提交
301 302
class BatchNormKernel<platform::CPUDeviceContext, T>
    : public framework::OpKernel<T> {
Q
Qiao Longfei 已提交
303 304 305
 public:
  void Compute(const framework::ExecutionContext &ctx) const override {
    const float epsilon = ctx.Attr<float>("epsilon");
306
    float momentum = ctx.Attr<float>("momentum");
Q
Qiao Longfei 已提交
307
    const bool is_test = ctx.Attr<bool>("is_test");
308
    const bool use_global_stats = ctx.Attr<bool>("use_global_stats");
309 310
    const bool trainable_stats = ctx.Attr<bool>("trainable_statistics");
    bool test_mode = is_test && (!trainable_stats);
311

312
    bool global_stats = test_mode || use_global_stats;
313

Q
QI JUN 已提交
314
    const std::string data_layout_str = ctx.Attr<std::string>("data_layout");
315
    DataLayout data_layout = framework::StringToDataLayout(data_layout_str);
Q
Qiao Longfei 已提交
316 317 318

    const auto *x = ctx.Input<Tensor>("X");
    const auto &x_dims = x->dims();
C
ceci3 已提交
319 320 321 322 323 324 325 326 327 328 329 330
    PADDLE_ENFORCE_GE(
        x_dims.size(), 2,
        platform::errors::InvalidArgument(
            "The size of input X's dimensions should be larger than 1."
            "But received: the size of input X's dimensions is [%d]",
            x_dims.size()));
    PADDLE_ENFORCE_LE(
        x_dims.size(), 5,
        platform::errors::InvalidArgument(
            "The size of input X's dimensions should be less than 6."
            "But received: the size of input X's dimensionss is [%d]",
            x_dims.size()));
Q
Qiao Longfei 已提交
331 332
    const int N = x_dims[0];
    const int C =
Q
QI JUN 已提交
333 334
        (data_layout == DataLayout::kNCHW ? x_dims[1]
                                          : x_dims[x_dims.size() - 1]);
Q
Qiao Longfei 已提交
335 336 337
    const int sample_size = x->numel() / N / C;

    auto *y = ctx.Output<Tensor>("Y");
K
Kaipeng Deng 已提交
338

Q
Qiao Longfei 已提交
339 340 341 342 343 344 345 346 347 348 349 350
    auto *mean_out = ctx.Output<Tensor>("MeanOut");
    auto *variance_out = ctx.Output<Tensor>("VarianceOut");
    auto *saved_mean = ctx.Output<Tensor>("SavedMean");
    auto *saved_variance = ctx.Output<Tensor>("SavedVariance");

    // alloc memory
    y->mutable_data<T>(ctx.GetPlace());
    mean_out->mutable_data<T>(ctx.GetPlace());
    variance_out->mutable_data<T>(ctx.GetPlace());
    saved_mean->mutable_data<T>(ctx.GetPlace());
    saved_variance->mutable_data<T>(ctx.GetPlace());

351 352 353 354 355 356
    // input dimension is 2 and the format is NCHW. The input can be regarded
    // as NHWC format
    if (x_dims.size() == 2 && data_layout == DataLayout::kNCHW) {
      data_layout = DataLayout::kNHWC;
    }

357
    if (!global_stats) {
Q
Qiao Longfei 已提交
358 359 360 361 362 363 364 365
      // saved_xx is use just in this batch of data
      EigenVectorArrayMap<T> saved_mean_e(
          saved_mean->mutable_data<T>(ctx.GetPlace()), C);
      EigenVectorArrayMap<T> saved_variance_e(
          saved_variance->mutable_data<T>(ctx.GetPlace()), C);
      saved_mean_e.setZero();
      saved_variance_e.setZero();

366 367 368 369 370 371
      EigenVectorArrayMap<T> running_mean_arr(
          mean_out->mutable_data<T>(ctx.GetPlace()), C);
      EigenVectorArrayMap<T> running_var_arr(
          variance_out->mutable_data<T>(ctx.GetPlace()), C);

      if ((N * sample_size) == 1) {
372 373
        // Only 1 element in normalization dimension,
        // we skip the batch norm calculation, let y = x.
374
        framework::TensorCopy(*x, ctx.GetPlace(), y);
375 376 377
        return;
      }

Q
QI JUN 已提交
378 379
      switch (data_layout) {
        case DataLayout::kNCHW: {
Q
Qiao Longfei 已提交
380 381 382 383 384 385 386 387 388 389 390 391
          ConstEigenArrayMap<T> x_arr(x->data<T>(), sample_size, N * C);
          for (int nc = 0; nc < N * C; ++nc) {
            saved_mean_e(nc % C) += x_arr.col(nc).sum();
          }
          saved_mean_e /= N * sample_size;
          for (int nc = 0; nc < N * C; ++nc) {
            saved_variance_e(nc % C) +=
                (x_arr.col(nc) - saved_mean_e(nc % C)).matrix().squaredNorm();
          }
          saved_variance_e /= N * sample_size;
          break;
        }
Q
QI JUN 已提交
392
        case DataLayout::kNHWC: {
Q
Qiao Longfei 已提交
393 394 395 396 397 398 399 400 401 402 403 404 405
          ConstEigenArrayMap<T> x_arr(x->data<T>(), C, N * sample_size);
          for (int i = 0; i < N * sample_size; ++i) {
            saved_mean_e += x_arr.col(i);
          }
          saved_mean_e /= N * sample_size;
          for (int i = 0; i < N * sample_size; ++i) {
            saved_variance_e +=
                (x_arr.col(i) - saved_mean_e) * (x_arr.col(i) - saved_mean_e);
          }
          saved_variance_e /= N * sample_size;
          break;
        }
        default:
406 407
          PADDLE_THROW(platform::errors::InvalidArgument(
              "Unknown storage order: %s", data_layout_str));
Q
Qiao Longfei 已提交
408 409
      }

410 411 412 413 414 415 416
      // if MomentumTensor is set, use MomentumTensor value, momentum
      // is only used in this training branch
      if (ctx.HasInput("MomentumTensor")) {
        const auto *mom_tensor = ctx.Input<Tensor>("MomentumTensor");
        momentum = mom_tensor->data<float>()[0];
      }

Q
Qiao Longfei 已提交
417 418 419 420 421 422 423 424
      running_mean_arr =
          running_mean_arr * momentum + saved_mean_e * (1. - momentum);
      running_var_arr =
          running_var_arr * momentum + saved_variance_e * (1. - momentum);
    }

    // use SavedMean and SavedVariance to do normalize
    Eigen::Array<T, Eigen::Dynamic, 1> inv_std(C);
425
    if (global_stats) {
Q
Qiao Longfei 已提交
426 427 428 429 430 431 432 433 434 435 436
      ConstEigenVectorArrayMap<T> var_arr(
          ctx.Input<Tensor>("Variance")->data<T>(), C);
      inv_std = (var_arr + epsilon).sqrt().inverse();
    } else {
      EigenVectorArrayMap<T> saved_inv_std(
          ctx.Output<Tensor>("SavedVariance")->data<T>(), C);
      // inverse SavedVariance first, gradient will use it too.
      saved_inv_std = (saved_inv_std + epsilon).inverse().sqrt();
      inv_std = saved_inv_std;
    }
    ConstEigenVectorArrayMap<T> mean_arr(
437 438
        global_stats ? ctx.Input<Tensor>("Mean")->data<T>()
                     : ctx.Output<Tensor>("SavedMean")->data<T>(),
Q
Qiao Longfei 已提交
439 440 441 442 443 444 445 446 447 448 449 450 451
        C);

    //   ((x - est_mean) * (inv_var) * scale + bias
    //   formula transform ====>
    //   (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
    const auto *scale = ctx.Input<Tensor>("Scale");
    const auto *bias = ctx.Input<Tensor>("Bias");
    ConstEigenVectorArrayMap<T> scale_arr(scale->data<T>(), C);
    ConstEigenVectorArrayMap<T> bias_arr(bias->data<T>(), C);
    Eigen::Array<T, Eigen::Dynamic, 1> new_scale = inv_std * scale_arr;
    Eigen::Array<T, Eigen::Dynamic, 1> new_bias =
        bias_arr - mean_arr * inv_std * scale_arr;

Q
QI JUN 已提交
452 453
    switch (data_layout) {
      case DataLayout::kNCHW: {
Q
Qiao Longfei 已提交
454 455 456 457 458 459 460 461
        EigenArrayMap<T> y_arr(y->mutable_data<T>(ctx.GetPlace()), sample_size,
                               N * C);
        ConstEigenArrayMap<T> x_arr(x->data<T>(), sample_size, N * C);
        for (int nc = 0; nc < N * C; ++nc) {
          y_arr.col(nc) = x_arr.col(nc) * new_scale(nc % C) + new_bias(nc % C);
        }
        break;
      }
Q
QI JUN 已提交
462
      case DataLayout::kNHWC: {
Q
Qiao Longfei 已提交
463 464 465 466 467 468 469 470 471
        EigenArrayMap<T>(y->mutable_data<T>(ctx.GetPlace()), C,
                         N * sample_size) =
            (ConstEigenArrayMap<T>(x->data<T>(), C, N * sample_size).colwise() *
             new_scale)
                .colwise() +
            new_bias;
        break;
      }
      default:
472 473
        PADDLE_THROW(platform::errors::InvalidArgument(
            "Unknown storage order: %d", data_layout));
Q
Qiao Longfei 已提交
474 475 476 477
    }
  }
};

Q
qingqing01 已提交
478 479
void BatchNormGradOp::InferShape(framework::InferShapeContext *ctx) const {
  // check input
480 481 482 483 484 485 486
  OP_INOUT_CHECK(ctx->HasInput("Scale"), "Input", "Scale", "BatchNormGrad");
  OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Y")), "Input",
                 framework::GradVarName("Y"), "BatchNormGrad");
  OP_INOUT_CHECK(ctx->HasInput("SavedMean"), "Input", "SavedMean",
                 "BatchNormGrad");
  OP_INOUT_CHECK(ctx->HasInput("SavedVariance"), "Input", "SavedVariance",
                 "BatchNormGrad");
Q
qingqing01 已提交
487 488

  // check output
489 490
  const bool has_scale_grad = ctx->HasOutput(framework::GradVarName("Scale"));
  const bool has_bias_grad = ctx->HasOutput(framework::GradVarName("Bias"));
491
  const bool has_x_grad = ctx->HasOutput(framework::GradVarName("X"));
492 493

  PADDLE_ENFORCE_EQ((has_scale_grad == has_bias_grad), true,
494
                    platform::errors::NotFound(
495 496 497 498 499
                        "Output(Scale@GRAD) and Output(Bias@GRAD) must be null "
                        "or not be null at same time. But now, "
                        "has Scale@Grad=[%d], has Bias@GRAD=[%d]",
                        has_scale_grad, has_bias_grad));

Q
qingqing01 已提交
500 501
  const bool use_global_stats = ctx->Attrs().Get<bool>("use_global_stats");
  if (use_global_stats) {
K
Kaipeng Deng 已提交
502 503 504 505 506
    PADDLE_ENFORCE_EQ(
        !ctx->Attrs().Get<bool>("use_mkldnn"), true,
        platform::errors::InvalidArgument(
            "Using global stats during training is not supported "
            "in gradient op kernel of batch_norm_mkldnn_op now."));
Q
qingqing01 已提交
507
  }
Q
Qiao Longfei 已提交
508

509 510 511 512
  OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "BatchNormGrad");
  const auto x_dims = ctx->GetInputDim("X");
  const DataLayout data_layout = framework::StringToDataLayout(
      ctx->Attrs().Get<std::string>("data_layout"));
Q
Qiao Longfei 已提交
513

514
  const int C =
515
      ((ctx->IsRunMKLDNNKernel() == true) || (data_layout == DataLayout::kNCHW)
516 517 518 519 520 521 522
           ? x_dims[1]
           : x_dims[x_dims.size() - 1]);

  // has_scale_grad == has_bias_grad, judge has_scale_grad is enough
  if (has_scale_grad) {
    ctx->SetOutputDim(framework::GradVarName("Scale"), {C});
    ctx->SetOutputDim(framework::GradVarName("Bias"), {C});
Q
Qiao Longfei 已提交
523
  }
524 525 526
  if (has_x_grad) {
    ctx->SetOutputDim(framework::GradVarName("X"), x_dims);
  }
Q
qingqing01 已提交
527
}
Q
Qiao Longfei 已提交
528

Q
qingqing01 已提交
529 530 531 532
framework::OpKernelType BatchNormGradOp::GetExpectedKernelType(
    const framework::ExecutionContext &ctx) const {
  const auto *var = ctx.InputVar(framework::GradVarName("Y"));
  if (var == nullptr) {
K
Kaipeng Deng 已提交
533 534
    PADDLE_THROW(
        platform::errors::InvalidArgument("can't find gradient variable of Y"));
Q
qingqing01 已提交
535 536 537 538 539 540 541 542
  }
  const Tensor *t = nullptr;
  if (var->IsType<Tensor>()) {
    t = &var->Get<Tensor>();
  } else if (var->IsType<LoDTensor>()) {
    t = &var->Get<LoDTensor>();
  }
  if (t == nullptr) {
K
Kaipeng Deng 已提交
543 544
    PADDLE_THROW(
        platform::errors::InvalidArgument("gradient variable of Y is empty"));
Q
qingqing01 已提交
545
  }
546

Q
qingqing01 已提交
547 548 549
  // TODO(pzelazko-intel): enable MKLDNN layout when it's ready
  framework::LibraryType library = framework::LibraryType::kPlain;
  framework::DataLayout layout = framework::DataLayout::kAnyLayout;
550
  auto data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X");
551

552
#ifdef PADDLE_WITH_MKLDNN
553 554
  if (library == framework::LibraryType::kPlain &&
      this->CanMKLDNNBeUsed(ctx, data_type)) {
Q
qingqing01 已提交
555 556 557
    library = framework::LibraryType::kMKLDNN;
    layout = framework::DataLayout::kMKLDNN;
  }
558
#endif
559

560
  return framework::OpKernelType(data_type, ctx.GetPlace(), layout, library);
Q
qingqing01 已提交
561
}
Q
Qiao Longfei 已提交
562

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
framework::OpKernelType BatchNormGradOp::GetKernelTypeForVar(
    const std::string &var_name, const Tensor &tensor,
    const framework::OpKernelType &expected_kernel_type) const {
#ifdef PADDLE_WITH_MKLDNN
  // Only input require reshaping, weights and
  // bias are having shape in NCHW order
  if (((var_name == "X") || (var_name == framework::GradVarName("Y"))) &&
      (expected_kernel_type.data_layout_ == framework::DataLayout::kMKLDNN) &&
      (tensor.layout() != framework::DataLayout::kMKLDNN)) {
    auto attrs = Attrs();
    auto ar = paddle::framework::AttrReader(attrs);
    const std::string data_layout = ar.Get<std::string>("data_layout");
    auto dl = framework::StringToDataLayout(data_layout);
    // Some models may have intentionally set "AnyLayout" for pool
    // op. Treat this as NCHW (default data_format value)
    if (dl != framework::DataLayout::kAnyLayout) {
      return framework::OpKernelType(expected_kernel_type.data_type_,
                                     tensor.place(), dl);
    }
  }
#endif
  return framework::OpKernelType(expected_kernel_type.data_type_,
                                 tensor.place(), tensor.layout());
}

Q
Qiao Longfei 已提交
588
template <typename T>
Q
QI JUN 已提交
589
class BatchNormGradKernel<platform::CPUDeviceContext, T>
Q
Qiao Longfei 已提交
590 591 592 593 594
    : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext &ctx) const override {
    const auto *d_y = ctx.Input<Tensor>(framework::GradVarName("Y"));
    const auto *scale = ctx.Input<Tensor>("Scale");
K
Kaipeng Deng 已提交
595
    const auto *bias = ctx.Input<Tensor>("Bias");
Q
Qiao Longfei 已提交
596 597 598
    const auto *saved_mean = ctx.Input<Tensor>("SavedMean");
    // SavedVariance have been reverted in forward operator
    const auto *saved_inv_variance = ctx.Input<Tensor>("SavedVariance");
Q
QI JUN 已提交
599
    const std::string data_layout_str = ctx.Attr<std::string>("data_layout");
C
ceci3 已提交
600
    bool use_global_stats = ctx.Attr<bool>("use_global_stats");
601
    const bool is_test = ctx.Attr<bool>("is_test");
602
    const float epsilon = ctx.Attr<float>("epsilon");
603
    DataLayout data_layout = framework::StringToDataLayout(data_layout_str);
Q
Qiao Longfei 已提交
604

K
Kaipeng Deng 已提交
605 606 607 608
    auto *d_x = ctx.Output<Tensor>(framework::GradVarName("X"));
    auto *d_scale = ctx.Output<Tensor>(framework::GradVarName("Scale"));
    auto *d_bias = ctx.Output<Tensor>(framework::GradVarName("Bias"));

C
ceci3 已提交
609 610
    use_global_stats = is_test || use_global_stats;

K
Kaipeng Deng 已提交
611 612 613 614 615 616 617 618 619
    // batch_norm with inplace as false will take X as grad input, which
    // is same as cuDNN batch_norm backward calculation, batch_norm
    // with inplace as true only take Y as input and X should be calculate
    // by inverse operation of batch_norm on Y
    const Tensor *x;
    bool is_inplace;
    if (ctx.HasInput("Y")) {
      x = ctx.Input<Tensor>("Y");
      is_inplace = true;
620 621 622 623 624 625
      // if the input of batch norm is stop_gradient, d_x is null.
      if (d_x) {
        PADDLE_ENFORCE_EQ(d_x, d_y,
                          platform::errors::InvalidArgument(
                              "X@GRAD and Y@GRAD not inplace in inplace mode"));
      }
K
Kaipeng Deng 已提交
626 627 628
    } else {
      x = ctx.Input<Tensor>("X");
      is_inplace = false;
629 630 631 632 633
      if (d_x) {
        PADDLE_ENFORCE_NE(
            d_x, d_y, platform::errors::InvalidArgument(
                          "X@GRAD and Y@GRAD inplaced in non-inplace mode"));
      }
K
Kaipeng Deng 已提交
634 635
    }

Q
Qiao Longfei 已提交
636 637 638
    // Get the size for each dimension.
    // NCHW [batch_size, in_channels, in_height, in_width]
    const auto &x_dims = x->dims();
C
ceci3 已提交
639 640 641 642 643 644 645 646 647 648 649 650
    PADDLE_ENFORCE_GE(
        x_dims.size(), 2,
        platform::errors::InvalidArgument(
            "The size of input X's dimensions should be larger than 1."
            "But received: the size of input X's dimensions is [%d]",
            x_dims.size()));
    PADDLE_ENFORCE_LE(
        x_dims.size(), 5,
        platform::errors::InvalidArgument(
            "The size of input X's dimensions should be less than 6."
            "But received: the size of input X's dimensions is [%d]",
            x_dims.size()));
Q
Qiao Longfei 已提交
651 652
    const int N = x_dims[0];
    const int C =
Q
QI JUN 已提交
653 654
        (data_layout == DataLayout::kNCHW ? x_dims[1]
                                          : x_dims[x_dims.size() - 1]);
Q
Qiao Longfei 已提交
655 656
    const int sample_size = x->numel() / N / C;

657 658 659 660 661 662
    // input dimension is 2 and the format is NCHW. The input can be regarded as
    // NHWC format
    if (x_dims.size() == 2 && data_layout == DataLayout::kNCHW) {
      data_layout = DataLayout::kNHWC;
    }

Q
Qiao Longfei 已提交
663
    // init output
664 665 666
    if (d_x) {
      d_x->mutable_data<T>(ctx.GetPlace());
    }
667 668 669 670 671 672 673 674

    const T *mean_data = saved_mean->data<T>();
    const T *inv_var_data = saved_inv_variance->data<T>();
    Tensor inv_var_tensor;
    if (use_global_stats) {
      const auto *running_mean = ctx.Input<Tensor>("Mean");
      const auto *running_variance = ctx.Input<Tensor>("Variance");
      mean_data = running_mean->data<T>();
Z
Zeng Jinle 已提交
675
      inv_var_tensor.Resize({C});
676 677 678 679
      T *running_inv_var_data = inv_var_tensor.mutable_data<T>(ctx.GetPlace());
      EigenVectorArrayMap<T> inv_var_tmp(running_inv_var_data, C);
      ConstEigenVectorArrayMap<T> var_arr(running_variance->data<T>(), C);

680
      inv_var_tmp = (var_arr + epsilon).sqrt().inverse();
681 682 683 684
      inv_var_data = running_inv_var_data;
    }

    ConstEigenVectorArrayMap<T> scale_arr(scale->data<T>(), C);
K
Kaipeng Deng 已提交
685
    ConstEigenVectorArrayMap<T> bias_arr(bias->data<T>(), C);
686 687 688 689 690 691 692 693 694 695 696
    ConstEigenVectorArrayMap<T> mean_arr(mean_data, C);
    ConstEigenVectorArrayMap<T> inv_var_arr(inv_var_data, C);

    T *d_bias_data = nullptr;
    T *d_scale_data = nullptr;
    if (d_scale && d_bias) {
      d_scale->mutable_data<T>(ctx.GetPlace());
      d_bias->mutable_data<T>(ctx.GetPlace());
      d_bias_data = d_bias->mutable_data<T>(ctx.GetPlace());
      d_scale_data = d_scale->mutable_data<T>(ctx.GetPlace());
    }
Q
Qiao Longfei 已提交
697 698 699 700 701

    // d_bias = np.sum(d_y, axis=0)
    // d_scale = np.sum((X - mean) / inv_std * dy, axis=0)
    // d_x = (1. / N) * scale * inv_var * (N * d_y - np.sum(d_y, axis=0)
    //   - (X - mean) * inv_var * inv_var * np.sum(d_y * (X - mean), axis=0))
702 703
    EigenVectorArrayMap<T> d_bias_arr(d_bias_data, C);
    EigenVectorArrayMap<T> d_scale_arr(d_scale_data, C);
Q
Qiao Longfei 已提交
704

705 706 707 708
    if (d_scale && d_bias) {
      d_bias_arr.setZero();
      d_scale_arr.setZero();
    }
Q
Qiao Longfei 已提交
709

710
    if (d_x && (N * sample_size) == 1 && !use_global_stats) {
711
      framework::TensorCopy(*d_y, ctx.GetPlace(), d_x);
712 713 714
      return;
    }

715 716
    int scale_coefff = use_global_stats ? 1 : N * sample_size;
    const auto scale_inv_var_nhw = scale_arr * inv_var_arr / scale_coefff;
Q
Qiao Longfei 已提交
717

L
lvmengsi 已提交
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
    Tensor dy_sum;
    dy_sum.Resize({C});
    dy_sum.mutable_data<T>(ctx.GetPlace());
    EigenVectorArrayMap<T> dy_sum_arr(dy_sum.mutable_data<T>(ctx.GetPlace()),
                                      C);

    Tensor dy_mul_x_sub_mean_mul_invstd_sum;
    dy_mul_x_sub_mean_mul_invstd_sum.Resize({C});
    dy_mul_x_sub_mean_mul_invstd_sum.mutable_data<T>(ctx.GetPlace());
    EigenVectorArrayMap<T> dy_mul_x_sub_mean_mul_invstd_sum_arr(
        dy_mul_x_sub_mean_mul_invstd_sum.mutable_data<T>(ctx.GetPlace()), C);

    dy_sum_arr.setZero();
    dy_mul_x_sub_mean_mul_invstd_sum_arr.setZero();

K
Kaipeng Deng 已提交
733 734 735 736 737 738 739
    // inplace calculation
    // Y:  ((x - est_mean) * (inv_var) * scale + bias
    //   formula transform ====>
    //   (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
    // X: (y - bias) / scale / (inv_var) + est_mean
    //   formula transform ====>
    //    (y - bias) / (scale * inv_var) + est_mean
Q
QI JUN 已提交
740 741
    switch (data_layout) {
      case DataLayout::kNCHW: {
K
Kaipeng Deng 已提交
742 743 744 745 746 747 748 749 750 751 752
        if (is_inplace) {
          auto px = *x;
          EigenArrayMap<T> x_data(px.mutable_data<T>(ctx.GetPlace()),
                                  sample_size, N * C);
          ConstEigenArrayMap<T> y_data(x->data<T>(), sample_size, N * C);
          for (int nc = 0; nc < N * C; ++nc) {
            x_data.col(nc) = (y_data.col(nc) - bias_arr(nc % C)) /
                                 scale_inv_var_nhw(nc % C) / scale_coefff +
                             mean_arr(nc % C);
          }
        }
Q
Qiao Longfei 已提交
753 754 755
        ConstEigenArrayMap<T> x_arr(x->data<T>(), sample_size, N * C);
        ConstEigenArrayMap<T> d_y_arr(d_y->data<T>(), sample_size, N * C);

L
lvmengsi 已提交
756 757 758 759 760 761 762 763
        for (int nc = 0; nc < N * C; ++nc) {
          int c = nc % C;
          dy_sum_arr(c) += d_y_arr.col(nc).sum();
          dy_mul_x_sub_mean_mul_invstd_sum_arr(c) +=
              ((x_arr.col(nc) - mean_arr(c)) * inv_var_arr(c) * d_y_arr.col(nc))
                  .sum();
        }

764
        if (d_scale && d_bias) {
L
lvmengsi 已提交
765 766
          d_bias_arr = dy_sum_arr;
          d_scale_arr = dy_mul_x_sub_mean_mul_invstd_sum_arr;
Q
Qiao Longfei 已提交
767
        }
L
lvmengsi 已提交
768

769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
        if (d_x) {
          EigenArrayMap<T> d_x_arr(d_x->mutable_data<T>(ctx.GetPlace()),
                                   sample_size, N * C);
          if (!use_global_stats) {
            for (int nc = 0; nc < N * C; ++nc) {
              int c = nc % C;
              d_x_arr.col(nc) =
                  scale_inv_var_nhw(c) *
                  (d_y_arr.col(nc) * N * sample_size - dy_sum_arr(c) -
                   (x_arr.col(nc) - mean_arr[c]) *
                       dy_mul_x_sub_mean_mul_invstd_sum_arr(c) *
                       inv_var_arr(c));
            }
          } else {
            for (int nc = 0; nc < N * C; ++nc) {
              int c = nc % C;
              d_x_arr.col(nc) = scale_inv_var_nhw(c) * d_y_arr.col(nc);
            }
787
          }
Q
Qiao Longfei 已提交
788 789 790
        }
        break;
      }
Q
QI JUN 已提交
791
      case DataLayout::kNHWC: {
K
Kaipeng Deng 已提交
792 793 794 795 796 797 798 799 800 801 802
        if (is_inplace) {
          auto px = *x;
          EigenArrayMap<T> x_data(px.mutable_data<T>(ctx.GetPlace()), C,
                                  N * sample_size);
          ConstEigenArrayMap<T> y_data(x->data<T>(), C, N * sample_size);
          for (int nhw = 0; nhw < N * sample_size; nhw++) {
            x_data.col(nhw) = (y_data.col(nhw) - bias_arr) / scale_inv_var_nhw /
                                  scale_coefff +
                              mean_arr;
          }
        }
Q
Qiao Longfei 已提交
803 804 805
        ConstEigenArrayMap<T> x_arr(x->data<T>(), C, N * sample_size);
        ConstEigenArrayMap<T> d_y_arr(d_y->data<T>(), C, N * sample_size);

L
lvmengsi 已提交
806 807 808 809 810
        for (int nhw = 0; nhw < N * sample_size; ++nhw) {
          dy_sum_arr += d_y_arr.col(nhw);
          dy_mul_x_sub_mean_mul_invstd_sum_arr +=
              (x_arr.col(nhw) - mean_arr) * inv_var_arr * d_y_arr.col(nhw);
        }
811 812

        if (d_scale && d_bias) {
L
lvmengsi 已提交
813 814
          d_bias_arr = dy_sum_arr;
          d_scale_arr = dy_mul_x_sub_mean_mul_invstd_sum_arr;
815 816
        }

817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
        if (d_x) {
          EigenArrayMap<T> d_x_arr(d_x->mutable_data<T>(ctx.GetPlace()), C,
                                   N * sample_size);
          if (!use_global_stats) {
            for (int nhw = 0; nhw < N * sample_size; ++nhw) {
              d_x_arr.col(nhw) =
                  scale_inv_var_nhw *
                  (d_y_arr.col(nhw) * N * sample_size - dy_sum_arr -
                   (x_arr.col(nhw) - mean_arr) *
                       dy_mul_x_sub_mean_mul_invstd_sum_arr * inv_var_arr);
            }
          } else {
            for (int nhw = 0; nhw < N * sample_size; ++nhw) {
              d_x_arr.col(nhw) = scale_inv_var_nhw * d_y_arr.col(nhw);
            }
832
          }
Q
Qiao Longfei 已提交
833 834 835 836
        }
        break;
      }
      default:
837 838
        PADDLE_THROW(platform::errors::InvalidArgument(
            "Unknown storage order: %s", data_layout_str));
Q
Qiao Longfei 已提交
839 840 841 842
    }
  }
};

H
hong 已提交
843
template <typename T>
844
void BatchNormGradMaker<T>::Apply(GradOpPtr<T> op) const {
845 846 847 848 849 850 851 852
  op->SetType(this->ForwardOpType() + "_grad");
  op->SetInput("X", this->Input("X"));
  op->SetInput(framework::GradVarName("Y"), this->OutputGrad("Y"));

  op->SetInput("Scale", this->Input("Scale"));
  op->SetInput("Bias", this->Input("Bias"));
  op->SetInput("SavedMean", this->Output("SavedMean"));
  op->SetInput("SavedVariance", this->Output("SavedVariance"));
853 854 855
  if (this->HasOutput("ReserveSpace")) {
    op->SetInput("ReserveSpace", this->Output("ReserveSpace"));
  }
856 857

  // used when setting use_global_stats True during training
858 859
  if (BOOST_GET_CONST(bool, this->GetAttr("use_global_stats")) ||
      BOOST_GET_CONST(bool, this->GetAttr("is_test"))) {
860 861 862
    op->SetInput("Mean", this->Output("MeanOut"));
    op->SetInput("Variance", this->Output("VarianceOut"));
  }
863

864
  op->SetAttrMap(this->Attrs());
Y
Yu Yang 已提交
865

866 867 868 869
  op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
  op->SetOutput(framework::GradVarName("Scale"), this->InputGrad("Scale"));
  op->SetOutput(framework::GradVarName("Bias"), this->InputGrad("Bias"));
}
Y
Yu Yang 已提交
870

871 872 873 874 875 876 877 878
template <typename T>
void BatchNormDoubleGradMaker<T>::Apply(GradOpPtr<T> op) const {
  op->SetType("batch_norm_grad_grad");
  op->SetInput("X", this->Input("X"));
  op->SetInput("Scale", this->Input("Scale"));
  op->SetInput("SavedMean", this->Input("SavedMean"));
  op->SetInput("SavedVariance", this->Input("SavedVariance"));
  if (BOOST_GET_CONST(bool, this->GetAttr("use_global_stats"))) {
879
    op->SetInput("Mean", this->Input("Mean"));
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
    op->SetInput("Variance", this->Input("Variance"));
  }
  op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
  op->SetInput("DDScale", this->OutputGrad(framework::GradVarName("Scale")));
  op->SetInput("DDBias", this->OutputGrad(framework::GradVarName("Bias")));
  op->SetInput("DY", this->Input(framework::GradVarName("Y")));

  op->SetAttrMap(this->Attrs());
  op->SetOutput("DX", this->InputGrad("X"));
  op->SetOutput("DScale", this->InputGrad("Scale"));
  op->SetOutput("DDY", this->InputGrad(framework::GradVarName("Y")));
}

void BatchNormDoubleGradOp::InferShape(
    framework::InferShapeContext *ctx) const {
  OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "BatchNormDoubleGrad");
  OP_INOUT_CHECK(ctx->HasInput("Scale"), "Input", "Scale",
                 "BatchNormDoubleGrad");
  OP_INOUT_CHECK(ctx->HasInput("SavedMean"), "Input", "SavedMean",
                 "BatchNormDoubleGrad");
  OP_INOUT_CHECK(ctx->HasInput("SavedVariance"), "Input", "SavedVariance",
                 "BatchNormDoubleGrad");

  const bool use_global_stats = ctx->Attrs().Get<bool>("use_global_stats");
  if (use_global_stats) {
    OP_INOUT_CHECK(ctx->HasInput("Variance"), "Input", "VarianceOut",
                   "BatchNormDoubleGrad");
  }

  OP_INOUT_CHECK(ctx->HasInput("DY"), "Input", "DY", "BatchNormDoubleGrad");

  // check output
  OP_INOUT_CHECK(ctx->HasOutput("DX"), "Output", "DX", "BatchNormDoubleGrad");

  const auto x_dims = ctx->GetInputDim("X");
915 916 917
  const DataLayout data_layout = framework::StringToDataLayout(
      ctx->Attrs().Get<std::string>("data_layout"));
  const int C =
918
      ((ctx->IsRunMKLDNNKernel() == true) || (data_layout == DataLayout::kNCHW)
919 920 921
           ? x_dims[1]
           : x_dims[x_dims.size() - 1]);

922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995
  if (ctx->HasOutput("DX")) {
    ctx->SetOutputDim("DX", x_dims);
  }
  if (ctx->HasOutput("DScale")) {
    ctx->SetOutputDim("DScale", {C});
  }
  if (ctx->HasOutput("DDY")) {
    ctx->ShareDim("X", "DDY");
  }
}

framework::OpKernelType BatchNormDoubleGradOp::GetExpectedKernelType(
    const framework::ExecutionContext &ctx) const {
  const auto *var = ctx.InputVar("DY");
  if (var == nullptr) {
    PADDLE_THROW(
        platform::errors::NotFound("cannot find gradient variable of Y"));
  }
  const Tensor *t = nullptr;
  if (var->IsType<Tensor>()) {
    t = &var->Get<Tensor>();
  } else if (var->IsType<LoDTensor>()) {
    t = &var->Get<LoDTensor>();
  }
  if (t == nullptr) {
    PADDLE_THROW(
        platform::errors::InvalidArgument("gradient variable of Y is empty"));
  }
  return framework::OpKernelType(
      OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
}

template <typename T>
class BatchNormDoubleGradKernel<platform::CPUDeviceContext, T>
    : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext &ctx) const override {
    const auto *X = ctx.Input<Tensor>("X");
    const auto *Scale = ctx.Input<Tensor>("Scale");
    const auto *dY = ctx.Input<Tensor>("DY");
    const auto *Saved_mean = ctx.Input<Tensor>("SavedMean");
    const auto *Saved_variance = ctx.Input<Tensor>("SavedVariance");
    const float epsilon = ctx.Attr<float>("epsilon");
    const bool use_global_stats = ctx.Attr<bool>("use_global_stats");
    const bool is_test = ctx.Attr<bool>("is_test");

    PADDLE_ENFORCE_EQ(
        is_test, false,
        platform::errors::InvalidArgument(
            "`is_test = True` CANNOT be used in train program. If "
            "you want to use global status in pre_train model, "
            "please set `use_global_stats = True`"));

    const std::string data_layout_str = ctx.Attr<std::string>("data_layout");
    const DataLayout data_layout =
        framework::StringToDataLayout(data_layout_str);

    const auto *ddX = ctx.Input<Tensor>("DDX");
    const auto *ddScale = ctx.Input<Tensor>("DDScale");
    const auto *ddBias = ctx.Input<Tensor>("DDBias");

    auto *dX = ctx.Output<Tensor>("DX");
    auto *dScale = ctx.Output<Tensor>("DScale");
    auto *ddY = ctx.Output<Tensor>("DDY");
    dX->mutable_data<T>(ctx.GetPlace());
    ddY->mutable_data<T>(ctx.GetPlace());

    auto &dev_ctx = ctx.template device_context<platform::CPUDeviceContext>();

    const auto &x_dims = X->dims();
    const int C =
        (data_layout == DataLayout::kNCHW ? x_dims[1]
                                          : x_dims[x_dims.size() - 1]);
    const int sample_size = X->numel() / C;
996
    phi::funcs::SetConstant<platform::CPUDeviceContext, T> set_constant;
997 998 999 1000 1001 1002

    const T *mean_data = Saved_mean->data<T>();
    const T *inv_var_data = Saved_variance->data<T>();

    Tensor inv_var_tensor;
    if (use_global_stats) {
1003
      const auto *running_mean = ctx.Input<Tensor>("Mean");
1004
      const auto *running_variance = ctx.Input<Tensor>("Variance");
1005
      mean_data = running_mean->data<T>();
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
      inv_var_tensor.Resize({C});

      T *running_inv_var_data = inv_var_tensor.mutable_data<T>(ctx.GetPlace());
      EigenVectorArrayMap<T> inv_var_tmp(running_inv_var_data, C);
      ConstEigenVectorArrayMap<T> var_arr(running_variance->data<T>(), C);

      inv_var_tmp = (var_arr + epsilon).sqrt().inverse();
      inv_var_data = running_inv_var_data;
    }

    // transpose NCHW -> NHWC for easy calculate
    Tensor transformed_x(X->type());
    Tensor transformed_dy(dY->type());
    Tensor transformed_ddx(ddX->type());

    Tensor transformed_dx(dX->type());
    Tensor transformed_ddy(ddY->type());
    if (data_layout == DataLayout::kNCHW && x_dims.size() > 2) {
      VLOG(3) << "Transform batchnorm output from NCHW to NHWC";
      // Input Tensor
      ResizeToChannelLast<platform::CPUDeviceContext, T>(ctx, X,
                                                         &transformed_x);
      TransToChannelLast<platform::CPUDeviceContext, T>(ctx, X, &transformed_x);
      ResizeToChannelLast<platform::CPUDeviceContext, T>(ctx, dY,
                                                         &transformed_dy);
      TransToChannelLast<platform::CPUDeviceContext, T>(ctx, dY,
                                                        &transformed_dy);
      ResizeToChannelLast<platform::CPUDeviceContext, T>(ctx, ddX,
                                                         &transformed_ddx);
      TransToChannelLast<platform::CPUDeviceContext, T>(ctx, ddX,
                                                        &transformed_ddx);
      // Output Tensor
      ResizeToChannelLast<platform::CPUDeviceContext, T>(ctx, dX,
                                                         &transformed_dx);
      ResizeToChannelLast<platform::CPUDeviceContext, T>(ctx, ddY,
                                                         &transformed_ddy);
    } else {
      transformed_x.ShareDataWith(*X);
      transformed_dy.ShareDataWith(*dY);
      transformed_ddx.ShareDataWith(*ddX);

      transformed_dx.ShareDataWith(*dX);
      transformed_ddy.ShareDataWith(*ddY);
    }

    ConstEigenArrayMap<T> x_arr(transformed_x.data<T>(), C, sample_size);
    ConstEigenVectorArrayMap<T> mean_arr(mean_data, C);
    ConstEigenVectorArrayMap<T> inv_var_arr(inv_var_data, C);

    Tensor mean_tile;
    mean_tile.Resize({C, sample_size});
    mean_tile.mutable_data<T>(ctx.GetPlace());
    EigenArrayMap<T> mean_tile_data(mean_tile.mutable_data<T>(ctx.GetPlace()),
                                    C, sample_size);

    Tensor inv_var_tile;
    inv_var_tile.Resize({C, sample_size});
    inv_var_tile.mutable_data<T>(ctx.GetPlace());
    EigenArrayMap<T> inv_var_tile_data(
        inv_var_tile.mutable_data<T>(ctx.GetPlace()), C, sample_size);

    mean_tile_data = mean_arr.replicate(1, sample_size);
    inv_var_tile_data = inv_var_arr.replicate(1, sample_size);

    Tensor Scale_data;
    if (!Scale) {
      Scale_data.mutable_data<T>({C}, ctx.GetPlace());
      set_constant(dev_ctx, &Scale_data, static_cast<T>(1));
    }
    ConstEigenVectorArrayMap<T> scale_arr(
        Scale ? Scale->data<T>() : Scale_data.data<T>(), C);

    Tensor scale_tile;
    scale_tile.Resize({C, sample_size});
    scale_tile.mutable_data<T>(ctx.GetPlace());
    EigenArrayMap<T> scale_tile_data(scale_tile.mutable_data<T>(ctx.GetPlace()),
                                     C, sample_size);
    scale_tile_data = scale_arr.replicate(1, sample_size);

    ConstEigenArrayMap<T> dy_arr(transformed_dy.data<T>(), C, sample_size);
    ConstEigenArrayMap<T> ddx_arr(transformed_ddx.data<T>(), C, sample_size);

    Tensor x_sub_mean_mul_invstd;
    x_sub_mean_mul_invstd.Resize({C, sample_size});
    x_sub_mean_mul_invstd.mutable_data<T>(ctx.GetPlace());
    EigenArrayMap<T> x_sub_mean_mul_invstd_arr(
        x_sub_mean_mul_invstd.mutable_data<T>(ctx.GetPlace()), C, sample_size);
    x_sub_mean_mul_invstd_arr = (x_arr - mean_tile_data) * inv_var_tile_data;

    if (dX) {
      dX->mutable_data<T>(ctx.GetPlace());
      EigenArrayMap<T> dx_arr(transformed_dx.mutable_data<T>(ctx.GetPlace()), C,
                              sample_size);
      dx_arr.setZero();
      if (use_global_stats) {
        // math: dx = (ddscale * dy) * inv_var
        if (ddScale) {
          ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
          Tensor ddscale_tile;
          ddscale_tile.Resize({C, sample_size});
          EigenArrayMap<T> ddscale_tile_data(
              ddscale_tile.mutable_data<T>(ctx.GetPlace()), C, sample_size);
          ddscale_tile_data = ddscale_arr.replicate(1, sample_size);

          dx_arr = dy_arr * ddscale_tile_data * inv_var_tile_data;
        }
      } else {
        // math: dx = scale * ((x - mean) * inv_var / NxHxW * (np.mean(ddx,
        // axis=(n,h,w)) *
        //          np.sum(dy, axis=(n,h,w)) -
        //          np.sum(dy * ddx, axis=(n,h,w)) + 3 * np.mean(dy * (x -
        //          mean),
        //          axis=(n,h,w)) * inv_var.pow(2) *
        //          np.sum(ddx * (x - mean), axis=(n,h,w))) + inv_var.pow(3) /
        //          NxHxW *
        //          np.sum(ddx * (x - mean)) *
        //          (np.mean(dy, axis=(n,h,w)) - dy) + inv_var.pow(3) / NxHxW *
        //          np.sum(dy,
        //          axis=(n,h,w)) * (x - mean) *
1125
        //          (np.mean(ddx, axis=(n,h,w)) - ddx)) + ddr * (dy * inv_var -
1126 1127 1128 1129
        //          inv_var
        //          *
        //          np.mean(dy, axis=(n,h,w)) -
        //          inv_var.pow(3) * (x - mean) * np.mean(dy * (x - mean),
1130
        //          axis=(n,h,w)))
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223

        if (ddX) {
          dx_arr +=
              (x_sub_mean_mul_invstd_arr * inv_var_tile_data *
               inv_var_tile_data / sample_size)
                  .colwise() *
              (ddx_arr.rowwise().sum() * dy_arr.rowwise().sum() / sample_size -
               (dy_arr * ddx_arr).rowwise().sum() +
               3. * (dy_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() *
                   (ddx_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
                   sample_size);

          dx_arr += (inv_var_tile_data * inv_var_tile_data).colwise() *
                    (ddx_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
                    sample_size *
                    (dy_arr.rowwise().sum() / sample_size - dy_arr);

          dx_arr += (inv_var_tile_data * inv_var_tile_data).colwise() *
                    (dy_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
                    sample_size *
                    (ddx_arr.rowwise().sum() / sample_size - ddx_arr);

          dx_arr = scale_tile_data * dx_arr;
        }
        if (ddScale) {
          ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
          Tensor ddscale_tile;
          ddscale_tile.Resize({C, sample_size});
          EigenArrayMap<T> ddscale_tile_data(
              ddscale_tile.mutable_data<T>(ctx.GetPlace()), C, sample_size);
          ddscale_tile_data = ddscale_arr.replicate(1, sample_size);

          dx_arr += (dy_arr * inv_var_tile_data -
                     (dy_arr.rowwise().sum().replicate(1, sample_size) /
                      sample_size) *
                         inv_var_tile_data -
                     x_sub_mean_mul_invstd_arr * inv_var_tile_data *
                         (dy_arr * x_sub_mean_mul_invstd_arr)
                             .rowwise()
                             .sum()
                             .replicate(1, sample_size) /
                         sample_size) *
                    ddscale_tile_data;
        }
      }
      if (data_layout == DataLayout::kNCHW) {
        VLOG(3) << "Transform batchnorm output from NHWC to NCHW";
        TransToChannelFirst<paddle::platform::CPUDeviceContext, T>(
            ctx, &transformed_dx, dX);
      }
    }
    if (dScale) {
      dScale->mutable_data<T>(ctx.GetPlace());
      EigenVectorArrayMap<T> dscale_arr(dScale->mutable_data<T>(ctx.GetPlace()),
                                        C);
      dscale_arr.setZero();
      if (use_global_stats) {
        // math: dscale = np.sum(ddx * dy, axis=(n,h,w)) * inv_var
        if (ddX) {
          dscale_arr = (ddx_arr * dy_arr * inv_var_tile_data).rowwise().sum();
        }
      } else {
        // math: dscale = inv_var * (dy - np.mean(dy, axis=(n,h,w) - (x-mean) *
        //            inv_var.pow(2) * np.mean(dy * (x-mean), axis=(n,h,w)))) *
        //            ddx
        if (ddX) {
          Tensor first_grad;
          first_grad.Resize({C, sample_size});
          EigenArrayMap<T> first_grad_arr(
              first_grad.mutable_data<T>(ctx.GetPlace()), C, sample_size);
          first_grad_arr.setZero();

          first_grad_arr +=
              inv_var_tile_data *
              (dy_arr -
               dy_arr.rowwise().sum().replicate(1, sample_size) / sample_size -
               x_sub_mean_mul_invstd_arr *
                   (dy_arr * x_sub_mean_mul_invstd_arr)
                       .rowwise()
                       .sum()
                       .replicate(1, sample_size) /
                   sample_size);
          dscale_arr = (first_grad_arr * ddx_arr).rowwise().sum();
        }
      }
    }

    if (ddY) {
      ddY->mutable_data<T>(ctx.GetPlace());
      EigenArrayMap<T> ddy_arr(transformed_ddy.mutable_data<T>(ctx.GetPlace()),
                               C, sample_size);
      ddy_arr.setZero();
      if (use_global_stats) {
1224 1225
        // math: ddy = r * ddx * inv_var + ddbias +
        //           ddscale * (x - mean) * inv_var
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
        if (ddX) {
          ddy_arr = scale_tile_data * ddx_arr * inv_var_tile_data;
        }
      } else {
        // math: ddy = (x - mean) * inv_var * ddscale + ddbias +
        //           scale * inv_var * (ddx - (x - mean) * inv_var.pow(2) *
        //           np.mean(ddx * (x - mean), axis=(n,h,w)))
        if (ddX) {
          ddy_arr +=
              scale_tile_data * inv_var_tile_data *
              (ddx_arr -
               ddx_arr.rowwise().sum().replicate(1, sample_size) / sample_size -
               x_sub_mean_mul_invstd_arr *
                   (ddx_arr * x_sub_mean_mul_invstd_arr)
                       .rowwise()
                       .sum()
                       .replicate(1, sample_size) /
                   sample_size);
        }
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
      }
      if (ddScale) {
        ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
        Tensor ddscale_tile;
        ddscale_tile.Resize({C, sample_size});
        EigenArrayMap<T> ddscale_tile_data(
            ddscale_tile.mutable_data<T>(ctx.GetPlace()), C, sample_size);
        ddscale_tile_data = ddscale_arr.replicate(1, sample_size);

        ddy_arr += x_sub_mean_mul_invstd_arr * ddscale_tile_data;
      }
1256

1257 1258 1259 1260 1261 1262 1263
      if (ddBias) {
        ConstEigenVectorArrayMap<T> ddbias_arr(ddBias->data<T>(), C);
        Tensor ddbias_tile;
        ddbias_tile.Resize({C, sample_size});
        EigenArrayMap<T> ddbias_tile_data(
            ddbias_tile.mutable_data<T>(ctx.GetPlace()), C, sample_size);
        ddbias_tile_data = ddbias_arr.replicate(1, sample_size);
1264

1265
        ddy_arr += ddbias_tile_data;
1266
      }
1267

1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
      if (data_layout == DataLayout::kNCHW) {
        VLOG(3) << "Transform batchnorm output from NHWC to NCHW";
        TransToChannelFirst<paddle::platform::CPUDeviceContext, T>(
            ctx, &transformed_ddy, ddY);
      }
    }
  }
};

DECLARE_INPLACE_OP_INFERER(BatchNormDoubleGradOpInplaceInferer, {"DY", "DDY"});

Q
Qiao Longfei 已提交
1279 1280 1281 1282
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
Y
Yu Yang 已提交
1283
REGISTER_OPERATOR(batch_norm, ops::BatchNormOp, ops::BatchNormOpMaker,
H
hong 已提交
1284 1285 1286
                  ops::BatchNormOpInferVarType,
                  ops::BatchNormGradMaker<paddle::framework::OpDesc>,
                  ops::BatchNormGradMaker<paddle::imperative::OpBase>);
1287 1288 1289 1290 1291
REGISTER_OPERATOR(batch_norm_grad, ops::BatchNormGradOp,
                  ops::BatchNormDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::BatchNormDoubleGradMaker<paddle::imperative::OpBase>);
REGISTER_OPERATOR(batch_norm_grad_grad, ops::BatchNormDoubleGradOp,
                  ops::BatchNormDoubleGradOpInplaceInferer);
Y
Yu Yang 已提交
1292

Q
QI JUN 已提交
1293
REGISTER_OP_CPU_KERNEL(
D
dzhwinter 已提交
1294 1295
    batch_norm, ops::BatchNormKernel<paddle::platform::CPUDeviceContext, float>,
    ops::BatchNormKernel<paddle::platform::CPUDeviceContext, double>);
Q
Qiao Longfei 已提交
1296 1297
REGISTER_OP_CPU_KERNEL(
    batch_norm_grad,
D
dzhwinter 已提交
1298 1299
    ops::BatchNormGradKernel<paddle::platform::CPUDeviceContext, float>,
    ops::BatchNormGradKernel<paddle::platform::CPUDeviceContext, double>);
1300 1301 1302 1303
REGISTER_OP_CPU_KERNEL(
    batch_norm_grad_grad,
    ops::BatchNormDoubleGradKernel<paddle::platform::CPUDeviceContext, float>,
    ops::BatchNormDoubleGradKernel<paddle::platform::CPUDeviceContext, double>);