activation_op.cc 42.2 KB
Newer Older
1
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Q
qijun 已提交
2

L
Luo Tao 已提交
3 4 5
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Q
qijun 已提交
6

L
Luo Tao 已提交
7
    http://www.apache.org/licenses/LICENSE-2.0
Q
qijun 已提交
8

L
Luo Tao 已提交
9 10 11 12 13
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
Q
qijun 已提交
14

Y
Yi Wang 已提交
15
#include "paddle/fluid/operators/activation_op.h"
T
tink2123 已提交
16
#include <memory>
D
dzhwinter 已提交
17
#include <string>
18
#include <type_traits>
T
tink2123 已提交
19
#include <unordered_map>
20
#include <vector>
21
#include "paddle/fluid/operators/mkldnn/mkldnn_activation_op.h"
D
dzhwinter 已提交
22
#include "paddle/fluid/platform/port.h"
23 24 25
#ifdef PADDLE_WITH_CUDA
#include "paddle/fluid/platform/cudnn_helper.h"
#endif
Q
qijun 已提交
26

A
Adam 已提交
27 28
DECLARE_bool(use_mkldnn);

Q
qijun 已提交
29 30 31
namespace paddle {
namespace operators {

32 33
using paddle::framework::Tensor;

34 35 36 37 38
template <typename GradFunctor>
static constexpr bool CanInplaceAct() {
  return GradFunctor::FwdDeps() == kDepOut || GradFunctor::FwdDeps() == kNoDeps;
}

39 40 41 42 43
#define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT)                    \
  class OP_NAME##OpMaker                                                     \
      : public ::paddle::framework::OpProtoAndCheckerMaker {                 \
   public:                                                                   \
    void Make() override {                                                   \
44 45 46 47 48
      AddInput("X", "Input of " #OP_NAME                                     \
                    " operator, an N-D Tensor, with data type float32, "     \
                    "float64 or float16.");                                  \
      AddOutput("Out", "Output of " #OP_NAME                                 \
                       " operator, a Tensor with shape same as input.");     \
49 50 51 52 53 54 55 56 57
      AddAttr<bool>("use_mkldnn",                                            \
                    "(bool, default false) Only used in mkldnn kernel")      \
          .SetDefault(false);                                                \
      AddAttr<bool>("use_cudnn",                                             \
                    "(bool, default false) Only used in cudnn kernel, need " \
                    "install cudnn")                                         \
          .SetDefault(false);                                                \
      AddComment(OP_COMMENT);                                                \
    }                                                                        \
D
dzhwinter 已提交
58
  }
D
dzhwinter 已提交
59

H
hong 已提交
60 61
template <ActBwdOpFwdDeps kDepValue, typename T>
class ActivationGradOpMaker : public framework::SingleGradOpMaker<T> {
62
 public:
H
hong 已提交
63
  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
64 65

 protected:
66
  void Apply(GradOpPtr<T> op) const override {
H
hong 已提交
67 68 69 70
    op->SetType(this->ForwardOpType() + "_grad");
    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
    op->SetAttrMap(this->Attrs());
71

A
Adam 已提交
72 73
    if ((static_cast<int>(kDepValue) &
         static_cast<int>(ActBwdOpFwdDeps::kDepX)) ||
74 75 76
        FLAGS_use_mkldnn ||
        (op->HasAttr("use_mkldnn") &&
         BOOST_GET_CONST(bool, op->GetAttr("use_mkldnn")))) {
H
hong 已提交
77
      op->SetInput("X", this->Input("X"));
78 79 80 81
    }

    if (static_cast<int>(kDepValue) &
        static_cast<int>(ActBwdOpFwdDeps::kDepOut)) {
H
hong 已提交
82
      op->SetInput("Out", this->Output("Out"));
83
    }
D
dzhwinter 已提交
84
  }
85
};
D
dzhwinter 已提交
86

87 88 89 90
framework::OpKernelType GetKernelType(const framework::ExecutionContext& ctx,
                                      const framework::OperatorWithKernel& oper,
                                      const std::string& name) {
  framework::LibraryType library{framework::LibraryType::kPlain};
M
mozga-intel 已提交
91
  framework::DataLayout layout = framework::DataLayout::kAnyLayout;
92 93 94 95 96 97 98 99 100 101
// FIXME(liuwei1031) temporarily disable the code to unblock users
// TODO(liuwei1031) figure out the reason behind
// https://github.com/PaddlePaddle/Paddle/issues/16096
// and re-enable this in the future
// #ifdef PADDLE_WITH_CUDA
//   auto it1 = oper.Attrs().find("use_cudnn");
//   if (it1 != oper.Attrs().end() && platform::CanCUDNNBeUsed(ctx)) {
//     library = framework::LibraryType::kCUDNN;
//   }
// #endif
102 103 104 105 106
#ifdef PADDLE_WITH_MKLDNN
  auto it = oper.Attrs().find("use_mkldnn");
  if (library == framework::LibraryType::kPlain && it != oper.Attrs().end() &&
      platform::CanMKLDNNBeUsed(ctx)) {
    library = framework::LibraryType::kMKLDNN;
M
mozga-intel 已提交
107
    layout = framework::DataLayout::kMKLDNN;
108 109
  }
#endif
110 111
  return framework::OpKernelType(oper.IndicateVarDataType(ctx, name),
                                 ctx.GetPlace(), layout, library);
112 113
}

Q
qijun 已提交
114 115 116 117
class ActivationOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

118
  void InferShape(framework::InferShapeContext* ctx) const override {
119
    ctx->ShareDim("X", /*->*/ "Out");
F
fengjiayi 已提交
120
    ctx->ShareLoD("X", /*->*/ "Out");
Q
qijun 已提交
121
  }
122

123
 protected:
124 125 126 127
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return GetKernelType(ctx, *this, "X");
  }
Q
qijun 已提交
128 129
};

C
chengduo 已提交
130 131 132
class ActivationOpInferVarType
    : public framework::PassInDtypeAndVarTypeToOutput {
 protected:
133
  std::unordered_map<std::string, std::string>& GetInputOutputWithSameType()
C
chengduo 已提交
134
      const override {
135 136
    static std::unordered_map<std::string, std::string> m{{"X", /*->*/ "Out"}};
    return m;
137 138 139
  }
};

Q
qijun 已提交
140 141 142 143
class ActivationOpGrad : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

144
  void InferShape(framework::InferShapeContext* ctx) const override {
145 146 147
    auto out_grad_name = framework::GradVarName("Out");
    ctx->ShareDim(out_grad_name, framework::GradVarName("X"));
    ctx->ShareLoD(out_grad_name, framework::GradVarName("X"));
Q
qijun 已提交
148
  }
149

150
 protected:
151 152
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
153
    return GetKernelType(ctx, *this, framework::GradVarName("Out"));
154
  }
Q
qijun 已提交
155 156
};

D
dzhwinter 已提交
157
UNUSED constexpr char SigmoidDoc[] = R"DOC(
158
Sigmoid Activation Operator
K
Kexin Zhao 已提交
159

160
$$out = \\frac{1}{1 + e^{-x}}$$
K
Kexin Zhao 已提交
161

D
dzhwinter 已提交
162
)DOC";
Q
qijun 已提交
163

D
dzhwinter 已提交
164
UNUSED constexpr char LogSigmoidDoc[] = R"DOC(
165
Logsigmoid Activation Operator
K
Kexin Zhao 已提交
166

167
$$out = \\log \\frac{1}{1 + e^{-x}}$$
K
Kexin Zhao 已提交
168

D
dzhwinter 已提交
169
)DOC";
170

D
dzhwinter 已提交
171
UNUSED constexpr char ExpDoc[] = R"DOC(
172
Exp Operator. Computes exp of x element-wise with a natural number :math:`e` as the base.
K
Kexin Zhao 已提交
173

174
$$out = e^x$$
K
Kexin Zhao 已提交
175

D
dzhwinter 已提交
176
)DOC";
Q
qijun 已提交
177

D
dzhwinter 已提交
178
UNUSED constexpr char ReluDoc[] = R"DOC(
K
kexinzhao 已提交
179
Relu Activation Operator.
K
Kexin Zhao 已提交
180

181
$$out = \max(x, 0)$$
K
Kexin Zhao 已提交
182

D
dzhwinter 已提交
183
)DOC";
K
Kexin Zhao 已提交
184

D
dzhwinter 已提交
185
UNUSED constexpr char TanhDoc[] = R"DOC(
K
kexinzhao 已提交
186
Tanh Activation Operator.
K
Kexin Zhao 已提交
187

Q
update  
qiaolongfei 已提交
188
$$out = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$
K
Kexin Zhao 已提交
189

D
dzhwinter 已提交
190
)DOC";
191

D
dzhwinter 已提交
192
UNUSED constexpr char TanhShrinkDoc[] = R"DOC(
K
kexinzhao 已提交
193
TanhShrink Activation Operator.
K
Kexin Zhao 已提交
194

Y
Yan Chunwei 已提交
195
$$out = x - \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$
K
Kexin Zhao 已提交
196

D
dzhwinter 已提交
197
)DOC";
K
Kexin Zhao 已提交
198

D
dzhwinter 已提交
199
UNUSED constexpr char SqrtDoc[] = R"DOC(
K
kexinzhao 已提交
200
Sqrt Activation Operator.
K
Kexin Zhao 已提交
201

202
.. math:: out=\sqrt x=x^{1/2}
203

204 205
**Note**:
  input value must be greater than or equal to zero.
K
Kexin Zhao 已提交
206

D
dzhwinter 已提交
207
)DOC";
208

Z
zhoukunsheng 已提交
209 210 211 212 213
UNUSED constexpr char RsqrtDoc[] = R"DOC(
Rsqrt Activation Operator.

Please make sure input is legal in case of numeric errors.

214
$$out = \frac{1}{\sqrt{x}}$$
Z
zhoukunsheng 已提交
215 216 217

)DOC";

D
dzhwinter 已提交
218
UNUSED constexpr char AbsDoc[] = R"DOC(
K
kexinzhao 已提交
219
Abs Activation Operator.
K
Kexin Zhao 已提交
220

221
$$out = |x|$$
K
Kexin Zhao 已提交
222

D
dzhwinter 已提交
223
)DOC";
224

D
dzhwinter 已提交
225
UNUSED constexpr char CeilDoc[] = R"DOC(
226
Ceil Operator. Computes ceil of x element-wise.
D
dzhwinter 已提交
227

228
$$out = \left \lceil x \right \rceil$$
D
dzhwinter 已提交
229

D
dzhwinter 已提交
230
)DOC";
D
dzhwinter 已提交
231

D
dzhwinter 已提交
232
UNUSED constexpr char FloorDoc[] = R"DOC(
D
dzhwinter 已提交
233 234
Floor Activation Operator.

235
$$out = \left \lfloor x \right \rfloor$$
D
dzhwinter 已提交
236

D
dzhwinter 已提交
237
)DOC";
D
dzhwinter 已提交
238

D
dzhwinter 已提交
239
UNUSED constexpr char CosDoc[] = R"DOC(
240
Cosine Operator. Computes cosine of x element-wise.
C
add cos  
chengduoZH 已提交
241

242
$$out = cos(x)$$
C
add cos  
chengduoZH 已提交
243

D
dzhwinter 已提交
244
)DOC";
C
add cos  
chengduoZH 已提交
245

D
dzhwinter 已提交
246
UNUSED constexpr char SinDoc[] = R"DOC(
C
add sin  
chengduoZH 已提交
247 248
Sine Activation Operator.

249
$$out = sin(x)$$
C
add sin  
chengduoZH 已提交
250

D
dzhwinter 已提交
251
)DOC";
C
add sin  
chengduoZH 已提交
252

D
dzhwinter 已提交
253
UNUSED constexpr char RoundDoc[] = R"DOC(
254
The OP rounds the values in the input to the nearest integer value.
D
dzhwinter 已提交
255

256 257 258 259 260 261 262 263 264
.. code-block:: python

  input:
    x.shape = [4]
    x.data = [1.2, -0.9, 3.4, 0.9]

  output:
    out.shape = [4]
    out.data = [1., -1., 3., 1.]
D
dzhwinter 已提交
265

D
dzhwinter 已提交
266
)DOC";
D
dzhwinter 已提交
267

D
dzhwinter 已提交
268
UNUSED constexpr char ReciprocalDoc[] = R"DOC(
K
kexinzhao 已提交
269
Reciprocal Activation Operator.
K
Kexin Zhao 已提交
270

271
$$out = \\frac{1}{x}$$
K
Kexin Zhao 已提交
272

D
dzhwinter 已提交
273
)DOC";
274

D
dzhwinter 已提交
275
UNUSED constexpr char LogDoc[] = R"DOC(
K
kexinzhao 已提交
276
Log Activation Operator.
K
Kexin Zhao 已提交
277

278
$$out = \ln(x)$$
K
Kexin Zhao 已提交
279 280 281

Natural logarithm of x.

D
dzhwinter 已提交
282 283
)DOC";

284 285 286 287 288 289 290 291 292
UNUSED constexpr char Log1pDoc[] = R"DOC(
Log Activation Operator.

$out = \ln(x+1)$

Natural logarithm of x.

)DOC";

D
dzhwinter 已提交
293
UNUSED constexpr char SquareDoc[] = R"DOC(
294
The OP square each elements of the inputs.
D
dzhwinter 已提交
295

296
$$out = x^2$$
297

D
dzhwinter 已提交
298 299
)DOC";

D
dzhwinter 已提交
300
UNUSED constexpr char SoftplusDoc[] = R"DOC(
D
dzhwinter 已提交
301 302
Softplus Activation Operator.

303
$$out = \ln(1 + e^{x})$$
D
dzhwinter 已提交
304 305 306

)DOC";

D
dzhwinter 已提交
307
UNUSED constexpr char SoftsignDoc[] = R"DOC(
D
dzhwinter 已提交
308 309
Softsign Activation Operator.

310
$$out = \\frac{x}{1 + \|x\|}$$
D
dzhwinter 已提交
311 312 313

)DOC";

T
tink2123 已提交
314 315 316 317 318 319
class AcosOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() override {
    AddInput("X", "Input of acos operator");
    AddOutput("Out", "Output of acos operator");
    AddComment(R"DOC(
320 321
Arccosine Activation Operator.

T
tink2123 已提交
322
$$out = \cos^{-1}(x)$$
323

T
tink2123 已提交
324 325 326
)DOC");
  }
};
327

T
tink2123 已提交
328 329 330 331 332 333
class AsinOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() override {
    AddInput("X", "Input of asin operator");
    AddOutput("Out", "Output of asin operator");
    AddComment(R"DOC(
334 335
Arcsine Activation Operator.

T
tink2123 已提交
336
$$out = \sin^{-1}(x)$$
337

T
tink2123 已提交
338 339 340
)DOC");
  }
};
341

T
tink2123 已提交
342 343 344 345 346 347
class AtanOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() override {
    AddInput("X", "Input of atan operator");
    AddOutput("Out", "Output of atan operator");
    AddComment(R"DOC(
348 349
Arctanh Activation Operator.

T
tink2123 已提交
350
$$out = \tanh^{-1}(x)$$
351

T
tink2123 已提交
352 353 354
)DOC");
  }
};
355

D
dzhwinter 已提交
356
class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker {
357
 public:
Y
Yu Yang 已提交
358
  void Make() override {
W
Wilber 已提交
359 360 361 362 363 364 365 366
    AddInput("X",
             "A LoDTensor or Tensor representing preactivation values. Must be "
             "one of the following types: float32, float64.");
    AddOutput(
        "Out",
        "A LoDTensor or Tensor with the same type and size as that of x.");
    AddAttr<float>("alpha", "Slope of the activation function at x < 0.")
        .SetDefault(0.02f);
A
Adam 已提交
367 368 369
    AddAttr<bool>("use_mkldnn",
                  "(bool, default false) Only used in mkldnn kernel")
        .SetDefault(false);
K
Kexin Zhao 已提交
370
    AddComment(R"DOC(
D
dzhwinter 已提交
371
LeakyRelu Activation Operator.
K
Kexin Zhao 已提交
372

W
Wilber 已提交
373
$$out = \max(x, \alpha * x)$$
K
Kexin Zhao 已提交
374 375

)DOC");
376 377 378
  }
};

D
dzhwinter 已提交
379
class SoftShrinkOpMaker : public framework::OpProtoAndCheckerMaker {
K
kexinzhao 已提交
380
 public:
Y
Yu Yang 已提交
381
  void Make() override {
D
dzhwinter 已提交
382 383 384
    AddInput("X", "Input of Softshrink operator");
    AddOutput("Out", "Output of Softshrink operator");
    AddAttr<float>("lambda", "non-negative offset").SetDefault(0.5f);
K
Kexin Zhao 已提交
385
    AddComment(R"DOC(
386 387 388
:strong:`Softshrink Activation Operator`

..  math::
389
    out = \begin{cases}
390 391 392 393
         x - \lambda, \text{if } x > \lambda \\
         x + \lambda, \text{if } x < -\lambda \\
         0,  \text{otherwise}
         \end{cases}
K
Kexin Zhao 已提交
394 395

)DOC");
K
kexinzhao 已提交
396 397 398
  }
};

D
dzhwinter 已提交
399
class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker {
400
 public:
Y
Yu Yang 已提交
401
  void Make() override {
D
dzhwinter 已提交
402 403
    AddInput("X", "Input of HardShrink operator");
    AddOutput("Out", "Output of HardShrink operator");
Y
yuyang18 已提交
404 405
    AddAttr<float>("threshold",
                   "The value of threshold for HardShrink. [default: 0.5]")
D
dzhwinter 已提交
406
        .SetDefault(0.5f);
K
Kexin Zhao 已提交
407
    AddComment(R"DOC(
Y
yuyang18 已提交
408
:strong:`HardShrink activation operator`
K
Kexin Zhao 已提交
409

Y
yuyang18 已提交
410 411 412 413 414 415
..  math::
    out = \begin{cases}
            x, \text{if } x > \lambda \\
            x, \text{if } x < -\lambda \\
            0,  \text{otherwise}
          \end{cases}
K
Kexin Zhao 已提交
416 417

)DOC");
418 419 420
  }
};

421 422
class BReluOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
423
  void Make() override {
424 425 426 427 428 429
    AddInput("X",
             "The input is a multi-dimensional Tensor. The data type is "
             "float32, float64.");
    AddOutput("Out",
              "The output is a multi-dimensional Tensor which has same "
              "dimension and data type as the ``X``.");
430 431 432 433
    AddAttr<float>("t_min", "The min marginal value of BRelu")
        .SetDefault(static_cast<float>(0));
    AddAttr<float>("t_max", "The max marginal value of BRelu")
        .SetDefault(static_cast<float>(24));
K
Kexin Zhao 已提交
434
    AddComment(R"DOC(
K
kexinzhao 已提交
435
BRelu Activation Operator.
K
Kexin Zhao 已提交
436

437
$$out = \min(\max(x, t_{min}), t_{max})$$
K
Kexin Zhao 已提交
438 439

)DOC");
440 441 442 443 444
  }
};

class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
445
  void Make() override {
446
    AddInput("X", "Input of SoftRelu operator");
F
fengjiayi 已提交
447
    AddOutput("Out", "Output of SoftRelu operator");
448 449
    AddAttr<float>("threshold", "The threshold value of SoftRelu")
        .SetDefault(40.0f);
K
Kexin Zhao 已提交
450
    AddComment(R"DOC(
K
kexinzhao 已提交
451
SoftRelu Activation Operator.
K
Kexin Zhao 已提交
452

453
$$out = \ln(1 + \exp(\max(\min(x, threshold), -threshold)))$$
K
Kexin Zhao 已提交
454 455

)DOC");
456 457 458
  }
};

459 460
class ELUOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
461
  void Make() override {
462 463 464 465 466 467
    AddInput("X",
             "The input is a multi-dimensional Tensor. The data type is "
             "float32 or float64.");
    AddOutput("Out",
              "The output is a multi-dimensional Tensor which has same "
              "dimension and data type as the ``x``.");
468
    AddAttr<float>("alpha", "The alpha value of ELU").SetDefault(1.0f);
469
    AddComment(R"DOC(
K
kexinzhao 已提交
470
ELU Activation Operator.
K
Kexin Zhao 已提交
471 472 473 474

Applies the following element-wise computation on the input according to
https://arxiv.org/abs/1511.07289.

475
$$out = \max(0, x) + \min(0, \alpha * (e^x - 1))$$
K
Kexin Zhao 已提交
476 477

)DOC");
478 479 480
  }
};

481 482
class Relu6OpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
483
  void Make() override {
Z
zhupengyang 已提交
484 485 486 487 488 489 490 491
    AddInput("X",
             "Input of relu6 operator, an N-D Tensor, "
             "with data type float32, float64.");
    AddOutput(
        "Out",
        "Output of relu6 operator, a Tensor with the same shape as input.");
    AddAttr<float>("threshold",
                   "The threshold value of Relu6. Default is 6.0. ")
492
        .SetDefault(6.0f);
K
Kexin Zhao 已提交
493
    AddComment(R"DOC(
K
kexinzhao 已提交
494
Relu6 Activation Operator.
K
Kexin Zhao 已提交
495

496
$$out = \min(\max(0, x), threshold)$$
K
Kexin Zhao 已提交
497 498

)DOC");
499 500 501
  }
};

502 503
class PowOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
504
  void Make() override {
505
    AddInput("X", "Input of Pow operator");
506 507 508 509 510
    AddInput("FactorTensor",
             "(Tensor<float>, optional). If provided, pow will use this"
             "The shape of FactorTensor MUST BE [1]."
             "it has higher priority than attr(factor).")
        .AsDispensable();
F
fengjiayi 已提交
511
    AddOutput("Out", "Output of Pow operator");
512
    AddAttr<float>("factor", "The exponential factor of Pow").SetDefault(1.0f);
K
Kexin Zhao 已提交
513
    AddComment(R"DOC(
K
kexinzhao 已提交
514
Pow Activation Operator.
K
Kexin Zhao 已提交
515

516
$$out = x^{factor}$$
K
Kexin Zhao 已提交
517 518

)DOC");
519 520 521 522 523
  }
};

class STanhOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
524
  void Make() override {
525 526 527 528 529 530
    AddInput("X",
             "Input of STanh operator."
             " A LoDTensor or Tensor with type float32, float64.");
    AddOutput("Out", "Output of STanh operator. A Tensor with type float32.");
    AddAttr<float>("scale_a", "The scale parameter of a for the input. ")
        .SetDefault(0.67f);
531 532
    AddAttr<float>("scale_b", "The scale parameter of b for the input")
        .SetDefault(1.7159f);
K
Kexin Zhao 已提交
533
    AddComment(R"DOC(
K
kexinzhao 已提交
534
STanh Activation Operator.
K
Kexin Zhao 已提交
535

Y
Yan Chunwei 已提交
536
$$out = b * \\frac{e^{a * x} - e^{-a * x}}{e^{a * x} + e^{-a * x}}$$
K
Kexin Zhao 已提交
537 538

)DOC");
Q
qijun 已提交
539 540 541
  }
};

542 543
class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
544
  void Make() override {
545
    AddInput("X", "Input of ThresholdedRelu operator");
F
fengjiayi 已提交
546
    AddOutput("Out", "Output of ThresholdedRelu operator");
Y
yuyang18 已提交
547 548
    AddAttr<float>("threshold",
                   "The threshold location of activation. [default 1.0].")
549
        .SetDefault(1.0f);
K
Kexin Zhao 已提交
550
    AddComment(R"DOC(
Y
yuyang18 已提交
551
:strong:`ThresholdedRelu activation operator`
K
Kexin Zhao 已提交
552

Y
yuyang18 已提交
553
..  math::
K
Kexin Zhao 已提交
554

Y
yuyang18 已提交
555
    out = \begin{cases}
Y
yuyang18 已提交
556
             x,  \text{if } x > threshold \\
Y
yuyang18 已提交
557 558
             0,  \text{otherwise}
          \end{cases}
K
Kexin Zhao 已提交
559
)DOC");
560 561 562
  }
};

563 564
class HardSigmoidOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
565
  void Make() override {
566 567 568 569 570
    AddInput("X", "An N-D Tensor with data type float32, float64. ");
    AddOutput("Out", "A Tensor with the same shape as input. ");
    AddAttr<float>("slope",
                   "The slope of the linear approximation of sigmoid. Its "
                   "value MUST BE positive. Default is 0.2. ")
571
        .SetDefault(0.2f);
572 573 574
    AddAttr<float>(
        "offset",
        "The offset of the linear approximation of sigmoid. Default is 0.5. ")
575
        .SetDefault(0.5f);
576
    AddComment(R"DOC(
K
kexinzhao 已提交
577
HardSigmoid Activation Operator.
578

579
A 3-part piecewise linear approximation of sigmoid(https://arxiv.org/abs/1603.00391),
K
Kexin Zhao 已提交
580
which is much faster than sigmoid.
581

582
$$out = \max(0, \min(1, slope * x + offset))$$
583

K
Kexin Zhao 已提交
584
)DOC");
585 586 587
  }
};

A
Abhinav Arora 已提交
588 589
class SwishOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
590
  void Make() override {
A
Abhinav Arora 已提交
591
    AddInput("X", "Input of Swish operator");
F
fengjiayi 已提交
592
    AddOutput("Out", "Output of Swish operator");
A
Abhinav Arora 已提交
593
    AddAttr<float>("beta", "Constant beta of swish operator").SetDefault(1.0f);
594 595 596
    AddAttr<bool>("use_mkldnn",
                  "(bool, default false) Only used in mkldnn kernel")
        .SetDefault(false);
A
Abhinav Arora 已提交
597 598 599
    AddComment(R"DOC(
Swish Activation Operator.

600
$$out = \\frac{x}{1 + e^{- \beta \ x}}$$
A
Abhinav Arora 已提交
601 602 603 604 605

)DOC");
  }
};

H
huangjun12 已提交
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
class HardSwishOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() override {
    AddInput("X", "Input of HardSwish operator");
    AddOutput("Out", "Output of HardSwish operator");
    AddAttr<float>("threshold", "The threshold parameter of HardSwish operator")
        .SetDefault(6.0f);
    AddAttr<float>("scale", "The scale parameter of HardSwish operator")
        .SetDefault(6.0f);
    AddAttr<float>("offset", "The offset parameter of HardSwish operator")
        .SetDefault(3.0f);
    AddComment(R"DOC(
HardSwish Activation Operator.

The hard version of swish(https://arxiv.org/pdf/1905.02244.pdf).

622
$$out = \frac{x * (min(max(0, x+offset), threshold))}{scale}$$
H
huangjun12 已提交
623 624 625 626 627 628 629 630 631

The threshold and scale should be positive. The offset can be either positive or negative.
The default parameters are set according to the above reference.
It is recommended to use the defaults for this activation.

)DOC");
  }
};

D
dzhwinter 已提交
632 633 634 635 636 637 638
REGISTER_ACTIVATION_OP_MAKER(Sigmoid, SigmoidDoc);
REGISTER_ACTIVATION_OP_MAKER(LogSigmoid, LogSigmoidDoc);
REGISTER_ACTIVATION_OP_MAKER(Exp, ExpDoc);
REGISTER_ACTIVATION_OP_MAKER(Relu, ReluDoc);
REGISTER_ACTIVATION_OP_MAKER(Tanh, TanhDoc);
REGISTER_ACTIVATION_OP_MAKER(TanhShrink, TanhShrinkDoc);
REGISTER_ACTIVATION_OP_MAKER(Sqrt, SqrtDoc);
Z
zhoukunsheng 已提交
639
REGISTER_ACTIVATION_OP_MAKER(Rsqrt, RsqrtDoc);
D
dzhwinter 已提交
640 641 642 643 644 645 646 647
REGISTER_ACTIVATION_OP_MAKER(Abs, AbsDoc);
REGISTER_ACTIVATION_OP_MAKER(Ceil, CeilDoc);
REGISTER_ACTIVATION_OP_MAKER(Floor, FloorDoc);
REGISTER_ACTIVATION_OP_MAKER(Cos, CosDoc);
REGISTER_ACTIVATION_OP_MAKER(Sin, SinDoc);
REGISTER_ACTIVATION_OP_MAKER(Round, RoundDoc);
REGISTER_ACTIVATION_OP_MAKER(Reciprocal, ReciprocalDoc);
REGISTER_ACTIVATION_OP_MAKER(Log, LogDoc);
648
REGISTER_ACTIVATION_OP_MAKER(Log1p, Log1pDoc);
D
dzhwinter 已提交
649 650 651 652
REGISTER_ACTIVATION_OP_MAKER(Square, SquareDoc);
REGISTER_ACTIVATION_OP_MAKER(Softplus, SoftplusDoc);
REGISTER_ACTIVATION_OP_MAKER(Softsign, SoftsignDoc);

653
template <ActBwdOpFwdDeps kDepValue>
654 655 656 657 658
class ActivationOpDoubleGrad : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
659
    if (static_cast<int>(kDepValue) & static_cast<int>(kDepX)) {
660
      if (ctx->HasOutput("DX")) {
661 662 663
        ctx->ShareDim("X", "DX");
        ctx->ShareLoD("X", "DX");
      }
664
      if (ctx->HasOutput("DDOut")) {
665 666 667
        ctx->ShareDim("X", "DDOut");
        ctx->ShareLoD("X", "DDOut");
      }
668
    }
669
    if (static_cast<int>(kDepValue) & static_cast<int>(kDepOut)) {
670
      if (ctx->HasOutput("DOut")) {
671 672 673
        ctx->ShareDim("Out", "DOut");
        ctx->ShareLoD("Out", "DOut");
      }
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
      if (ctx->HasOutput("DDOut")) {
        ctx->ShareDim("Out", "DDOut");
        ctx->ShareLoD("Out", "DDOut");
      }
    }
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return GetKernelType(ctx, *this, "DDX");
  }
};

template <ActBwdOpFwdDeps kDepValue>
class ActivationOpDoubleGrad2 : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    if (static_cast<int>(kDepValue) & static_cast<int>(kDepX)) {
      if (ctx->HasOutput("DDOut")) {
        ctx->ShareDim("X", "DDOut");
        ctx->ShareLoD("X", "DDOut");
      }
    }
    if (static_cast<int>(kDepValue) & static_cast<int>(kDepOut)) {
      if (ctx->HasOutput("DDOut")) {
702 703 704
        ctx->ShareDim("Out", "DDOut");
        ctx->ShareLoD("Out", "DDOut");
      }
705 706 707 708 709 710 711 712 713 714
    }
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return GetKernelType(ctx, *this, "DDX");
  }
};

715 716 717 718
//
// ReluGrad: dx = dy if y >= 0 else 0
// ReluGradGrad: ddy = ddx if y >= 0 else 0
//
H
hong 已提交
719 720
template <typename T>
class ReluDoubleGradMaker : public ::paddle::framework::SingleGradOpMaker<T> {
721
 public:
H
hong 已提交
722
  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;
723 724

 protected:
725
  void Apply(GradOpPtr<T> op) const override {
726 727
    op->SetType("relu_grad_grad");
    // input1: Out
H
hong 已提交
728
    op->SetInput("Out", this->Input("Out"));
Q
qingqing01 已提交
729
    // input2: ddx
H
hong 已提交
730 731
    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
    op->SetAttrMap(this->Attrs());
732
    // output: ddy
H
hong 已提交
733
    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
734 735 736
  }
};

737 738
// leaky_relu Grad: dx=dy if y>=0 else alpha * dy
// leaky_relu GradGrad: ddy=ddx if y>=0 else alpha * ddx
H
hong 已提交
739
template <typename T>
740
class LeakyReluDoubleGradMaker
H
hong 已提交
741
    : public ::paddle::framework::SingleGradOpMaker<T> {
742
 public:
H
hong 已提交
743
  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;
744 745

 protected:
746
  void Apply(GradOpPtr<T> op) const override {
747
    op->SetType("leaky_relu_grad_grad");
Z
Zeng Jinle 已提交
748
    // input1: Out
H
hong 已提交
749
    op->SetInput("Out", this->Input("Out"));
750
    // X@GRAD@GRAD: ddx
H
hong 已提交
751 752
    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
    op->SetAttrMap(this->Attrs());
753
    // Out@GRAD@GRAD: ddy
H
hong 已提交
754
    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
755 756 757
  }
};

D
Double_V 已提交
758 759 760 761 762 763 764 765
// elu grad: dx=dy if y>0 else alpha*dy*x.exp()
// elu gradgrad: ddx=ddy if y>0 else alpha*ddy*x.exp()
template <typename T>
class ELUDoubleGradMaker : public ::paddle::framework::SingleGradOpMaker<T> {
 public:
  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;

 protected:
766
  void Apply(GradOpPtr<T> op) const override {
D
Double_V 已提交
767 768 769 770 771 772 773 774 775 776 777 778 779 780
    op->SetType("elu_grad_grad");

    op->SetInput("X", this->Input("X"));
    op->SetInput("DOut", this->Input(framework::GradVarName("Out")));
    // X@GRAD@GRAD: ddx
    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
    op->SetAttrMap(this->Attrs());

    // Out@GRAD@GRAD: ddy
    op->SetOutput("DX", this->InputGrad("X"));
    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
  }
};

L
lvmengsi 已提交
781 782
// sqrt Grad: dx = 0.5 * dy / y
// sqrt GradGrad: ddy = 0.5 * ddx / y, dy = -1 * dx * ddx
H
hong 已提交
783 784
template <typename T>
class SqrtDoubleGradMaker : public ::paddle::framework::SingleGradOpMaker<T> {
L
lvmengsi 已提交
785
 public:
H
hong 已提交
786
  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;
L
lvmengsi 已提交
787 788

 protected:
789
  void Apply(GradOpPtr<T> op) const override {
L
lvmengsi 已提交
790
    op->SetType("sqrt_grad_grad");
H
hong 已提交
791 792 793 794 795 796
    op->SetInput("Out", this->Input("Out"));
    op->SetInput("DX", this->Output(framework::GradVarName("X")));
    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
    op->SetAttrMap(this->Attrs());
    op->SetOutput("DOut", this->InputGrad("Out"));
    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
L
lvmengsi 已提交
797 798 799
  }
};

800 801
// square Grad: dx=2x*dy
// square GradGrad: ddy=2x*ddx, dx=2dy*ddx
H
hong 已提交
802 803
template <typename T>
class SquareDoubleGradMaker : public ::paddle::framework::SingleGradOpMaker<T> {
804
 public:
H
hong 已提交
805
  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;
806 807

 protected:
808
  void Apply(GradOpPtr<T> op) const override {
809
    op->SetType("square_grad_grad");
H
hong 已提交
810
    op->SetInput("X", this->Input("X"));
811
    // Out@GRAD: dy
H
hong 已提交
812
    op->SetInput("DOut", this->Input(framework::GradVarName("Out")));
813
    // X@GRAD@GRAD: ddx
H
hong 已提交
814
    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
815

H
hong 已提交
816
    op->SetAttrMap(this->Attrs());
817 818

    // X@GRAD: dx
H
hong 已提交
819
    op->SetOutput("DX", this->InputGrad("X"));
820
    // Out@GRAD@GRAD: ddy
H
hong 已提交
821
    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
822 823 824
  }
};

825 826 827
DECLARE_INPLACE_OP_INFERER(ActivationGradOpInplaceInference,
                           {framework::GradVarName("Out"),
                            framework::GradVarName("X")});
828 829
DECLARE_INPLACE_OP_INFERER(ActivationDoubleGradOpInplaceInference,
                           {"DDX", "DDOut"});
830

H
hong 已提交
831 832
template <typename T>
class PowGradOpMaker : public framework::SingleGradOpMaker<T> {
833
 public:
H
hong 已提交
834
  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
835 836

 protected:
837
  void Apply(GradOpPtr<T> op) const override {
838
    op->SetType("pow_grad");
H
hong 已提交
839 840 841 842 843
    op->SetInput("X", this->Input("X"));
    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
    op->SetInput("FactorTensor", this->Input("FactorTensor"));
    op->SetAttrMap(this->Attrs());
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
  }
};
class PowOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    ctx->ShareDim("X", /*->*/ "Out");
    ctx->ShareLoD("X", /*->*/ "Out");
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return GetKernelType(ctx, *this, "X");
  }

  framework::OpKernelType GetKernelTypeForVar(
      const std::string& var_name, const Tensor& tensor,
      const framework::OpKernelType& expected_kernel_type) const override {
    if (var_name == "FactorTensor") {
      return expected_kernel_type;
    }
    return framework::OpKernelType(expected_kernel_type.data_type_,
                                   tensor.place(), tensor.layout());
  }
};

class PowOpGrad : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    auto out_grad_name = framework::GradVarName("Out");
    ctx->ShareDim(out_grad_name, framework::GradVarName("X"));
    ctx->ShareLoD(out_grad_name, framework::GradVarName("X"));
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return GetKernelType(ctx, *this, framework::GradVarName("Out"));
  }

  framework::OpKernelType GetKernelTypeForVar(
      const std::string& var_name, const Tensor& tensor,
      const framework::OpKernelType& expected_kernel_type) const override {
    if (var_name == "FactorTensor") {
      return expected_kernel_type;
    }
    return framework::OpKernelType(expected_kernel_type.data_type_,
                                   tensor.place(), tensor.layout());
  }
};
898
DECLARE_INPLACE_OP_INFERER(ActFwdInplaceInferer, {"X", "Out"});
Q
qijun 已提交
899 900 901 902
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
903
namespace plat = paddle::platform;
904

905 906 907 908
#define REGISTER_ACTIVATION_OP(KERNEL_TYPE, OP_NAME, functor, grad_functor) \
  REGISTER_OPERATOR(                                                        \
      KERNEL_TYPE, ops::ActivationOp, ops::OP_NAME##OpMaker,                \
      ops::ActivationOpInferVarType,                                        \
H
hong 已提交
909 910 911 912
      ops::ActivationGradOpMaker<ops::grad_functor<float>::FwdDeps(),       \
                                 paddle::framework::OpDesc>,                \
      ops::ActivationGradOpMaker<ops::grad_functor<float>::FwdDeps(),       \
                                 paddle::imperative::OpBase>,               \
913
      std::conditional<ops::CanInplaceAct<ops::grad_functor<float>>(),      \
914
                       ops::ActFwdInplaceInferer, void>::type);             \
915 916
  REGISTER_OPERATOR(KERNEL_TYPE##_grad, ops::ActivationOpGrad,              \
                    ops::ActivationGradOpInplaceInference);
917 918 919

#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, op_name, functor,        \
                                       grad_functor)                      \
Q
QI JUN 已提交
920 921 922 923 924 925 926 927 928 929
  REGISTER_OP_CPU_KERNEL(                                                 \
      act_type, ops::ActivationKernel<paddle::platform::CPUDeviceContext, \
                                      ops::functor<float>>,               \
      ops::ActivationKernel<paddle::platform::CPUDeviceContext,           \
                            ops::functor<double>>);                       \
  REGISTER_OP_CPU_KERNEL(                                                 \
      act_type##_grad,                                                    \
      ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,       \
                                ops::grad_functor<float>>,                \
      ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,       \
Y
Yu Yang 已提交
930
                                ops::grad_functor<double>>);
931

932 933
FOR_EACH_ACTIVATION_OP(REGISTER_ACTIVATION_OP);
FOR_EACH_ACTIVATION_OP(REGISTER_ACTIVATION_CPU_KERNEL);
934

935
/* ==========================    relu register  ============================= */
936 937
REGISTER_OPERATOR(
    relu, ops::ActivationOp, ops::ReluOpMaker, ops::ActivationOpInferVarType,
H
hong 已提交
938 939 940 941
    ops::ActivationGradOpMaker<ops::ReluGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::ReluGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
942
    ops::ActFwdInplaceInferer);
943
REGISTER_OPERATOR(relu_grad, ops::ActivationOpGrad,
944
                  ops::ActivationGradOpInplaceInference,
H
hong 已提交
945 946
                  ops::ReluDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::ReluDoubleGradMaker<paddle::imperative::OpBase>);
947 948
REGISTER_OPERATOR(
    relu_grad_grad,
949 950
    ops::ActivationOpDoubleGrad2<ops::ReluGradFunctor<float>::FwdDeps()>,
    ops::ActivationDoubleGradOpInplaceInference);
951 952 953 954 955 956 957 958 959 960 961

REGISTER_ACTIVATION_CPU_KERNEL(relu, Relu, ReluFunctor, ReluGradFunctor);

REGISTER_OP_CPU_KERNEL(
    relu_grad_grad,
    ops::ActivationDoubleGradKernel<plat::CPUDeviceContext,
                                    ops::ReluGradGradFunctor<float>>,
    ops::ActivationDoubleGradKernel<plat::CPUDeviceContext,
                                    ops::ReluGradGradFunctor<double>>,
    ops::ActivationDoubleGradKernel<plat::CPUDeviceContext,
                                    ops::ReluGradGradFunctor<plat::float16>>);
962
/* ========================================================================== */
963

964
/* ======================== leaky relu register  ============================ */
965 966 967
REGISTER_OPERATOR(
    leaky_relu, ops::ActivationOp, ops::LeakyReluOpMaker,
    ops::ActivationOpInferVarType,
H
hong 已提交
968 969 970 971
    ops::ActivationGradOpMaker<ops::LeakyReluGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::LeakyReluGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
972
    ops::ActFwdInplaceInferer);
973
REGISTER_OPERATOR(leaky_relu_grad, ops::ActivationOpGrad,
974
                  ops::ActivationGradOpInplaceInference,
H
hong 已提交
975 976
                  ops::LeakyReluDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::LeakyReluDoubleGradMaker<paddle::imperative::OpBase>);
977 978
REGISTER_OPERATOR(
    leaky_relu_grad_grad,
979 980
    ops::ActivationOpDoubleGrad2<ops::LeakyReluGradFunctor<float>::FwdDeps()>,
    ops::ActivationDoubleGradOpInplaceInference);
981

982 983 984 985 986 987 988 989 990 991
REGISTER_ACTIVATION_CPU_KERNEL(leaky_relu, LeakyRelu, LeakyReluFunctor,
                               LeakyReluGradFunctor);
REGISTER_OP_CPU_KERNEL(
    leaky_relu_grad_grad,
    ops::ActivationDoubleGradKernel<plat::CPUDeviceContext,
                                    ops::LeakyReluGradGradFunctor<float>>,
    ops::ActivationDoubleGradKernel<plat::CPUDeviceContext,
                                    ops::LeakyReluGradGradFunctor<double>>,
    ops::ActivationDoubleGradKernel<
        plat::CPUDeviceContext, ops::LeakyReluGradGradFunctor<plat::float16>>);
992 993
/* ========================================================================== */

D
Double_V 已提交
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
/* ========================    elu  register     ============================ */
REGISTER_OPERATOR(
    elu, ops::ActivationOp, ops::ELUOpMaker, ops::ActivationOpInferVarType,
    ops::ActivationGradOpMaker<ops::ELUGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::ELUGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
    ops::ActFwdInplaceInferer);
REGISTER_OPERATOR(elu_grad, ops::ActivationOpGrad,
                  ops::ActivationGradOpInplaceInference,
                  ops::ELUDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::ELUDoubleGradMaker<paddle::imperative::OpBase>);
REGISTER_OPERATOR(
    elu_grad_grad,
    ops::ActivationOpDoubleGrad<ops::ELUGradFunctor<float>::FwdDeps()>,
    ops::ActivationDoubleGradOpInplaceInference);

REGISTER_ACTIVATION_CPU_KERNEL(elu, ELU, ELUFunctor, ELUGradFunctor);
REGISTER_OP_CPU_KERNEL(
    elu_grad_grad, ops::ELUDoubleGradKernel<plat::CPUDeviceContext,
                                            ops::ELUGradGradFunctor<float>>,
    ops::ELUDoubleGradKernel<plat::CPUDeviceContext,
                             ops::ELUGradGradFunctor<double>>,
    ops::ELUDoubleGradKernel<plat::CPUDeviceContext,
                             ops::ELUGradGradFunctor<plat::float16>>);

/* ========================================================================== */

L
lvmengsi 已提交
1022 1023 1024
/* ===========================   sqrt register  ============================= */
REGISTER_OPERATOR(
    sqrt, ops::ActivationOp, ops::SqrtOpMaker, ops::ActivationOpInferVarType,
H
hong 已提交
1025 1026 1027 1028
    ops::ActivationGradOpMaker<ops::SqrtGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::SqrtGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
1029
    ops::ActFwdInplaceInferer);
L
lvmengsi 已提交
1030
REGISTER_OPERATOR(sqrt_grad, ops::ActivationOpGrad,
1031
                  ops::ActivationGradOpInplaceInference,
H
hong 已提交
1032 1033
                  ops::SqrtDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::SqrtDoubleGradMaker<paddle::imperative::OpBase>);
L
lvmengsi 已提交
1034 1035
REGISTER_OPERATOR(
    sqrt_grad_grad,
1036 1037 1038
    ops::ActivationOpDoubleGrad<ops::SqrtGradGradFunctor<float>::FwdDeps()>,
    ops::ActivationDoubleGradOpInplaceInference);

L
lvmengsi 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
REGISTER_ACTIVATION_CPU_KERNEL(sqrt, Sqrt, SqrtFunctor, SqrtGradFunctor);
REGISTER_OP_CPU_KERNEL(
    sqrt_grad_grad, ops::SqrtDoubleGradKernel<plat::CPUDeviceContext,
                                              ops::SqrtGradGradFunctor<float>>,
    ops::SqrtDoubleGradKernel<plat::CPUDeviceContext,
                              ops::SqrtGradGradFunctor<double>>,
    ops::SqrtDoubleGradKernel<plat::CPUDeviceContext,
                              ops::SqrtGradGradFunctor<plat::float16>>);
/* ========================================================================== */

1049 1050 1051 1052
/* ==========================   square register  ============================ */
REGISTER_OPERATOR(
    square, ops::ActivationOp, ops::SquareOpMaker,
    ops::ActivationOpInferVarType,
H
hong 已提交
1053 1054 1055 1056
    ops::ActivationGradOpMaker<ops::SquareGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::SquareGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
1057
    ops::ActFwdInplaceInferer);
1058
REGISTER_OPERATOR(square_grad, ops::ActivationOpGrad,
1059
                  ops::ActivationGradOpInplaceInference,
H
hong 已提交
1060 1061
                  ops::SquareDoubleGradMaker<paddle::framework::OpDesc>,
                  ops::SquareDoubleGradMaker<paddle::imperative::OpBase>);
1062 1063
REGISTER_OPERATOR(
    square_grad_grad,
1064 1065
    ops::ActivationOpDoubleGrad<ops::SquareGradGradFunctor<float>::FwdDeps()>,
    ops::ActivationDoubleGradOpInplaceInference);
1066

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
REGISTER_OP_CPU_KERNEL(square,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::SquareFunctor<float>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::SquareFunctor<double>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::SquareFunctor<int>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::SquareFunctor<int64_t>>);
REGISTER_OP_CPU_KERNEL(
    square_grad, ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                                           ops::SquareGradFunctor<float>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::SquareGradFunctor<double>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::SquareGradFunctor<int>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::SquareGradFunctor<int64_t>>);
1085 1086 1087 1088 1089 1090 1091 1092

REGISTER_OP_CPU_KERNEL(
    square_grad_grad,
    ops::SquareDoubleGradKernel<plat::CPUDeviceContext,
                                ops::SquareGradGradFunctor<float>>,
    ops::SquareDoubleGradKernel<plat::CPUDeviceContext,
                                ops::SquareGradGradFunctor<double>>,
    ops::SquareDoubleGradKernel<plat::CPUDeviceContext,
1093 1094 1095 1096 1097
                                ops::SquareGradGradFunctor<plat::float16>>,
    ops::SquareDoubleGradKernel<plat::CPUDeviceContext,
                                ops::SquareGradGradFunctor<int>>,
    ops::SquareDoubleGradKernel<plat::CPUDeviceContext,
                                ops::SquareGradGradFunctor<int64_t>>);
1098
/* ========================================================================== */
1099 1100 1101 1102 1103

/* ==========================   pow register  ============================ */

REGISTER_OPERATOR(
    pow, ops::PowOp, ops::PowOpMaker, ops::ActivationOpInferVarType,
H
hong 已提交
1104 1105
    ops::PowGradOpMaker<paddle::framework::OpDesc>,
    ops::PowGradOpMaker<paddle::imperative::OpBase>,
1106
    std::conditional<ops::CanInplaceAct<ops::PowGradFunctor<float>>(),
1107
                     ops::ActFwdInplaceInferer, void>::type);
1108 1109 1110 1111 1112
REGISTER_OPERATOR(pow_grad, ops::PowOpGrad,
                  ops::ActivationGradOpInplaceInference);

REGISTER_OP_CPU_KERNEL(
    pow, ops::PowKernel<plat::CPUDeviceContext, ops::PowFunctor<float>>,
1113 1114 1115
    ops::PowKernel<plat::CPUDeviceContext, ops::PowFunctor<double>>,
    ops::PowKernel<plat::CPUDeviceContext, ops::PowFunctor<int>>,
    ops::PowKernel<plat::CPUDeviceContext, ops::PowFunctor<int64_t>>);
1116 1117 1118
REGISTER_OP_CPU_KERNEL(
    pow_grad,
    ops::PowGradKernel<plat::CPUDeviceContext, ops::PowGradFunctor<float>>,
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
    ops::PowGradKernel<plat::CPUDeviceContext, ops::PowGradFunctor<double>>,
    ops::PowGradKernel<plat::CPUDeviceContext, ops::PowGradFunctor<int>>,
    ops::PowGradKernel<plat::CPUDeviceContext, ops::PowGradFunctor<int64_t>>);
/* ========================================================================== */

/* ==========================   exp register  ============================ */
REGISTER_OPERATOR(
    exp, ops::ActivationOp, ops::ExpOpMaker, ops::ActivationOpInferVarType,
    ops::ActivationGradOpMaker<ops::ExpGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::ExpGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
    std::conditional<ops::CanInplaceAct<ops::ExpGradFunctor<float>>(),
                     ops::ActFwdInplaceInferer, void>::type);
REGISTER_OPERATOR(exp_grad, ops::ActivationOpGrad,
                  ops::ActivationGradOpInplaceInference);

REGISTER_OP_CPU_KERNEL(exp,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::ExpFunctor<float>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::ExpFunctor<double>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::ExpFunctor<int>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::ExpFunctor<int64_t>>);
REGISTER_OP_CPU_KERNEL(
    exp_grad, ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                                        ops::ExpGradFunctor<float>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::ExpGradFunctor<double>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::ExpGradFunctor<int>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::ExpGradFunctor<int64_t>>);
/* ========================================================================== */

/* ==========================   abs register  ============================ */
REGISTER_OPERATOR(
    abs, ops::ActivationOp, ops::AbsOpMaker, ops::ActivationOpInferVarType,
    ops::ActivationGradOpMaker<ops::AbsGradFunctor<float>::FwdDeps(),
                               paddle::framework::OpDesc>,
    ops::ActivationGradOpMaker<ops::AbsGradFunctor<float>::FwdDeps(),
                               paddle::imperative::OpBase>,
    std::conditional<ops::CanInplaceAct<ops::AbsGradFunctor<float>>(),
                     ops::ActFwdInplaceInferer, void>::type);
REGISTER_OPERATOR(abs_grad, ops::ActivationOpGrad,
                  ops::ActivationGradOpInplaceInference);

REGISTER_OP_CPU_KERNEL(abs,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::AbsFunctor<float>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::AbsFunctor<double>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::AbsFunctor<int>>,
                       ops::ActivationKernel<paddle::platform::CPUDeviceContext,
                                             ops::AbsFunctor<int64_t>>);
REGISTER_OP_CPU_KERNEL(
    abs_grad, ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                                        ops::AbsGradFunctor<float>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::AbsGradFunctor<double>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::AbsGradFunctor<int>>,
    ops::ActivationGradKernel<paddle::platform::CPUDeviceContext,
                              ops::AbsGradFunctor<int64_t>>);
1186
/* ========================================================================== */