【PaddlePaddle Hackathon 2】16 新增 API RRelu (#41823)

* rrelu逻辑部分

* unregistered op kernel (unresolved)

* commit before merge

* 丰富测试用例

* 修复rrelu-sig的bug

* 修复cpu环境测试

* 修改拼写错误

* 修改code format

* 尝试优化测试用例timeout的问题

* 优化测试用例

* 移除seed, 优化随机函数

* update en doc for rrelu

* fix rrelu en docs, test=document_fix

* add paper link for en docs, test=document_fix

* udpate en doc

* add r,test=document_fix

paddle/fluid/operators/rrelu_op.cc

+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <memory>
+#include <string>
+#include "paddle/fluid/framework/infershape_utils.h"
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/phi/infermeta/unary.h"
+
+namespace paddle {
+namespace operators {
+
+using framework::Tensor;
+
+class RReluOp : public framework::OperatorWithKernel {
+ public:
+  using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+  framework::OpKernelType GetExpectedKernelType(
+      const framework::ExecutionContext& ctx) const override {
+    return framework::OpKernelType(
+        OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
+  }
+};
+
+class RReluOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  void Make() override {
+    AddInput("X", "The input of RReLU op.");
+    AddOutput("Out", "The output of RReLU op.");
+    AddOutput("Noise", "The random sampled RReLU noise.")
+        .AsIntermediate()
+        .AsExtra();
+    AddAttr<bool>("is_test",
+                  "(bool, default false) Set to true for inference only, false "
+                  "for training. Some layers may run faster when this is true.")
+        .SetDefault(false);
+    float default_lower = 1. / 8.;
+    AddAttr<float>("lower", "Lower bound of the uniform distribution.")
+        .SetDefault(default_lower)
+        .AddCustomChecker([](const float& lower) {
+          PADDLE_ENFORCE_EQ(lower >= 0.0f && lower < 1.0f, true,
+                            platform::errors::InvalidArgument(
+                                "'RRelu_lower' must be between 0.0 and 1.0."));
+        });
+    float defalut_upper = 1. / 3.;
+    AddAttr<float>("upper", "Upper bound of the uniform distribution.")
+        .SetDefault(defalut_upper)
+        .AddCustomChecker([](const float& upper) {
+          PADDLE_ENFORCE_EQ(upper > 0.0f && upper <= 1.0f, true,
+                            platform::errors::InvalidArgument(
+                                "'RRelu_upper' must be between 0.0 and 1.0."));
+        });
+    AddComment(R"DOC(
+RReLU Operator.
+
+Applies the randomized leaky rectified liner unit function, element-wise,
+as described in the paper:
+
+`Empirical Evaluation of Rectified Activations in Convolutional Network`_.
+
+The function is defined as:
+
+.. math::
+    \text{RReLU}(x) =
+    \begin{cases}
+        x & \text{if } x \geq 0 \\
+        ax & \text{ otherwise }
+    \end{cases}
+
+where :math:`a` is randomly sampled from uniform distribution
+:math:`\mathcal{U}(\text{lower}, \text{upper})`.
+
+ See: https://arxiv.org/pdf/1505.00853.pdf
+
+)DOC");
+  }
+};
+
+class RReluGradOp : public framework::OperatorWithKernel {
+ public:
+  using framework::OperatorWithKernel::OperatorWithKernel;
+};
+
+template <typename T>
+class RReluGradOpMaker : public framework::SingleGradOpMaker<T> {
+ public:
+  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
+
+ protected:
+  void Apply(GradOpPtr<T> op) const override {
+    op->SetType("rrelu_grad");
+    op->SetInput("X", this->Input("X"));
+    op->SetInput("Noise", this->Output("Noise"));
+    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
+    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle
+
+namespace ops = paddle::operators;
+DECLARE_INFER_SHAPE_FUNCTOR(rrelu, RReluInferShapeFunctor,
+                            PD_INFER_META(phi::RReluInferMeta));
+
+REGISTER_OPERATOR(rrelu, ops::RReluOp, ops::RReluOpMaker,
+                  ops::RReluGradOpMaker<paddle::framework::OpDesc>,
+                  ops::RReluGradOpMaker<paddle::imperative::OpBase>,
+                  RReluInferShapeFunctor);
+
+DECLARE_INFER_SHAPE_FUNCTOR(rrelu_grad, RReluGradInferShapeFunctor,
+                            PD_INFER_META(phi::RReluGradInferMeta));
+REGISTER_OPERATOR(rrelu_grad, ops::RReluGradOp, RReluGradInferShapeFunctor);

paddle/phi/infermeta/unary.cc

@@ -1977,6 +1977,55 @@ void RollInferMeta(const MetaTensor& x,
   out->set_dtype(x.dtype());
 }
 
+void RReluInferMeta(const MetaTensor& x,
+                    float lower,
+                    float upper,
+                    bool is_test,
+                    MetaTensor* out,
+                    MetaTensor* noise) {
+  auto x_dims = x.dims();
+  PADDLE_ENFORCE_GE(lower,
+                    0,
+                    phi::errors::InvalidArgument(
+                        "The lower value should be greater than or equal to 0. "
+                        "But received lower value = %f.",
+                        lower));
+  PADDLE_ENFORCE_LE(upper,
+                    1,
+                    phi::errors::InvalidArgument(
+                        "The upper value should be less than or equal to 1. "
+                        "But received upper value = %f.",
+                        upper));
+  PADDLE_ENFORCE_GE(
+      upper,
+      lower,
+      phi::errors::InvalidArgument(
+          "The upper value should be greater than or equal to lower value "
+          "But received upper value = %f, lower value = %f.",
+          upper,
+          lower));
+
+  out->set_dims(x_dims);
+  out->set_dtype(x.dtype());
+  out->set_layout(x.layout());
+  out->share_lod(x);
+
+  if (noise != nullptr) {
+    noise->set_dims(x_dims);
+    noise->set_dtype(x.dtype());
+    noise->set_layout(x.layout());
+  }
+}
+
+void RReluGradInferMeta(const MetaTensor& out_grad,
+                        const MetaTensor& noise,
+                        MetaTensor* x_grad) {
+  auto do_dims = out_grad.dims();
+  x_grad->set_dims(do_dims);
+  x_grad->set_dtype(out_grad.dtype());
+  x_grad->share_lod(out_grad);
+}
+
 void SetValueInferMeta(const MetaTensor& x, MetaTensor* out) {
   auto in_dims = x.dims();
   PADDLE_ENFORCE_LT(

paddle/phi/infermeta/unary.h

@@ -281,6 +281,17 @@ void RollInferMeta(const MetaTensor& x,
                    const std::vector<int64_t>& axis,
                    MetaTensor* out);
 
+void RReluInferMeta(const MetaTensor& x,
+                    float lower,
+                    float upper,
+                    bool is_test,
+                    MetaTensor* out,
+                    MetaTensor* noise);
+
+void RReluGradInferMeta(const MetaTensor& out_grad,
+                        const MetaTensor& noise,
+                        MetaTensor* x_grad);
+
 void SetValueInferMeta(const MetaTensor& x, MetaTensor* out);
 
 void ShapeInferMeta(const MetaTensor& input, MetaTensor* out);

paddle/phi/kernels/cpu/rrelu_grad_kernel.cc

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/rrelu_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RReluGradKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const DenseTensor& noise,
+                     const DenseTensor& out_grad,
+                     DenseTensor* x_grad) {
+  const T* n_ptr = noise.data<T>();
+  const T* x_ptr = x.data<T>();
+  const T* out_grad_ptr = out_grad.data<T>();
+  int numel = x.numel();
+  if (!x_grad) return;
+
+  int i = 0;
+  T* x_grad_ptr = dev_ctx.template Alloc<T>(x_grad);
+  for (i = 0; i < numel; i++) {
+    x_grad_ptr[i] = x_ptr[i] > 0 ? out_grad_ptr[i] : n_ptr[i] * out_grad_ptr[i];
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    rrelu_grad, CPU, ALL_LAYOUT, phi::RReluGradKernel, float, double) {}

paddle/phi/kernels/cpu/rrelu_kernel.cc

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/rrelu_kernel.h"
+
+#include "paddle/fluid/framework/generator.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RReluKernel(const Context& dev_ctx,
+                 const DenseTensor& x,
+                 const float lower,
+                 const float upper,
+                 bool is_test,
+                 DenseTensor* out,
+                 DenseTensor* noise) {
+  const T* x_ptr = x.data<T>();
+  T* o_ptr = dev_ctx.template Alloc<T>(out);
+  T* n_ptr = dev_ctx.template Alloc<T>(noise);
+  T zero = static_cast<T>(0);
+  int numel = x.numel();
+  int i = 0;
+
+  if (is_test) {
+    T mid_val = static_cast<T>((lower + upper) / 2.0);
+    for (i = 0; i < numel; i++) {
+      if (x_ptr[i] < zero) {
+        o_ptr[i] = mid_val * x_ptr[i];
+        n_ptr[i] = mid_val;
+      } else {
+        o_ptr[i] = x_ptr[i];
+        n_ptr[i] = 1.0;
+      }
+    }
+
+    return;
+  }
+
+  auto engine = paddle::framework::GetCPURandomEngine(0);
+
+  std::uniform_real_distribution<float> dist(lower, upper);
+
+  for (i = 0; i < numel; i++) {
+    if (x_ptr[i] < zero) {
+      T scale = static_cast<T>(dist(*engine));
+      o_ptr[i] = scale * x_ptr[i];
+      n_ptr[i] = scale;
+    } else {
+      o_ptr[i] = x_ptr[i];
+      n_ptr[i] = 1.0;
+    }
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(rrelu,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::RReluKernel,
+                   float,
+                   phi::dtype::float16,
+                   double) {}

paddle/phi/kernels/gpu/rrelu_grad_kernel.cu

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/rrelu_grad_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/funcs/reduce_function.h"
+#include "paddle/phi/kernels/gpu/prelu_funcs.h"
+#include "paddle/phi/kernels/primitive/functor_primitives.h"
+
+namespace phi {
+
+template <typename T>
+__global__ void RReluOpGradKernel(const T* x_ptr,
+                                  const T* noise_ptr,
+                                  const T* out_grad_ptr,
+                                  T* x_grad_ptr,
+                                  int numel) {
+  CUDA_KERNEL_LOOP(index, numel) {
+    T scale = noise_ptr[index];
+    T x = x_ptr[index];
+    T out_grad = out_grad_ptr[index];
+    T zero = static_cast<T>(0);
+    x_grad_ptr[index] = (x < zero) ? scale * out_grad : out_grad;
+  }
+}
+
+template <typename T>
+class RReluOpGradFunctor {
+ public:
+  void operator()(gpuStream_t stream,
+                  const T* x,
+                  const T* noise,
+                  const T* out_grad,
+                  T* x_grad,
+                  int numel) {
+    RReluOpGradKernel<
+        T><<<PADDLE_GET_BLOCKS(numel), CUDA_NUM_THREADS, 0, stream>>>(
+        x, noise, out_grad, x_grad, numel);
+  }
+};
+
+template <typename T, typename Context>
+void RReluGradKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const DenseTensor& noise,
+                     const DenseTensor& out_grad,
+                     DenseTensor* x_grad) {
+  if (!x_grad) return;
+  dev_ctx.template Alloc<T>(x_grad);
+
+  const T* x_ptr = x.data<T>();
+  const T* n_ptr = noise.data<T>();
+  const T* out_grad_ptr = out_grad.data<T>();
+  T* x_grad_ptr = dev_ctx.template Alloc<T>(x_grad);
+
+  int numel = x.numel();
+  auto stream = dev_ctx.stream();
+
+  RReluOpGradFunctor<T> rrelu_grad;
+  rrelu_grad(stream, x_ptr, n_ptr, out_grad_ptr, x_grad_ptr, numel);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(rrelu_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::RReluGradKernel,
+                   float,
+                   phi::dtype::float16,
+                   double) {}

paddle/phi/kernels/gpu/rrelu_kernel.cu

+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/distribution_helper.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/rrelu_kernel.h"
+
+namespace phi {
+
+template <typename T>
+struct RReluTrainCudaFunctor {
+ public:
+  RReluTrainCudaFunctor(const T* in, T* out, T* noise)
+      : in_(in), out_(out), noise_(noise) {
+    zero_ = static_cast<T>(0);
+  }
+
+  __device__ void operator()(int64_t idx) {
+    T x = in_[idx];
+    if (x < zero_) {
+      out_[idx] = noise_[idx] * x;
+    } else {
+      out_[idx] = x;
+      noise_[idx] = 1.0;
+    }
+  }
+
+ private:
+  const T* in_;
+  T* out_;
+  T* noise_;
+  T zero_;
+};
+
+template <typename T>
+struct RReluTestCudaFunctor {
+ public:
+  RReluTestCudaFunctor(const T* in, T* out, T* noise, T mid_val)
+      : in_(in), out_(out), noise_(noise), mid_val_(mid_val) {
+    zero_ = static_cast<T>(0);
+  }
+
+  __device__ void operator()(int64_t idx) {
+    T x = in_[idx];
+    if (x < zero_) {
+      out_[idx] = mid_val_ * x;
+      noise_[idx] = mid_val_;
+    } else {
+      out_[idx] = x;
+      noise_[idx] = 1.0;
+    }
+  }
+
+ private:
+  const T* in_;
+  T* out_;
+  T* noise_;
+  T zero_;
+  T mid_val_;
+};
+
+template <typename T, typename Context>
+void RReluKernel(const Context& ctx,
+                 const DenseTensor& x,
+                 const float lower,
+                 const float upper,
+                 bool is_test,
+                 DenseTensor* out,
+                 DenseTensor* noise) {
+  const T* x_data = x.data<T>();
+  T* out_data = ctx.template Alloc<T>(out);
+  T* noise_data = ctx.template Alloc<T>(noise);
+  auto size = x.numel();
+  if (size <= 0) return;
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  if (is_test) {
+    T mid_val = static_cast<T>((lower + upper) / 2.0);
+    RReluTestCudaFunctor<T> functor(x_data, out_data, noise_data, mid_val);
+    for_range(functor);
+  } else {
+    using MT = typename kps::details::MPTypeTrait<T>::Type;
+    funcs::uniform_distribution<MT> dist;
+    funcs::uniform_real_transform<MT> trans(lower, upper);
+    funcs::distribution_and_transform<T>(ctx, noise, dist, trans);
+    RReluTrainCudaFunctor<T> functor(x_data, out_data, noise_data);
+    for_range(functor);
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(rrelu,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::RReluKernel,
+                   float,
+                   phi::dtype::float16,
+                   double) {}

paddle/phi/kernels/rrelu_grad_kernel.h

+
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RReluGradKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const DenseTensor& noise,
+                     const DenseTensor& out_grad,
+                     DenseTensor* x_grad);
+}  // namespace phi

paddle/phi/kernels/rrelu_kernel.h

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RReluKernel(const Context& dev_ctx,
+                 const DenseTensor& x,
+                 const float lower,
+                 const float upper,
+                 bool is_test,
+                 DenseTensor* out,
+                 DenseTensor* noise);
+}  // namespace phi

paddle/phi/ops/compat/rrelu_sig.cc

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature RReluOpArgumentMapping(const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "rrelu", {"X"}, {"lower", "upper", "is_test"}, {"Out", "Noise"});
+}
+
+KernelSignature RReluGradGradOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "rrelu_grad", {"X", "Noise", "Out@GRAD"}, {}, {"X@GRAD"});
+}
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(rrelu, phi::RReluOpArgumentMapping);
+PD_REGISTER_ARG_MAPPING_FN(rrelu_grad, phi::RReluGradGradOpArgumentMapping);

python/paddle/fluid/tests/unittests/test_rrelu_op.py

+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import unittest
+import numpy as np
+import paddle.fluid as fluid
+import paddle.fluid.core as core
+from op_test import OpTest
+import paddle
+import paddle.nn.functional as F
+from paddle.fluid import dygraph
+
+paddle.seed(102)
+np.random.seed(102)
+
+
+def ref_rrelu(x, lower, upper):
+    x_t = x.copy()
+    alpha = (lower + upper) / 2.0
+    return np.where(x_t <= 0, alpha * x_t, x_t)
+
+
+def ref_rrelu_nn(x, lower, upper):
+    return ref_rrelu(x, lower, upper)
+
+
+def check_output(input, output, lower, upper):
+    lower_res = np.where(input <= 0, lower * input, input)
+    upper_res = np.where(input <= 0, upper * input, input)
+    return (output <= lower_res).all() and (output >= upper_res).all()
+
+
+class TestFunctionalRReluAPI(unittest.TestCase):
+    def setUp(self):
+        self.x_np = np.random.uniform(-1., 1., [1, 2, 3, 4]).astype('float64')
+        self.lower_0 = 0.05
+        self.lower_1 = 0.1
+        self.upper_0 = 0.25
+        self.upper_1 = 0.33
+
+        self.places = [
+            fluid.CUDAPlace(0)
+            if core.is_compiled_with_cuda() else fluid.CPUPlace()
+        ]
+
+    def check_static_result(self, place):
+        with fluid.program_guard(fluid.Program(), fluid.Program()):
+            input = fluid.data(
+                name="input", shape=[2, 3, 4, 5], dtype="float32")
+            res1 = F.rrelu(
+                x=input, lower=self.lower_0, upper=self.upper_0, training=False)
+            res2 = F.rrelu(
+                x=input, lower=self.lower_1, upper=self.upper_1, training=False)
+            in_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype("float32")
+
+            res_np1 = ref_rrelu(in_np, self.lower_0, self.upper_0)
+            exe = fluid.Executor(place)
+            fetches = exe.run(fluid.default_main_program(),
+                              feed={"input": in_np},
+                              fetch_list=[res1])
+
+            self.assertTrue(np.allclose(fetches[0], res_np1))
+
+            res_np2 = ref_rrelu(in_np, self.lower_1, self.upper_1)
+            fetches = exe.run(fluid.default_main_program(),
+                              feed={"input": in_np},
+                              fetch_list=[res2])
+            self.assertTrue(np.allclose(fetches[0], res_np2))
+
+    def test_static(self):
+        for place in self.places:
+            self.check_static_result(place=place)
+
+    def test_static_graph_functional(self):
+        '''test_static_graph_functional'''
+
+        for place in self.places:
+            paddle.enable_static()
+            x_1 = paddle.fluid.data(
+                name="x", shape=self.x_np.shape, dtype="float64")
+            x_2 = paddle.fluid.data(
+                name="x2", shape=self.x_np.shape, dtype="float64")
+            out_1 = F.rrelu(x_1, self.lower_0, self.upper_0, training=False)
+            out_2 = F.rrelu(x_2, self.lower_1, self.upper_1, training=False)
+            out_3 = F.rrelu(x_2, self.lower_1, self.upper_1, training=True)
+
+            exe = paddle.static.Executor(place=place)
+            res_1 = exe.run(fluid.default_main_program(),
+                            feed={"x": self.x_np},
+                            fetch_list=out_1,
+                            use_prune=True)
+            res_2 = exe.run(fluid.default_main_program(),
+                            feed={"x2": self.x_np},
+                            fetch_list=out_2,
+                            use_prune=True)
+            res_3 = exe.run(fluid.default_main_program(),
+                            feed={"x2": self.x_np},
+                            fetch_list=out_3,
+                            use_prune=True)
+
+            out_ref_1 = ref_rrelu(self.x_np, self.lower_0, self.upper_0)
+            out_ref_2 = ref_rrelu(self.x_np, self.lower_1, self.upper_1)
+            self.assertEqual(np.allclose(out_ref_1, res_1), True)
+            self.assertEqual(np.allclose(out_ref_2, res_2), True)
+            self.assertTrue(
+                check_output(self.x_np, res_3[0], self.lower_1, self.upper_1))
+
+    def test_static_graph_layer(self):
+        '''test_static_graph_layer'''
+
+        for place in self.places:
+            paddle.enable_static()
+            x_1 = paddle.fluid.data(
+                name="x", shape=self.x_np.shape, dtype="float64")
+            x_2 = paddle.fluid.data(
+                name="x2", shape=self.x_np.shape, dtype="float64")
+            # init instance
+            rrelu_1 = paddle.nn.RReLU(self.lower_0, self.upper_0)
+            rrelu_2 = paddle.nn.RReLU(self.lower_1, self.upper_1)
+            out_1 = rrelu_1(x_1)
+            out_2 = rrelu_2(x_2)
+
+            exe = paddle.static.Executor(place=place)
+            res_1 = exe.run(fluid.default_main_program(),
+                            feed={"x": self.x_np},
+                            fetch_list=out_1,
+                            use_prune=True)
+            res_2 = exe.run(fluid.default_main_program(),
+                            feed={"x2": self.x_np},
+                            fetch_list=out_2,
+                            use_prune=True)
+
+            self.assertTrue(
+                check_output(self.x_np, res_1[0], self.lower_0, self.upper_0))
+            self.assertTrue(
+                check_output(self.x_np, res_2[0], self.lower_1, self.upper_1))
+
+    def dygraph_check(self, lower, upper):
+        for place in self.places:
+            paddle.disable_static(place)
+            x = paddle.to_tensor(self.x_np)
+            out = F.rrelu(x, lower, upper, training=False)
+            out_ref = ref_rrelu(self.x_np, lower, upper)
+            self.assertEqual(np.allclose(out_ref, out), True)
+            paddle.enable_static()
+
+    def test_dygraph_functional(self):
+        '''test_dygraph_functional'''
+
+        self.dygraph_check(self.lower_0, self.upper_0)
+        self.dygraph_check(self.lower_1, self.upper_1)
+
+    def test_dygraph_layer(self):
+        '''test_dygraph_layer'''
+
+        for place in self.places:
+            paddle.disable_static(place=place)
+            rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0)
+            result = rrelu(paddle.to_tensor(self.x_np))
+            self.assertTrue(
+                check_output(self.x_np,
+                             result.numpy(), self.lower_0, self.upper_0))
+            paddle.enable_static()
+
+    def test_dygraph(self):
+        for place in self.places:
+            paddle.disable_static(place=place)
+            with dygraph.guard():
+                rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0)
+                out_np = rrelu(paddle.to_tensor(self.x_np))
+            self.assertTrue(
+                check_output(self.x_np,
+                             out_np.numpy(), self.lower_0, self.upper_0))
+            paddle.enable_static()
+
+    def test_error_functional(self):
+        paddle.enable_static()
+        with paddle.static.program_guard(paddle.static.Program()):
+            # The input type must be Variable.
+            self.assertRaises(
+                TypeError, F.rrelu, x=1, lower=self.lower_0, upper=self.upper_0)
+            # The input dtype must be float16, float32, float64.
+            x_int32 = paddle.fluid.data(
+                name='x_int32', shape=[2, 3], dtype='int32')
+            self.assertRaises(
+                TypeError,
+                F.rrelu,
+                x=x_int32,
+                lower=self.lower_0,
+                upper=self.upper_0)
+            x_bool = paddle.fluid.data(
+                name='x_bool', shape=[2, 3], dtype='int32')
+            self.assertRaises(
+                TypeError,
+                F.rrelu,
+                x=x_bool,
+                lower=self.lower_0,
+                upper=self.upper_0)
+            # lower and upper must be float
+            x_fp32 = paddle.fluid.data(
+                name='x_fp32', shape=[2, 3], dtype='float32')
+            self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0, upper=0.5)
+            self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0.5, upper=1)
+            # lower and upper must be in (0, 1)
+            self.assertRaises(
+                ValueError, F.rrelu, x=x_fp32, lower=-1., upper=0.5)
+            self.assertRaises(
+                ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=2.)
+            # upper should not be less than lower
+            self.assertRaises(
+                ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=0.2)
+            # support the input dtype is float16
+            x_fp16 = paddle.fluid.data(
+                name='x_fp16', shape=[2, 3], dtype='float16')
+            F.rrelu(x=x_fp16, lower=self.lower_0, upper=self.upper_0)
+
+    def test_error_layer(self):
+        def error_int_dtype():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float64")
+                rrelu = paddle.nn.RReLU(2, 3)
+                rrelu(paddle.to_tensor(x))
+
+        def error_lower_dtype():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float32")
+                rrelu = paddle.nn.RReLU(0, 0.5)
+                rrelu(paddle.to_tensor(x))
+
+        def error_upper_dtype():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float32")
+                rrelu = paddle.nn.RReLU(0.5, 1)
+                rrelu(paddle.to_tensor(x))
+
+        def error_lower_range():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float32")
+                rrelu = paddle.nn.RReLU(-1.0, 0.5)
+                rrelu(paddle.to_tensor(x))
+
+        def error_upper_range():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float32")
+                rrelu = paddle.nn.RReLU(0.5, 2.0)
+                rrelu(paddle.to_tensor(x))
+
+        def error_lower_upper():
+            with paddle.fluid.dygraph.guard():
+                x = np.random.random([2, 3]).astype("float32")
+                rrelu = paddle.nn.RReLU(0.5, 0.2)
+                rrelu(paddle.to_tensor(x))
+
+        self.assertRaises(TypeError, error_int_dtype)
+        self.assertRaises(TypeError, error_lower_dtype)
+        self.assertRaises(TypeError, error_upper_dtype)
+        self.assertRaises(ValueError, error_lower_range)
+        self.assertRaises(ValueError, error_upper_range)
+        self.assertRaises(ValueError, error_lower_upper)
+
+
+class RReluTest(OpTest):
+    def setUp(self):
+        self.op_type = "rrelu"
+        self.lower = 0.1
+        self.upper = 0.3
+        self.is_test = True
+        self.init_prams()
+
+    def init_prams(self):
+        self.dtype = "float64"
+        self.x_shape = [2, 3, 4, 5]
+
+        x_np = np.random.uniform(-1, 1, self.x_shape).astype(self.dtype)
+        out_np = ref_rrelu(x_np, self.lower, self.upper)
+        noise_np = np.ones(self.x_shape).astype(self.dtype)
+        noise_np[x_np < 0] = (self.lower + self.upper) / 2.0
+
+        self.inputs = {'X': x_np}
+        self.outputs = {'Out': out_np, 'Noise': noise_np}
+        self.attrs = {
+            'lower': self.lower,
+            "upper": self.upper,
+            "is_test": self.is_test
+        }
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(['X'], 'Out')
+
+
+class RReluTrainingTest(OpTest):
+    def setUp(self):
+        self.op_type = "rrelu"
+        self.lower = 0.3
+        self.upper = 0.3000009
+        self.is_test = False
+        self.init_prams()
+
+
+class RReluTrainingTest(OpTest):
+    def setUp(self):
+        self.op_type = "rrelu"
+        self.lower = 0.3
+        self.upper = 0.3000009
+        self.is_test = False
+        self.init_prams()
+
+
+if __name__ == "__main__":
+    unittest.main()

python/paddle/nn/__init__.py

@@ -51,6 +51,7 @@ from .layer.activation import Tanhshrink  # noqa: F401
 from .layer.activation import ThresholdedReLU  # noqa: F401
 from .layer.activation import LogSoftmax  # noqa: F401
 from .layer.activation import Maxout  # noqa: F401
+from .layer.activation import RReLU  # noqa: F401
 from .layer.common import Pad1D  # noqa: F401
 from .layer.common import Pad2D  # noqa: F401
 from .layer.common import ZeroPad2D  # noqa: F401
@@ -313,4 +314,5 @@ __all__ = [     #noqa
            'MaxUnPool3D',
            'HingeEmbeddingLoss',
            'Identity',
+           'RReLU',
 ]

python/paddle/nn/functional/__init__.py

@@ -47,6 +47,7 @@ from .activation import thresholded_relu  # noqa: F401
 from .activation import log_softmax  # noqa: F401
 from .activation import glu  # noqa: F401
 from .activation import gumbel_softmax  # noqa: F401
+from .activation import rrelu  # noqa: F401
 from .common import dropout  # noqa: F401
 from .common import dropout2d  # noqa: F401
 from .common import dropout3d  # noqa: F401
@@ -228,4 +229,5 @@ __all__ = [     #noqa
            'class_center_sample',
            'sparse_attention',
            'fold',
+           'rrelu',
 ]

python/paddle/nn/functional/activation.py

@@ -550,6 +550,122 @@ def prelu(x, weight, data_format="NCHW", name=None):
     return out
 
 
+def rrelu(x, lower=1. / 8., upper=1. / 3., training=True, name=None):
+    r"""
+    rrelu activation.
+
+    Applies the randomized leaky rectified liner unit function to improve generalization performance,
+    as described in the paper:
+    `Empirical Evaluation of Rectified Activations in Convolutional Network <https://arxiv.org/abs/1505.00853>`_
+
+    During training, randomly samples the negative slope for activation values as described below:
+
+    .. math::
+
+        rrelu(x)=
+            \left\{
+                \begin{array}{rcl}
+                    x, & & if \ x >= 0 \\
+                    a * x, & & otherwise \\
+                \end{array}
+            \right.
+
+    where :math:`x` is the input tensor,
+    :math:`a` is randomly sampled from uniform distribution in range (:math:`lower`, :math:`upper`),
+
+    In the test phase, the negative slope will take the average value of :math:`lower` and :math:`upper`:
+
+    .. math::
+
+        rrelu(x)=
+            \left\{
+                \begin{array}{rcl}
+                    x, & & if \ x >= 0 \\
+                    (lower + upper) * 0.5 * x, & & otherwise \\
+                \end{array}
+            \right.
+
+    where :math:`x` is the input tensor,
+    :math:`lower` and :math:`upper` are the bounds of uniform distribution.
+
+    Parameters:
+        x (Tensor): The input Tensor with data type float16, float32, float64.
+        lower (float, optional): The lower bound of uniform distribution. Default: 0.125.
+        upper (float, optional): The upper bound of uniform distribution. Default: 0.333.
+        training (bool, optional): Current mode is in training or others.  Default is True.
+        name (str, optional): Name for the operation (optional, default is None).
+            For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        A Tensor with the same data type and shape as ``x`` .
+
+    Examples:
+        .. code-block:: python
+            :name: rrelu-example
+
+            import paddle
+            import paddle.nn.functional as F
+
+            input_tensor = paddle.to_tensor([[[[-2.0,  3.0, -4.0,  5.0],
+                                            [ 3.0, -4.0,  5.0, -6.0],
+                                            [-7.0, -8.0,  8.0,  9.0]],
+                                            [[ 1.0, -2.0, -3.0,  4.0],
+                                            [-5.0,  6.0,  7.0, -8.0],
+                                            [ 6.0,  7.0,  8.0,  9.0]]]], dtype='float32')
+
+            out = F.rrelu(input_tensor, 0.1, 0.3)
+            #[[[[-0.20000899  3.         -0.8810822   5.        ]
+            #   [ 3.         -0.55175185  5.         -1.0776101 ]
+            #   [-1.0680687  -1.9896201   8.          9.        ]]
+            #  [[ 1.         -0.5238267  -0.65515125  4.        ]
+            #   [-1.3766339   6.          7.         -2.3465784 ]
+            #   [ 6.          7.          8.          9.        ]]]]
+    """
+
+    if not in_dynamic_mode():
+        check_variable_and_dtype(x, 'X', ['float16', 'float32', 'float64'],
+                                 'rrelu')
+
+    if not isinstance(lower, float) or not isinstance(upper, float):
+        raise TypeError(
+            "The lower and upper values must be float type. Received: lower {}, upper {}.".
+            format(lower, upper))
+
+    if lower < 0 or lower > 1:
+        raise ValueError(
+            "The lower value must be no less than zero or greater than one. Received: {}.".
+            format(lower))
+
+    if upper < lower:
+        raise ValueError(
+            "The upper value must be greater than lower value. Received: lower {}, upper {}.".
+            format(lower, upper))
+
+    if upper > 1:
+        raise ValueError(
+            "The upper value must be no greater than one. Received: {}.".format(
+                upper))
+
+    is_test = not training
+
+    if _in_legacy_dygraph():
+        out, noise = _C_ops.rrelu(x, 'lower', lower, 'upper', upper, 'is_test',
+                                  is_test)
+        return out
+
+    helper = LayerHelper('rrelu', **locals())
+    out = helper.create_variable_for_type_inference(x.dtype)
+    noise = helper.create_variable_for_type_inference(dtype=x.dtype)
+    attrs = {'lower': lower, 'upper': upper, 'is_test': is_test}
+    helper.append_op(
+        type='rrelu',
+        inputs={"X": x},
+        outputs={"Out": out,
+                 "Noise": noise},
+        attrs=attrs)
+    return out
+
+
 def relu(x, name=None):
     """
     relu activation.

python/paddle/nn/layer/__init__.py

@@ -26,6 +26,7 @@ from .activation import LeakyReLU  # noqa: F401
 from .activation import Sigmoid  # noqa: F401
 from .activation import Softmax  # noqa: F401
 from .activation import LogSoftmax  # noqa: F401
+from .activation import RReLU  # noqa: F401
 from .activation import Softmax2D  # noqa: F401
 from .common import Bilinear  # noqa: F401
 from .common import Pad1D  # noqa: F401

python/paddle/nn/layer/activation.py

@@ -436,6 +436,93 @@ class PReLU(Layer):
             name_str)
 
 
+class RReLU(Layer):
+    r"""
+    RReLU activation layer.
+
+    Applies the randomized leaky rectified liner unit function to improve generalization performance,
+    as described in the paper:
+    `Empirical Evaluation of Rectified Activations in Convolutional Network <https://arxiv.org/abs/1505.00853>`_
+
+    During training, randomly samples the negative slope for activation values as described below:
+
+    .. math::
+
+        RReLU(x)=
+            \left\{
+                \begin{array}{rcl}
+                    x, & & if \ x >= 0 \\
+                    a * x, & & otherwise \\
+                \end{array}
+            \right.
+
+    where :math:`x` is the input tensor,
+    :math:`a` is randomly sampled from uniform distribution in range (:math:`lower`, :math:`upper`),
+
+    In the test phase, the negative slope will take the average value of :math:`lower` and :math:`upper`:
+
+    .. math::
+
+        RReLU(x)=
+            \left\{
+                \begin{array}{rcl}
+                    x, & & if \ x >= 0 \\
+                    (lower + upper) * 0.5 * x, & & otherwise \\
+                \end{array}
+            \right.
+
+    where :math:`x` is the input tensor,
+    :math:`lower` and :math:`upper` are the bounds of uniform distribution.
+
+    Parameters:
+        lower (float, optional): The lower bound of uniform distribution. Default: 0.125.
+        upper (float, optional): The upper bound of uniform distribution. Default: 0.333.
+        name (str, optional): Name for the operation (optional, default is None).
+            For more information, please refer to :ref:`api_guide_Name`.
+
+    Shape:
+        - input: Tensor with any shape. Default dtype is float32.
+        - output: Tensor with the same shape as input.
+
+    Examples:
+        .. code-block:: python
+            :name: RReLU-example
+
+            import paddle
+
+            input_tensor = paddle.to_tensor([[[[-2.0,  3.0, -4.0,  5.0],
+                                            [ 3.0, -4.0,  5.0, -6.0],
+                                            [-7.0, -8.0,  8.0,  9.0]],
+                                            [[ 1.0, -2.0, -3.0,  4.0],
+                                            [-5.0,  6.0,  7.0, -8.0],
+                                            [ 6.0,  7.0,  8.0,  9.0]]]], dtype='float32')
+
+            rrelu_layer = paddle.nn.RReLU(0.1, 0.3)
+            output = rrelu_layer(input_tensor)
+            #[[[[-0.20000899  3.         -0.88108218  5.        ]
+            #   [ 3.         -0.55175185  5.         -1.07761011]
+            #   [-1.06806871 -1.98962009  8.          9.        ]]
+            #  [[ 1.         -0.52382672 -0.65515128  4.        ]
+            #   [-1.37663394  6.          7.         -2.34657836]
+            #   [ 6.          7.          8.          9.        ]]]]
+    """
+
+    def __init__(self, lower=1. / 8., upper=1. / 3., name=None):
+        super(RReLU, self).__init__()
+        self._lower = lower
+        self._upper = upper
+        self._name = name
+
+    def forward(self, x):
+        return F.rrelu(
+            x, lower=self._lower, upper=self._upper, training=self.training)
+
+    def extra_repr(self):
+        name_str = ', name={}'.format(self._name) if self._name else ''
+        return 'lower={}, upper={}, training={}, dtype={}{}'.format(
+            self._lower, self._upper, self.training, self._dtype, name_str)
+
+
 class ReLU(Layer):
     """
     ReLU Activation.

tools/static_mode_white_list.py

@@ -399,6 +399,7 @@ STATIC_MODE_TESTING_LIST = [
     'test_positive_negative_pair_op',
     'test_precision_recall_op',
     'test_prelu_op',
+    'test_rrelu_op',
     'test_prelu_mkldnn_op',
     'test_print_op',
     'test_prior_box_op',