From 0c3eee09ff7c00c3c279b21a20278c154f045923 Mon Sep 17 00:00:00 2001
From: Abhinav Arora <aroraabhinav@baidu.com>
Date: Thu, 28 Sep 2017 18:15:16 -0700
Subject: [PATCH] Implementing the SoftSign activation operator

---
 paddle/operators/activation_op.cc             | 20 +++++++++++++++++++
 paddle/operators/activation_op.cu             |  7 +++++++
 paddle/operators/activation_op.h              | 20 +++++++++++++++++++
 .../v2/framework/tests/test_activation_op.py  | 17 ++++++++++++++++
 4 files changed, 64 insertions(+)

diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index f77e1c572e3..1e1d3cf7f76 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -132,6 +132,17 @@ class SquareOpMaker : public framework::OpProtoAndCheckerMaker {
   }
 };
 
+class SoftsignOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  SoftsignOpMaker(framework::OpProto *proto,
+                  framework::OpAttrChecker *op_checker)
+      : OpProtoAndCheckerMaker(proto, op_checker) {
+    AddInput("X", "Input of Softsign operator");
+    AddOutput("Y", "Output of Softsign operator");
+    AddComment("Softsign activation operator, softsign(x) = x / (1 + |x|)");
+  }
+};
+
 template <typename AttrType>
 class BReluOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
@@ -277,6 +288,15 @@ REGISTER_OP_CPU_KERNEL(
     square_grad, ops::ActivationGradKernel<paddle::platform::CPUPlace, float,
                                            ops::SquareGradFunctor<float>>);
 
+REGISTER_OP(softsign, ops::ActivationOp, ops::SoftsignOpMaker, softsign_grad,
+            ops::ActivationOpGrad);
+REGISTER_OP_CPU_KERNEL(softsign,
+                       ops::ActivationKernel<paddle::platform::CPUPlace, float,
+                                             ops::SoftsignFunctor<float>>);
+REGISTER_OP_CPU_KERNEL(
+    softsign_grad, ops::ActivationGradKernel<paddle::platform::CPUPlace, float,
+                                             ops::SoftsignGradFunctor<float>>);
+
 REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker<float>, brelu_grad,
             ops::ActivationOpGrad);
 REGISTER_OP_CPU_KERNEL(brelu,
diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu
index feed1302b29..56886d8b1b9 100644
--- a/paddle/operators/activation_op.cu
+++ b/paddle/operators/activation_op.cu
@@ -80,6 +80,13 @@ REGISTER_OP_GPU_KERNEL(
     square_grad, ops::ActivationGradKernel<paddle::platform::GPUPlace, float,
                                            ops::SquareGradFunctor<float>>);
 
+REGISTER_OP_GPU_KERNEL(softsign,
+                       ops::ActivationKernel<paddle::platform::GPUPlace, float,
+                                             ops::SoftsignFunctor<float>>);
+REGISTER_OP_GPU_KERNEL(
+    softsign_grad, ops::ActivationGradKernel<paddle::platform::GPUPlace, float,
+                                             ops::SoftsignGradFunctor<float>>);
+
 REGISTER_OP_GPU_KERNEL(brelu,
                        ops::BReluKernel<paddle::platform::GPUPlace, float>);
 REGISTER_OP_GPU_KERNEL(brelu_grad,
diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h
index e400992ae29..b9f52e1af39 100644
--- a/paddle/operators/activation_op.h
+++ b/paddle/operators/activation_op.h
@@ -201,6 +201,26 @@ struct SquareGradFunctor {
   }
 };
 
+// softsign(x) = x / (1 + |x|)
+template <typename T>
+struct SoftsignFunctor {
+  template <typename Device, typename X, typename Y>
+  void operator()(Device d, X x, Y y) {
+    y.device(d) = x / (static_cast<T>(1) + x.abs());
+  }
+};
+
+// d(softsign(x))/dx = 1 / (1 + |x|)^2
+// Taken from https://en.wikipedia.org/wiki/Activation_function
+template <typename T>
+struct SoftsignGradFunctor {
+  template <typename Device, typename X, typename Y, typename dY, typename dX>
+  void operator()(Device d, X x, Y y, dY dy, dX dx) {
+    dx.device(d) =
+        dy * (static_cast<T>(1) / (static_cast<T>(1) + x.abs()).square());
+  }
+};
+
 template <typename Place, typename T, typename AttrType = T>
 class BReluKernel : public framework::OpKernel<T> {
  public:
diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py
index 8f6d2be1775..c44eb849063 100644
--- a/python/paddle/v2/framework/tests/test_activation_op.py
+++ b/python/paddle/v2/framework/tests/test_activation_op.py
@@ -219,5 +219,22 @@ class TestSTanh(OpTest):
         self.check_grad(['X'], 'Y', max_relative_error=0.007)
 
 
+class TestSoftsign(OpTest):
+    def setUp(self):
+        self.op_type = "softsign"
+        self.inputs = {
+            'X': np.random.uniform(-1, 1, [11, 17]).astype("float32")
+        }
+        self.outputs = {
+            'Y': np.divide(self.inputs['X'], 1 + np.abs(self.inputs['X']))
+        }
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(['X'], 'Y', max_relative_error=0.007)
+
+
 if __name__ == "__main__":
     unittest.main()
-- 
GitLab