Merge pull request #4487 from abhinavarora/softsign_activation

Implementing the SoftSign activation operator

paddle/operators/activation_op.cc

@@ -132,6 +132,17 @@ class SquareOpMaker : public framework::OpProtoAndCheckerMaker {
   }
 };
 
+class SoftsignOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  SoftsignOpMaker(framework::OpProto *proto,
+                  framework::OpAttrChecker *op_checker)
+      : OpProtoAndCheckerMaker(proto, op_checker) {
+    AddInput("X", "Input of Softsign operator");
+    AddOutput("Y", "Output of Softsign operator");
+    AddComment("Softsign activation operator, softsign(x) = x / (1 + |x|)");
+  }
+};
+
 template <typename AttrType>
 class BReluOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
@@ -277,6 +288,15 @@ REGISTER_OP_CPU_KERNEL(
     square_grad, ops::ActivationGradKernel<paddle::platform::CPUPlace, float,
                                            ops::SquareGradFunctor<float>>);
 
+REGISTER_OP(softsign, ops::ActivationOp, ops::SoftsignOpMaker, softsign_grad,
+            ops::ActivationOpGrad);
+REGISTER_OP_CPU_KERNEL(softsign,
+                       ops::ActivationKernel<paddle::platform::CPUPlace, float,
+                                             ops::SoftsignFunctor<float>>);
+REGISTER_OP_CPU_KERNEL(
+    softsign_grad, ops::ActivationGradKernel<paddle::platform::CPUPlace, float,
+                                             ops::SoftsignGradFunctor<float>>);
+
 REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker<float>, brelu_grad,
             ops::ActivationOpGrad);
 REGISTER_OP_CPU_KERNEL(brelu,

paddle/operators/activation_op.cu

@@ -80,6 +80,13 @@ REGISTER_OP_GPU_KERNEL(
     square_grad, ops::ActivationGradKernel<paddle::platform::GPUPlace, float,
                                            ops::SquareGradFunctor<float>>);
 
+REGISTER_OP_GPU_KERNEL(softsign,
+                       ops::ActivationKernel<paddle::platform::GPUPlace, float,
+                                             ops::SoftsignFunctor<float>>);
+REGISTER_OP_GPU_KERNEL(
+    softsign_grad, ops::ActivationGradKernel<paddle::platform::GPUPlace, float,
+                                             ops::SoftsignGradFunctor<float>>);
+
 REGISTER_OP_GPU_KERNEL(brelu,
                        ops::BReluKernel<paddle::platform::GPUPlace, float>);
 REGISTER_OP_GPU_KERNEL(brelu_grad,

paddle/operators/activation_op.h

@@ -201,6 +201,26 @@ struct SquareGradFunctor {
   }
 };
 
+// softsign(x) = x / (1 + |x|)
+template <typename T>
+struct SoftsignFunctor {
+  template <typename Device, typename X, typename Y>
+  void operator()(Device d, X x, Y y) {
+    y.device(d) = x / (static_cast<T>(1) + x.abs());
+  }
+};
+
+// d(softsign(x))/dx = 1 / (1 + |x|)^2
+// Taken from https://en.wikipedia.org/wiki/Activation_function
+template <typename T>
+struct SoftsignGradFunctor {
+  template <typename Device, typename X, typename Y, typename dY, typename dX>
+  void operator()(Device d, X x, Y y, dY dy, dX dx) {
+    dx.device(d) =
+        dy * (static_cast<T>(1) / (static_cast<T>(1) + x.abs()).square());
+  }
+};
+
 template <typename Place, typename T, typename AttrType = T>
 class BReluKernel : public framework::OpKernel<T> {
  public:

python/paddle/v2/framework/tests/test_activation_op.py

@@ -219,5 +219,22 @@ class TestSTanh(OpTest):
         self.check_grad(['X'], 'Y', max_relative_error=0.007)
 
 
+class TestSoftsign(OpTest):
+    def setUp(self):
+        self.op_type = "softsign"
+        self.inputs = {
+            'X': np.random.uniform(-1, 1, [11, 17]).astype("float32")
+        }
+        self.outputs = {
+            'Y': np.divide(self.inputs['X'], 1 + np.abs(self.inputs['X']))
+        }
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(['X'], 'Y', max_relative_error=0.007)
+
+
 if __name__ == "__main__":
     unittest.main()