Added oneDNN BF16 relu (#36265)

* Added oneDNN BF16 relu

* fixed typo

* refactored test, review fixes

paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc

@@ -257,7 +257,6 @@ namespace ops = paddle::operators;
           ops::grad_functor<paddle::platform::bfloat16>>);
 
 #define FOR_EACH_MKLDNN_KERNEL_FUNCTOR(__macro)                           \
-  __macro(relu, ReluMKLDNNFunctor, ReluMKLDNNGradFunctor);                \
   __macro(relu6, Relu6MKLDNNFunctor, Relu6MKLDNNGradFunctor);             \
   __macro(leaky_relu, ReluMKLDNNFunctor, ReluMKLDNNGradFunctor);          \
   __macro(swish, SwishMKLDNNFunctor, SwishMKLDNNGradFunctor);             \
@@ -267,6 +266,8 @@ namespace ops = paddle::operators;
   __macro(abs, AbsMKLDNNFunctor, AbsMKLDNNGradFunctor);
 
 FOR_EACH_MKLDNN_KERNEL_FUNCTOR(REGISTER_ACTIVATION_MKLDNN_KERNEL);
+REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(relu, ReluMKLDNNFunctor,
+                                       ReluMKLDNNGradFunctor);
 REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(gelu, GeluMKLDNNFunctor,
                                        GeluMKLDNNGradFunctor);
 REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(sigmoid, SigmoidMKLDNNFunctor,

python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py

@@ -14,6 +14,8 @@
 
 from __future__ import print_function
 
+import six
+import abc
 import unittest
 import numpy as np
 from scipy.special import expit, erf
@@ -24,15 +26,19 @@ from paddle.fluid.tests.unittests.test_gelu_op import gelu
 
 
 @OpTestTool.skip_if_not_cpu_bf16()
-class TestMKLDNNSigmoidBF16Op(TestActivation):
+@six.add_metaclass(abc.ABCMeta)
+class MKLDNNBF16ActivationOp(object):
+    @abc.abstractmethod
     def config(self):
-        self.op_type = "sigmoid"
+        pass
 
+    @abc.abstractmethod
     def op_forward(self, x):
-        return 1 / (1 + np.exp(-x))
+        pass
 
+    @abc.abstractmethod
     def op_grad(self, dout, x):
-        return dout * self.op_forward(x) * (1 - self.op_forward(x))
+        pass
 
     def set_attrs(self):
         self.attrs = {"use_mkldnn": True}
@@ -65,7 +71,18 @@ class TestMKLDNNSigmoidBF16Op(TestActivation):
             user_defined_grad_outputs=[convert_float_to_uint16(self.out)])
 
 
-class TestMKLDNNGeluErfBF16Op(TestMKLDNNSigmoidBF16Op):
+class TestMKLDNNSigmoidBF16Op(MKLDNNBF16ActivationOp, TestActivation):
+    def config(self):
+        self.op_type = "sigmoid"
+
+    def op_forward(self, x):
+        return 1 / (1 + np.exp(-x))
+
+    def op_grad(self, dout, x):
+        return dout * self.op_forward(x) * (1 - self.op_forward(x))
+
+
+class TestMKLDNNGeluErfBF16Op(MKLDNNBF16ActivationOp, TestActivation):
     def config(self):
         self.op_type = "gelu"
 
@@ -83,7 +100,7 @@ class TestMKLDNNGeluErfDim2BF16Op(TestMKLDNNGeluErfBF16Op):
         self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32)
 
 
-class TestMKLDNNGeluTanhBF16Op(TestMKLDNNSigmoidBF16Op):
+class TestMKLDNNGeluTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation):
     def config(self):
         self.op_type = "gelu"
 
@@ -104,3 +121,18 @@ class TestMKLDNNGeluTanhBF16Op(TestMKLDNNSigmoidBF16Op):
 class TestMKLDNNGeluTanhDim2BF16Op(TestMKLDNNGeluTanhBF16Op):
     def init_data(self):
         self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32)
+
+
+class TestMKLDNNReluBF16Op(MKLDNNBF16ActivationOp, TestActivation):
+    def config(self):
+        self.op_type = "relu"
+
+    def op_forward(self, x):
+        return np.maximum(x, 0)
+
+    def op_grad(self, dout, x):
+        return dout
+
+
+if __name__ == '__main__':
+    unittest.main()