From 1bd9cfef4e27baa84fd40ed1e65e80017d0cf232 Mon Sep 17 00:00:00 2001 From: arlesniak Date: Fri, 8 Oct 2021 05:33:09 +0200 Subject: [PATCH] Added oneDNN BF16 relu (#36265) * Added oneDNN BF16 relu * fixed typo * refactored test, review fixes --- .../operators/mkldnn/activation_mkldnn_op.cc | 3 +- .../mkldnn/test_activation_bf16_mkldnn_op.py | 44 ++++++++++++++++--- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc index d992890adee..603a70458b0 100644 --- a/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc @@ -257,7 +257,6 @@ namespace ops = paddle::operators; ops::grad_functor>); #define FOR_EACH_MKLDNN_KERNEL_FUNCTOR(__macro) \ - __macro(relu, ReluMKLDNNFunctor, ReluMKLDNNGradFunctor); \ __macro(relu6, Relu6MKLDNNFunctor, Relu6MKLDNNGradFunctor); \ __macro(leaky_relu, ReluMKLDNNFunctor, ReluMKLDNNGradFunctor); \ __macro(swish, SwishMKLDNNFunctor, SwishMKLDNNGradFunctor); \ @@ -267,6 +266,8 @@ namespace ops = paddle::operators; __macro(abs, AbsMKLDNNFunctor, AbsMKLDNNGradFunctor); FOR_EACH_MKLDNN_KERNEL_FUNCTOR(REGISTER_ACTIVATION_MKLDNN_KERNEL); +REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(relu, ReluMKLDNNFunctor, + ReluMKLDNNGradFunctor); REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(gelu, GeluMKLDNNFunctor, GeluMKLDNNGradFunctor); REGISTER_ACTIVATION_MKLDNN_BF16_KERNEL(sigmoid, SigmoidMKLDNNFunctor, diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py index 3d5a0139158..cd9987b3c8e 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py @@ -14,6 +14,8 @@ from __future__ import print_function +import six +import abc import unittest import numpy as np from scipy.special import expit, erf @@ -24,15 +26,19 @@ from paddle.fluid.tests.unittests.test_gelu_op import gelu @OpTestTool.skip_if_not_cpu_bf16() -class TestMKLDNNSigmoidBF16Op(TestActivation): +@six.add_metaclass(abc.ABCMeta) +class MKLDNNBF16ActivationOp(object): + @abc.abstractmethod def config(self): - self.op_type = "sigmoid" + pass + @abc.abstractmethod def op_forward(self, x): - return 1 / (1 + np.exp(-x)) + pass + @abc.abstractmethod def op_grad(self, dout, x): - return dout * self.op_forward(x) * (1 - self.op_forward(x)) + pass def set_attrs(self): self.attrs = {"use_mkldnn": True} @@ -65,7 +71,18 @@ class TestMKLDNNSigmoidBF16Op(TestActivation): user_defined_grad_outputs=[convert_float_to_uint16(self.out)]) -class TestMKLDNNGeluErfBF16Op(TestMKLDNNSigmoidBF16Op): +class TestMKLDNNSigmoidBF16Op(MKLDNNBF16ActivationOp, TestActivation): + def config(self): + self.op_type = "sigmoid" + + def op_forward(self, x): + return 1 / (1 + np.exp(-x)) + + def op_grad(self, dout, x): + return dout * self.op_forward(x) * (1 - self.op_forward(x)) + + +class TestMKLDNNGeluErfBF16Op(MKLDNNBF16ActivationOp, TestActivation): def config(self): self.op_type = "gelu" @@ -83,7 +100,7 @@ class TestMKLDNNGeluErfDim2BF16Op(TestMKLDNNGeluErfBF16Op): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) -class TestMKLDNNGeluTanhBF16Op(TestMKLDNNSigmoidBF16Op): +class TestMKLDNNGeluTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): def config(self): self.op_type = "gelu" @@ -104,3 +121,18 @@ class TestMKLDNNGeluTanhBF16Op(TestMKLDNNSigmoidBF16Op): class TestMKLDNNGeluTanhDim2BF16Op(TestMKLDNNGeluTanhBF16Op): def init_data(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) + + +class TestMKLDNNReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): + def config(self): + self.op_type = "relu" + + def op_forward(self, x): + return np.maximum(x, 0) + + def op_grad(self, dout, x): + return dout + + +if __name__ == '__main__': + unittest.main() -- GitLab