diff --git a/python/paddle/fluid/dygraph_utils.py b/python/paddle/fluid/dygraph_utils.py
index baa6e05289f6dfe79003b99877dabb26278cc06b..f8faa3d55b751583046200494e4a1897f4582b75 100644
--- a/python/paddle/fluid/dygraph_utils.py
+++ b/python/paddle/fluid/dygraph_utils.py
@@ -19,8 +19,8 @@ from .framework import dygraph_only
 @dygraph_only
 def _append_activation_in_dygraph(input,
                                   act=None,
-                                  use_cudnn=False,
-                                  use_mkldnn=False):
+                                  use_cudnn=None,
+                                  use_mkldnn=None):
     """Append activation in dygraph mode.
 
         Args:
@@ -33,8 +33,11 @@ def _append_activation_in_dygraph(input,
     """
     if not act:
         return input
-
-    attrs = {'use_cudnn': use_cudnn, 'use_mkldnn': use_mkldnn}
+    attrs = {}
+    if (use_cudnn is not None) and use_cudnn:
+        attrs['use_cudnn'] = use_cudnn
+    if (use_mkldnn is not None) and use_mkldnn:
+        attrs['use_mkldnn'] = use_mkldnn
     inputs = {"X": [input]}
     act_op = getattr(core.ops, act)
     res = act_op(inputs, attrs)
@@ -42,10 +45,7 @@ def _append_activation_in_dygraph(input,
 
 
 @dygraph_only
-def _append_bias_in_dygraph(
-        input,
-        bias=None,
-        axis=1, ):
+def _append_bias_in_dygraph(input, bias=None, axis=1):
     """Append bias operation in dygraph mode.
 
         Args:
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py
index b43e284d8c49a4c154b0e581ddc5d2fc52189c3c..32c887a618f3ed5df46c0bfd06fc38c417845487 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py
@@ -20,6 +20,7 @@ import paddle.fluid as fluid
 from paddle.fluid import core
 from paddle.fluid import Linear
 from test_imperative_base import new_program_scope
+import paddle.fluid.dygraph_utils as dygraph_utils
 
 
 class MyLayer(fluid.Layer):
@@ -536,5 +537,48 @@ class TestImperative(unittest.TestCase):
         self.assertEqual(len(my_layer.sublayers()), 0)
 
 
+class TestDygraphUtils(unittest.TestCase):
+    def test_append_activation_in_dygraph_exception(self):
+        with new_program_scope():
+            np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32)
+            a = fluid.layers.data("a", [10, 20])
+            func = dygraph_utils._append_activation_in_dygraph
+            self.assertRaises(AssertionError, func, a, act="sigmoid")
+
+    def test_append_activation_in_dygraph1(self):
+        a_np = np.random.random(size=(10, 20, 30)).astype(np.float32)
+        func = dygraph_utils._append_activation_in_dygraph
+        with fluid.dygraph.guard():
+            a = fluid.dygraph.to_variable(a_np)
+            res1 = func(a, act="hard_sigmoid")
+            res2 = fluid.layers.hard_sigmoid(a)
+            self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
+
+    def test_append_activation_in_dygraph2(self):
+        a_np = np.random.random(size=(10, 20, 30)).astype(np.float32)
+        func = dygraph_utils._append_activation_in_dygraph
+        with fluid.dygraph.guard():
+            a = fluid.dygraph.to_variable(a_np)
+            res1 = func(a, act="sigmoid", use_mkldnn=True, use_cudnn=True)
+            res2 = fluid.layers.sigmoid(a)
+            self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
+
+    def test_append_bias_in_dygraph_exception(self):
+        with new_program_scope():
+            np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32)
+            a = fluid.layers.data("a", [10, 20])
+            func = dygraph_utils._append_bias_in_dygraph
+            self.assertRaises(AssertionError, func, a)
+
+    def test_append_bias_in_dygraph(self):
+        a_np = np.random.random(size=(10, 20, 30)).astype(np.float32)
+        func = dygraph_utils._append_bias_in_dygraph
+        with fluid.dygraph.guard():
+            a = fluid.dygraph.to_variable(a_np)
+            res1 = func(a, bias=a)
+            res2 = a + a
+            self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
+
+
 if __name__ == '__main__':
     unittest.main()