diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index ab6b8edd52eae4f9e04821611b2cd34d00f54f1d..ae00953f2cf4118f7e55486341b228656978ba7e 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -1417,6 +1417,16 @@ static PyObject* tensor_method_get_non_zero_cols(TensorObject* self,
   EAGER_CATCH_AND_THROW_RETURN_NULL
 }
 
+static PyObject* tensor_method_is_dense(TensorObject* self, PyObject* args,
+                                        PyObject* kwargs) {
+  EAGER_TRY
+  if (!self->tensor.defined()) {
+    return ToPyObject(false);
+  }
+  return ToPyObject(self->tensor.is_dense_tensor());
+  EAGER_CATCH_AND_THROW_RETURN_NULL
+}
+
 static PyObject* tensor_method_is_sparse(TensorObject* self, PyObject* args,
                                          PyObject* kwargs) {
   EAGER_TRY
@@ -1682,6 +1692,8 @@ PyMethodDef variable_methods[] = {
      METH_VARARGS | METH_KEYWORDS, NULL},
     {"clear_gradient", (PyCFunction)(void (*)(void))tensor_clear_gradient,
      METH_VARARGS | METH_KEYWORDS, NULL},
+    {"is_dense", (PyCFunction)(void (*)(void))tensor_method_is_dense,
+     METH_VARARGS | METH_KEYWORDS, NULL},
     {"_zero_grads", (PyCFunction)(void (*)(void))tensor__zero_grads,
      METH_VARARGS | METH_KEYWORDS, NULL},
     {"_share_buffer_to", (PyCFunction)(void (*)(void))tensor__share_buffer_to,
diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index ec367c7c710eda3ba37d5afbda0cce6065f82d3f..4534c39b0082af5e952c4cdd92e9e6361b39b077 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -1032,7 +1032,9 @@ class Optimizer(object):
         assert regularization_term is not None
 
         if framework.in_dygraph_mode():
-            return _C_ops.final_state_add_n([grad, regularization_term])
+            if grad.is_dense() and regularization_term.is_dense():
+                return _C_ops.final_state_add_n([grad, regularization_term])
+            return _C_ops.sum([grad, regularization_term])
         elif framework._in_legacy_dygraph():
             return _C_ops.sum([grad, regularization_term])
 
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 4611cbb20c96a961da2495cc5e7cf7be73bbd1b4..ffca233ff16bf92f97737329a2bf89bc575b950c 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -1377,6 +1377,9 @@ def add_n(inputs, name=None):
     if in_dygraph_mode():
         if isinstance(inputs, Variable):
             inputs = [inputs]
+        for x in inputs:
+            if not x.is_dense():
+                return _C_ops.sum(inputs, 'use_mkldnn', False)
         return _C_ops.final_state_add_n(inputs)
     if _in_legacy_dygraph():
         if isinstance(inputs, Variable):