diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index 95a261be1a70131dd1282c2ce1f027c52a41c5ce..2e5e09251aaa733b079f24e2b8dd421dc94394e6 100644
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -148,6 +148,7 @@ cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc
 cc_library(op_registry SRCS op_registry.cc DEPS op_proto_maker op_info operator glog proto_desc)
 
 cc_library(op_call_stack SRCS op_call_stack.cc DEPS op_proto_maker enforce)
+cc_test(op_call_stack_test SRCS op_call_stack_test.cc DEPS op_call_stack)
 
 nv_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry)
 
diff --git a/paddle/fluid/framework/grad_op_desc_maker.h b/paddle/fluid/framework/grad_op_desc_maker.h
index 368e4c1f90f15792bd030f5430eb742e3b41ec28..7a3ba0863cf20d69a37d515dd17089c9f46cca26 100644
--- a/paddle/fluid/framework/grad_op_desc_maker.h
+++ b/paddle/fluid/framework/grad_op_desc_maker.h
@@ -18,7 +18,9 @@ limitations under the License. */
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
+#include "paddle/fluid/framework/op_call_stack.h"
 #include "paddle/fluid/framework/op_desc.h"
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/imperative/dygraph_grad_maker.h"
@@ -195,7 +197,14 @@ class SingleGradOpMaker<OpDesc> : public GradOpDescMakerBase {
   std::vector<std::unique_ptr<OpDesc>> operator()() const final {
     std::vector<std::unique_ptr<OpDesc>> retv;
     retv.emplace_back(new OpDesc());
-    this->Apply(retv.front().get());
+    try {
+      this->Apply(retv.front().get());
+    } catch (platform::EnforceNotMet& exception) {
+      framework::AppendErrorOpHint(retv.front().get()->Type(), &exception);
+      throw std::move(exception);
+    } catch (...) {
+      std::rethrow_exception(std::current_exception());
+    }
     return retv;
   }
 
@@ -213,7 +222,14 @@ class SingleGradOpMaker<imperative::OpBase>
     auto node = this->NewGradNode();
     {
       imperative::TracedGradOp traced_grad_op(node);
-      this->Apply(&traced_grad_op);
+      try {
+        this->Apply(&traced_grad_op);
+      } catch (platform::EnforceNotMet& exception) {
+        framework::AppendErrorOpHint(traced_grad_op.Type(), &exception);
+        throw std::move(exception);
+      } catch (...) {
+        std::rethrow_exception(std::current_exception());
+      }
     }
     return node->empty() ? nullptr : node;
   }
diff --git a/paddle/fluid/framework/op_call_stack.cc b/paddle/fluid/framework/op_call_stack.cc
index fdbcf74d64b2423c4cdaa29cd1b0cacbc7ad0869..3a9b113ceac573c831ce39993d7e2f6df37ee5fe 100644
--- a/paddle/fluid/framework/op_call_stack.cc
+++ b/paddle/fluid/framework/op_call_stack.cc
@@ -56,9 +56,15 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
   }
   // Step 3. Construct final call stack & append error op name
   sout << exception->err_str_;
-  if (callstack) {
-    sout << "  [operator < " << type << " > error]";
-  }
+  sout << "  [operator < " << type << " > error]";
+  exception->err_str_ = sout.str();
+}
+
+void AppendErrorOpHint(const std::string &type,
+                       platform::EnforceNotMet *exception) {
+  std::ostringstream sout;
+  sout << exception->err_str_;
+  sout << "  [operator < " << type << " > error]";
   exception->err_str_ = sout.str();
 }
 
diff --git a/paddle/fluid/framework/op_call_stack.h b/paddle/fluid/framework/op_call_stack.h
index 4408601abf0b3542c9850f9264d162faaa6a50ce..d48cf27285a0a5040b5e375a27ccf6a8b00bd8c0 100644
--- a/paddle/fluid/framework/op_call_stack.h
+++ b/paddle/fluid/framework/op_call_stack.h
@@ -20,7 +20,14 @@ limitations under the License. */
 
 namespace paddle {
 namespace framework {
+
+// insert python call stack & append error op for exception message
 void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
                          platform::EnforceNotMet *exception);
+
+// only append error op for exception message
+void AppendErrorOpHint(const std::string &type,
+                       platform::EnforceNotMet *exception);
+
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/op_call_stack_test.cc b/paddle/fluid/framework/op_call_stack_test.cc
new file mode 100644
index 0000000000000000000000000000000000000000..93db97a93f4ca0182cdfc996015637565eb2e35e
--- /dev/null
+++ b/paddle/fluid/framework/op_call_stack_test.cc
@@ -0,0 +1,61 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/framework/op_call_stack.h"
+
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+namespace paddle {
+namespace framework {
+namespace details {
+
+static void ThrowEnforceNotMet() {
+  PADDLE_THROW(platform::errors::InvalidArgument(
+      "\n----------------------\nError Message "
+      "Summary:\n----------------------\n"
+      "Created error."));
+}
+
+}  // namespace details
+}  // namespace framework
+}  // namespace paddle
+
+TEST(OpCallStack, InsertCallStackInfo) {
+  try {
+    paddle::framework::details::ThrowEnforceNotMet();
+  } catch (paddle::platform::EnforceNotMet &exception) {
+    paddle::framework::AttributeMap attr_map;
+    std::string stack_test_str = "test for op callstack";
+    std::vector<std::string> stack_test_vec;
+    stack_test_vec.emplace_back(stack_test_str);
+    attr_map["op_callstack"] = stack_test_vec;
+    paddle::framework::InsertCallStackInfo("test", attr_map, &exception);
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find(stack_test_str) != std::string::npos);
+    EXPECT_TRUE(ex_msg.find("[operator < test > error]") != std::string::npos);
+  }
+}
+
+TEST(OpCallStack, AppendErrorOpHint) {
+  try {
+    paddle::framework::details::ThrowEnforceNotMet();
+  } catch (paddle::platform::EnforceNotMet &exception) {
+    paddle::framework::AppendErrorOpHint("test", &exception);
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find("[operator < test > error]") != std::string::npos);
+  }
+}
diff --git a/paddle/fluid/imperative/dygraph_grad_maker.h b/paddle/fluid/imperative/dygraph_grad_maker.h
index a7bb47d40d5f1a4bc9b3ee355cd239d7255c7e73..f21781fbbecfb4b168c6a0ec0276707fbe7ddec1 100644
--- a/paddle/fluid/imperative/dygraph_grad_maker.h
+++ b/paddle/fluid/imperative/dygraph_grad_maker.h
@@ -258,6 +258,8 @@ class TracedGradOp {
     }
   }
 
+  std::string Type() const { return op_->Type(); }
+
   void SetType(const std::string& type) { op_->SetType(type); }
 
   void SetAttrMap(const framework::AttributeMap& attrs) {