diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index 361f25d09e70006e0dcba31977379a756ba8c96f..5e77ac4c6a3b36630059e4f4ce4dec0542f74bc6 100644
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -148,6 +148,7 @@ cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc
 cc_library(op_registry SRCS op_registry.cc DEPS op_proto_maker op_info operator glog proto_desc)
 
 cc_library(op_call_stack SRCS op_call_stack.cc DEPS op_proto_maker enforce)
+cc_test(op_call_stack_test SRCS op_call_stack_test.cc DEPS op_call_stack)
 
 nv_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry)
 
diff --git a/paddle/fluid/framework/grad_op_desc_maker.h b/paddle/fluid/framework/grad_op_desc_maker.h
index 8d55c79a0ddd540d5fd4ff114b71f6192af6a254..b1ca10c61553daffb077bf2c3010a7d5496568b8 100644
--- a/paddle/fluid/framework/grad_op_desc_maker.h
+++ b/paddle/fluid/framework/grad_op_desc_maker.h
@@ -18,7 +18,9 @@ limitations under the License. */
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
+#include "paddle/fluid/framework/op_call_stack.h"
 #include "paddle/fluid/framework/op_desc.h"
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/imperative/dygraph_grad_maker.h"
@@ -195,7 +197,14 @@ class SingleGradOpMaker<OpDesc> : public GradOpDescMakerBase {
   std::vector<std::unique_ptr<OpDesc>> operator()() const {
     std::vector<std::unique_ptr<OpDesc>> retv;
     retv.emplace_back(new OpDesc());
-    this->Apply(retv.front().get());
+    try {
+      this->Apply(retv.front().get());
+    } catch (platform::EnforceNotMet& exception) {
+      framework::AppendErrorOpHint(retv.front().get()->Type(), &exception);
+      throw std::move(exception);
+    } catch (...) {
+      std::rethrow_exception(std::current_exception());
+    }
     return retv;
   }
 
@@ -213,7 +222,14 @@ class SingleGradOpMaker<imperative::OpBase>
     auto node = this->NewGradNode();
     {
       imperative::TracedGradOp traced_grad_op(node);
-      this->Apply(&traced_grad_op);
+      try {
+        this->Apply(&traced_grad_op);
+      } catch (platform::EnforceNotMet& exception) {
+        framework::AppendErrorOpHint(traced_grad_op.Type(), &exception);
+        throw std::move(exception);
+      } catch (...) {
+        std::rethrow_exception(std::current_exception());
+      }
     }
     return node->empty() ? nullptr : node;
   }
diff --git a/paddle/fluid/framework/op_call_stack.cc b/paddle/fluid/framework/op_call_stack.cc
index c3c56210b62aad2d18aacf6b3c075e22b3909994..dee98969902af153984514e1da39397f5ca5546f 100644
--- a/paddle/fluid/framework/op_call_stack.cc
+++ b/paddle/fluid/framework/op_call_stack.cc
@@ -56,9 +56,15 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
   }
   // Step 3. Construct final call stack & append error op name
   sout << exception->err_str_;
-  if (callstack) {
-    sout << "  [operator < " << type << " > error]";
-  }
+  sout << "  [operator < " << type << " > error]";
+  exception->err_str_ = sout.str();
+}
+
+void AppendErrorOpHint(const std::string &type,
+                       platform::EnforceNotMet *exception) {
+  std::ostringstream sout;
+  sout << exception->err_str_;
+  sout << "  [operator < " << type << " > error]";
   exception->err_str_ = sout.str();
 }
 
diff --git a/paddle/fluid/framework/op_call_stack.h b/paddle/fluid/framework/op_call_stack.h
index 4408601abf0b3542c9850f9264d162faaa6a50ce..d48cf27285a0a5040b5e375a27ccf6a8b00bd8c0 100644
--- a/paddle/fluid/framework/op_call_stack.h
+++ b/paddle/fluid/framework/op_call_stack.h
@@ -20,7 +20,14 @@ limitations under the License. */
 
 namespace paddle {
 namespace framework {
+
+// insert python call stack & append error op for exception message
 void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs,
                          platform::EnforceNotMet *exception);
+
+// only append error op for exception message
+void AppendErrorOpHint(const std::string &type,
+                       platform::EnforceNotMet *exception);
+
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/op_call_stack_test.cc b/paddle/fluid/framework/op_call_stack_test.cc
new file mode 100644
index 0000000000000000000000000000000000000000..93db97a93f4ca0182cdfc996015637565eb2e35e
--- /dev/null
+++ b/paddle/fluid/framework/op_call_stack_test.cc
@@ -0,0 +1,61 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/framework/op_call_stack.h"
+
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+namespace paddle {
+namespace framework {
+namespace details {
+
+static void ThrowEnforceNotMet() {
+  PADDLE_THROW(platform::errors::InvalidArgument(
+      "\n----------------------\nError Message "
+      "Summary:\n----------------------\n"
+      "Created error."));
+}
+
+}  // namespace details
+}  // namespace framework
+}  // namespace paddle
+
+TEST(OpCallStack, InsertCallStackInfo) {
+  try {
+    paddle::framework::details::ThrowEnforceNotMet();
+  } catch (paddle::platform::EnforceNotMet &exception) {
+    paddle::framework::AttributeMap attr_map;
+    std::string stack_test_str = "test for op callstack";
+    std::vector<std::string> stack_test_vec;
+    stack_test_vec.emplace_back(stack_test_str);
+    attr_map["op_callstack"] = stack_test_vec;
+    paddle::framework::InsertCallStackInfo("test", attr_map, &exception);
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find(stack_test_str) != std::string::npos);
+    EXPECT_TRUE(ex_msg.find("[operator < test > error]") != std::string::npos);
+  }
+}
+
+TEST(OpCallStack, AppendErrorOpHint) {
+  try {
+    paddle::framework::details::ThrowEnforceNotMet();
+  } catch (paddle::platform::EnforceNotMet &exception) {
+    paddle::framework::AppendErrorOpHint("test", &exception);
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find("[operator < test > error]") != std::string::npos);
+  }
+}
diff --git a/paddle/fluid/imperative/dygraph_grad_maker.h b/paddle/fluid/imperative/dygraph_grad_maker.h
index 757f419369030ef2027fa6639e0e3cc97e32c232..07a18a1a0dcd84dd78eb096733a5d6e90f46396e 100644
--- a/paddle/fluid/imperative/dygraph_grad_maker.h
+++ b/paddle/fluid/imperative/dygraph_grad_maker.h
@@ -258,6 +258,8 @@ class TracedGradOp {
     }
   }
 
+  std::string Type() const { return op_->Type(); }
+
   void SetType(const std::string& type) { op_->SetType(type); }
 
   void SetAttrMap(const framework::AttributeMap& attrs) {
diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc
index 873963db1a14444a3287771d19efcbee22454706..ee4c5617397b39d6847fecd1c884af8b0e14440f 100644
--- a/paddle/fluid/imperative/tracer.cc
+++ b/paddle/fluid/imperative/tracer.cc
@@ -53,7 +53,23 @@ void Tracer::TraceOp(const std::string& type, const NameVarBaseMap& ins,
     attr_checker->Check(&attrs, true);
   }
 
-  OpBase::Run(*op, ins, outs, attrs, place);
+  try {
+    OpBase::Run(*op, ins, outs, attrs, place);
+  } catch (platform::EnforceNotMet& exception) {
+    framework::AppendErrorOpHint(type, &exception);
+    throw std::move(exception);
+  } catch (std::exception& ex) {
+    PADDLE_THROW(platform::errors::Fatal(
+        "Operator %s raises an %s exception.\n"
+        "The exception content is\n:%s.",
+        type, platform::demangle(typeid(ex).name()), ex.what()));
+  } catch (...) {
+    // NOTE: this branch represents a very serious bug with
+    // low probability of occurrence, and we can't get its
+    // exception content here.
+    PADDLE_THROW(platform::errors::Fatal(
+        "Operator %s raises an unknown exception.", type));
+  }
 
   if (enable_program_desc_tracing_) {
     VLOG(5) << "Trace op " << type << " into ProgramDesc";