diff --git a/paddle/fluid/inference/capi/pd_predictor.cc b/paddle/fluid/inference/capi/pd_predictor.cc
index d4d5e9c35f9ee5e06af8ea73e624b33d3c1a2a7a..114c6c0310303c5006c320ad4e587e3d5239ba82 100644
--- a/paddle/fluid/inference/capi/pd_predictor.cc
+++ b/paddle/fluid/inference/capi/pd_predictor.cc
@@ -46,7 +46,8 @@ inline void VisitDataType(PD_DataType type, Visitor visitor) {
 
   _DataType_(VisitDataTypeCallback);
 #undef VisitDataTypeCallback
-  PADDLE_THROW_ERROR("Unsupported data type. ");
+  PADDLE_THROW(
+      paddle::platform::errors::InvalidArgument("Unsupported data type."));
 }
 
 struct PD_ZeroCopyFunctor {
diff --git a/paddle/fluid/operators/distributed_ops/recv_save_op.cc b/paddle/fluid/operators/distributed_ops/recv_save_op.cc
index 766e378e106bc7cd47c375aef22aaa863ddc6aba..565e9f9886e5872e540d08484f22761d31ff7643 100644
--- a/paddle/fluid/operators/distributed_ops/recv_save_op.cc
+++ b/paddle/fluid/operators/distributed_ops/recv_save_op.cc
@@ -165,9 +165,8 @@ class RecvSaveOpKernel : public framework::OpKernel<T> {
     auto overwrite = ctx.Attr<bool>("overwrite");
 
     if (FileExists(filename) && !overwrite) {
-      PADDLE_THROW_ERROR(
-          "%s is existed, cannot save to it when overwrite=false", filename,
-          overwrite);
+      PADDLE_THROW(platform::errors::AlreadyExists(
+          "%s is existed, cannot save to it when overwrite=false", filename));
     }
 
     MkDirRecursively(DirName(filename).c_str());
diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h
index c558c5f91d3f5ba559ce9ccc64cef53c9a8f7554..d422b93cead0e4feafb8a6d7f501b0cffe71a83f 100644
--- a/paddle/fluid/platform/enforce.h
+++ b/paddle/fluid/platform/enforce.h
@@ -223,6 +223,13 @@ inline void throw_on_error(const platform::ErrorSummary& error) {
 #endif
 }
 
+// Note: This Macro can only be used within enforce.h
+#define __THROW_ERROR_INTERNAL__(...)                                \
+  do {                                                               \
+    throw ::paddle::platform::EnforceNotMet(                         \
+        ::paddle::string::Sprintf(__VA_ARGS__), __FILE__, __LINE__); \
+  } while (0)
+
 /** ENFORCE EXCEPTION AND MACROS **/
 
 struct EnforceNotMet : public std::exception {
@@ -251,12 +258,6 @@ struct EnforceNotMet : public std::exception {
         ::paddle::platform::ErrorSummary(__VA_ARGS__), __FILE__, __LINE__); \
   } while (0)
 
-#define PADDLE_THROW_ERROR(...)                                      \
-  do {                                                               \
-    throw ::paddle::platform::EnforceNotMet(                         \
-        ::paddle::string::Sprintf(__VA_ARGS__), __FILE__, __LINE__); \
-  } while (0)
-
 #if defined(__CUDA_ARCH__)
 // For cuda, the assertions can affect performance and it is therefore
 // recommended to disable them in production code
@@ -298,10 +299,11 @@ struct EnforceNotMet : public std::exception {
  *    extra messages is also supported, for example:
  *    PADDLE_ENFORCE(a, b, "some simple enforce failed between %d numbers", 2)
  */
+
 #define PADDLE_ENFORCE_NOT_NULL(__VAL, ...)                          \
   do {                                                               \
     if (UNLIKELY(nullptr == (__VAL))) {                              \
-      PADDLE_THROW_ERROR(                                            \
+      __THROW_ERROR_INTERNAL__(                                      \
           "%s\n  [Hint: " #__VAL " should not be null.]",            \
           ::paddle::platform::ErrorSummary(__VA_ARGS__).ToString()); \
     }                                                                \
@@ -323,7 +325,7 @@ struct EnforceNotMet : public std::exception {
       constexpr bool __kCanToString__ =                                        \
           ::paddle::platform::details::CanToString<__TYPE1__>::kValue &&       \
           ::paddle::platform::details::CanToString<__TYPE2__>::kValue;         \
-      PADDLE_THROW_ERROR(                                                      \
+      __THROW_ERROR_INTERNAL__(                                                \
           "%s\n  [Hint: Expected %s " #__CMP                                   \
           " %s, but received %s " #__INV_CMP " %s.]",                          \
           ::paddle::platform::ErrorSummary(__VA_ARGS__).ToString(), #__VAL1,   \