diff --git a/paddle/fluid/framework/ir/mkldnn/quant_dequant_mkldnn_pass.cc b/paddle/fluid/framework/ir/mkldnn/quant_dequant_mkldnn_pass.cc
index 808d043a4b226c8b49945775ef3987a180fdc029..28363f8f167609db9f893f94e8b78bb9fa34dd69 100644
--- a/paddle/fluid/framework/ir/mkldnn/quant_dequant_mkldnn_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/quant_dequant_mkldnn_pass.cc
@@ -354,10 +354,9 @@ bool QuantDequantMkldnnPass::IsInt8Weight(
   auto* op_desc = op_node->Op();
   auto var_name = op_desc->Input(weight_name)[0];
   auto* var = scope->FindVar(var_name);
-  PADDLE_ENFORCE_NOT_NULL(
-      var, platform::errors::NotFound(
-               "The input persistable [%s] var of [%s] op is not found.",
-               var_name, op_desc->Type()));
+  if (var == nullptr) {
+    return false;
+  }
   auto* weight_tensor = var->GetMutable<LoDTensor>();
   auto* weight_data = weight_tensor->data<float>();
   bool is_int8 = true;