diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
index ac9ad7937a49ed249989e2bf36afba5305fdf451..1e23539c80fb9b1e89a7f5ffc38adddfabbab67d 100644
--- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
@@ -177,7 +177,10 @@ void CPUQuantizeSquashPass::ConvDequantSquash(Graph* graph) const {
     GET_IR_NODE_FROM_SUBGRAPH(dequant_out, dequant_out, conv_dequant_pattern);
 
     // if conv2d has one output
-    if (conv_out->outputs.size() == 1) {
+    // and there is no fuse residual connection
+    // because residual fusion does not support force output with fp32
+    if (conv_out->outputs.size() == 1 &&
+        !(conv_op->Op()->GetAttrIfExists<bool>("fuse_residual_connection"))) {
       conv_op->Op()->SetAttr("force_fp32_output", true);
       conv_op->Op()->SetOutput("Output",
                                std::vector<std::string>({dequant_out->Name()}));