diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
index c663ba2f886809f55f8da09fe15d4477c82f99ea..cce835e6bc0354a23710874d7acb4f3a6195c1f2 100644
--- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
@@ -217,9 +217,10 @@ class ConvMKLDNNHandlerT
       const auto fwd_prop_kind = is_test ? mkldnn::prop_kind::forward_inference
                                          : mkldnn::prop_kind::forward_training;
 
-      float sum_scale;
+      float sum_scale = 1.0f;
       std::vector<float> output_shift_scale;
-      std::tie(sum_scale, output_shift_scale) = get_int8_scales(ctx);
+      if (platform::is_int8<T>())
+        std::tie(sum_scale, output_shift_scale) = get_int8_scales(ctx);
 
       const mkldnn::primitive_attr conv_attr = CreatePostOps(
           fuse_activation, fuse_alpha, fuse_beta, fuse_residual_conn,