diff --git a/paddle/fluid/operators/conv_mkldnn_op.cc b/paddle/fluid/operators/conv_mkldnn_op.cc
index aa99658acc04733bed415e541c903292a04c110f..e9c2deb6ba51b80a5a5978fde57dbf25446a7f86 100644
--- a/paddle/fluid/operators/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/conv_mkldnn_op.cc
@@ -785,7 +785,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
     pipeline.push_back(*conv_p);
     stream(stream::kind::eager).submit(pipeline).wait();
 
-    if(need_s8_to_u8){
+    if(need_s8_to_u8 && !force_fp32_output){
         output->mutable_data<uint8_t>(ctx.GetPlace());
     }