diff --git a/paddle/fluid/operators/conv_mkldnn_op.cc b/paddle/fluid/operators/conv_mkldnn_op.cc
index 05f44e1c0b65f9b9cccf794f21cb4a0f43551328..385e1a798fb2b78e5970746b324c9eaa3d3483ff 100644
--- a/paddle/fluid/operators/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/conv_mkldnn_op.cc
@@ -511,8 +511,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
             src_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                 src_tz, memory::data_type::u8, chosen_memory_format)));
             weights_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
-                weights_tz, memory::data_type::s8,
-                (g == 1) ? chosen_memory_format : mkldnn::memory::format::goihw)));
+                weights_tz, memory::data_type::s8, chosen_memory_format)));
             auto dst_dt = fuse_relu? paddle::framework::ToMKLDNNDataType(std::type_index(typeid(unsigned char))) : paddle::framework::ToMKLDNNDataType(std::type_index(typeid(signed char)));
             if(fuse_residual_conn){
                 auto residual = ctx.Input<Tensor>("ResidualData");