diff --git a/src/operators/feed_op.cpp b/src/operators/feed_op.cpp
index bff02499e61ac9f3f2d88bd8b9f1e88d371060a1..d03f5c3ecb8afd47e2c3b82fdc8935f27b3a45f1 100644
--- a/src/operators/feed_op.cpp
+++ b/src/operators/feed_op.cpp
@@ -23,11 +23,7 @@ void FeedOp<DeviceType, T>::InferShape() const {
   out_dims[0] = this->param_.BatchSize();
   int col = this->param_.Col();
   auto input_dims = this->param_.InputX()->at(col).dims();
-  if (input_dims.size() == 4) {
-    this->param_.Out()->Resize(input_dims);
-  } else {
-    this->param_.Out()->Resize(out_dims);
-  }
+  this->param_.Out()->Resize(input_dims);
 }
 
 }  // namespace operators